Merge remote-tracking branch 'origin/develop' into state-history-plugin

contracts/eosio.bios/eosio.bios.abi

@@ -53,6 +53,28 @@
         {"name":"accounts",  "type":"permission_level_weight[]"},
         {"name":"waits",     "type":"wait_weight[]"}
       ]
+    },{
+      "name": "blockchain_parameters",
+      "base": "",
+      "fields": [
+         {"name":"max_block_net_usage",                 "type":"uint64"},
+         {"name":"target_block_net_usage_pct",          "type":"uint32"},
+         {"name":"max_transaction_net_usage",           "type":"uint32"},
+         {"name":"base_per_transaction_net_usage",      "type":"uint32"},
+         {"name":"net_usage_leeway",                    "type":"uint32"},
+         {"name":"context_free_discount_net_usage_num", "type":"uint32"},
+         {"name":"context_free_discount_net_usage_den", "type":"uint32"},
+         {"name":"max_block_cpu_usage",                 "type":"uint32"},
+         {"name":"target_block_cpu_usage_pct",          "type":"uint32"},
+         {"name":"max_transaction_cpu_usage",           "type":"uint32"},
+         {"name":"min_transaction_cpu_usage",           "type":"uint32"},
+         {"name":"max_transaction_lifetime",            "type":"uint32"},
+         {"name":"deferred_trx_expiration_window",      "type":"uint32"},
+         {"name":"max_transaction_delay",               "type":"uint32"},
+         {"name":"max_inline_action_size",              "type":"uint32"},
+         {"name":"max_inline_action_depth",             "type":"uint16"},
+         {"name":"max_authority_depth",                 "type":"uint16"}
+      ]
     },{
       "name": "newaccount",
       "base": "",
@@ -160,6 +182,12 @@
       "fields": [
         {"name":"schedule",   "type":"producer_key[]"}
       ]
+    },{
+      "name": "setparams",
+      "base": "",
+      "fields": [
+        {"name":"params", "type":"blockchain_parameters"}
+      ]
     },{
       "name": "require_auth",
       "base": "",
@@ -219,6 +247,10 @@
       "name": "setprods",
       "type": "set_producers",
       "ricardian_contract": ""
+    },{
+      "name": "setparams",
+      "type": "setparams",
+      "ricardian_contract": ""
     },{
       "name": "reqauth",
       "type": "require_auth",

contracts/eosio.bios/eosio.bios.cpp

 #include <eosio.bios/eosio.bios.hpp>
 
-EOSIO_ABI( eosio::bios, (setpriv)(setalimits)(setglimits)(setprods)(reqauth) )
+EOSIO_ABI( eosio::bios, (setpriv)(setalimits)(setglimits)(setprods)(setparams)(reqauth) )

contracts/eosio.bios/eosio.bios.hpp

@@ -34,6 +34,11 @@ namespace eosio {
             set_proposed_producers(buffer, size);
          }
 
+         void setparams( const eosio::blockchain_parameters& params ) {
+            require_auth( _self );
+            set_blockchain_parameters( params );
+         }
+
          void reqauth( action_name from ) {
             require_auth( from );
          }

eosio_build.sh

@@ -258,7 +258,7 @@
       -DCMAKE_C_COMPILER="${C_COMPILER}" -DWASM_ROOT="${WASM_ROOT}" -DCORE_SYMBOL_NAME="${CORE_SYMBOL_NAME}" \
       -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" -DBUILD_MONGO_DB_PLUGIN=true \
       -DENABLE_COVERAGE_TESTING="${ENABLE_COVERAGE_TESTING}" -DBUILD_DOXYGEN="${DOXYGEN}" \
-      -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" "${SOURCE_DIR}"
+      -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" ${LOCAL_CMAKE_FLAGS} "${SOURCE_DIR}"
    then
       printf "\\n\\t>>>>>>>>>>>>>>>>>>>> CMAKE building EOSIO has exited with the above error.\\n\\n"
       exit -1

libraries/chain/apply_context.cpp

@@ -272,8 +272,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
    }
 
    uint32_t trx_size = 0;
-   auto& d = control.db();
-   if ( auto ptr = d.find<generated_transaction_object,by_sender_id>(boost::make_tuple(receiver, sender_id)) ) {
+   if ( auto ptr = db.find<generated_transaction_object,by_sender_id>(boost::make_tuple(receiver, sender_id)) ) {
       EOS_ASSERT( replace_existing, deferred_tx_duplicate, "deferred transaction with the same sender_id and payer already exists" );
 
       // TODO: Remove the following subjective check when the deferred trx replacement RAM bug has been fixed with a hard fork.
@@ -283,7 +282,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
       // TODO: The logic of the next line needs to be incorporated into the next hard fork.
       // add_ram_usage( ptr->payer, -(config::billable_size_v<generated_transaction_object> + ptr->packed_trx.size()) );
 
-      d.modify<generated_transaction_object>( *ptr, [&]( auto& gtx ) {
+      db.modify<generated_transaction_object>( *ptr, [&]( auto& gtx ) {
             gtx.sender      = receiver;
             gtx.sender_id   = sender_id;
             gtx.payer       = payer;
@@ -294,7 +293,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
             trx_size = gtx.set( trx );
          });
    } else {
-      d.create<generated_transaction_object>( [&]( auto& gtx ) {
+      db.create<generated_transaction_object>( [&]( auto& gtx ) {
             gtx.trx_id      = trx.id();
             gtx.sender      = receiver;
             gtx.sender_id   = sender_id;

libraries/chain/controller.cpp

@@ -275,9 +275,10 @@ struct controller_impl {
             std::cerr<< "\n";
             ilog( "${n} blocks replayed", ("n", head->block_num) );
 
-            // the irreverible log is played without undo sessions enabled, so we need to sync the
+            // if the irreverible log is played without undo sessions enabled, we need to sync the
             // revision ordinal to the appropriate expected value here.
-            db.set_revision(head->block_num);
+            if( self.skip_db_sessions( controller::block_status::irreversible ) )
+               db.set_revision(head->block_num);
 
             int rev = 0;
             while( auto obj = reversible_blocks.find<reversible_block_object,by_num>(head->block_num+1) ) {
@@ -1051,7 +1052,7 @@ struct controller_impl {
          // on replay irreversible is not emitted by fork database, so emit it explicitly here
          if( s == controller::block_status::irreversible )
             emit( self.irreversible_block, new_header_state );
-            
+
       } FC_LOG_AND_RETHROW( )
    }
 
@@ -1387,12 +1388,11 @@ controller::~controller() {
    my->fork_db.close();
 }
 
-
-void controller::startup() {
-
-   // ilog( "${c}", ("c",fc::json::to_pretty_string(cfg)) );
+void controller::add_indices() {
    my->add_indices();
+}
 
+void controller::startup() {
    my->head = my->fork_db.head();
    if( !my->head ) {
       elog( "No head block in fork db, perhaps we need to replay" );
@@ -1400,9 +1400,11 @@ void controller::startup() {
    my->init();
 }
 
-chainbase::database& controller::db()const { return my->db; }
+const chainbase::database& controller::db()const { return my->db; }
+
+chainbase::database& controller::mutable_db()const { return my->db; }
 
-fork_database& controller::fork_db()const { return my->fork_db; }
+const fork_database& controller::fork_db()const { return my->fork_db; }
 
 
 void controller::start_block( block_timestamp_type when, uint16_t confirm_block_count) {

libraries/chain/include/eosio/chain/apply_context.hpp

@@ -453,7 +453,7 @@ class apply_context {
    public:
       apply_context(controller& con, transaction_context& trx_ctx, const action& a, uint32_t depth=0)
       :control(con)
-      ,db(con.db())
+      ,db(con.mutable_db())
       ,trx_context(trx_ctx)
       ,act(a)
       ,receiver(act.account)

libraries/chain/include/eosio/chain/controller.hpp

@@ -87,6 +87,7 @@ namespace eosio { namespace chain {
          controller( const config& cfg );
          ~controller();
 
+         void add_indices();
          void startup();
 
          /**
@@ -145,9 +146,9 @@ namespace eosio { namespace chain {
           */
          void push_confirmation( const header_confirmation& c );
 
-         chainbase::database& db()const;
+         const chainbase::database& db()const;
 
-         fork_database& fork_db()const;
+         const fork_database& fork_db()const;
 
          const account_object&                 get_account( account_name n )const;
          const global_property_object&         get_global_properties()const;
@@ -285,6 +286,10 @@ namespace eosio { namespace chain {
          }
 
       private:
+         friend class apply_context;
+         friend class transaction_context;
+
+         chainbase::database& mutable_db()const;
 
          std::unique_ptr<controller_impl> my;
 

libraries/chain/transaction_context.cpp

@@ -23,7 +23,7 @@ namespace eosio { namespace chain {
    ,pseudo_start(s)
    {
       if (!c.skip_db_sessions()) {
-         undo_session = c.db().start_undo_session(true);
+         undo_session = c.mutable_db().start_undo_session(true);
       }
       trace->id = id;
       trace->block_num = c.pending_block_state()->block_num;
@@ -451,7 +451,7 @@ namespace eosio { namespace chain {
       auto first_auth = trx.first_authorizor();
 
       uint32_t trx_size = 0;
-      const auto& cgto = control.db().create<generated_transaction_object>( [&]( auto& gto ) {
+      const auto& cgto = control.mutable_db().create<generated_transaction_object>( [&]( auto& gto ) {
         gto.trx_id      = id;
         gto.payer       = first_auth;
         gto.sender      = account_name(); /// delayed transactions have no sender
@@ -467,7 +467,7 @@ namespace eosio { namespace chain {
 
    void transaction_context::record_transaction( const transaction_id_type& id, fc::time_point_sec expire ) {
       try {
-          control.db().create<transaction_object>([&](transaction_object& transaction) {
+          control.mutable_db().create<transaction_object>([&](transaction_object& transaction) {
               transaction.trx_id = id;
               transaction.expiration = expire;
           });

libraries/testing/include/eosio/testing/tester.hpp

@@ -346,6 +346,7 @@ namespace eosio { namespace testing {
          vcfg.trusted_producers = trusted_producers;
 
          validating_node = std::make_unique<controller>(vcfg);
+         validating_node->add_indices();
          validating_node->startup();
 
          init(true);
@@ -360,6 +361,7 @@ namespace eosio { namespace testing {
          vcfg.state_dir  = vcfg.state_dir.parent_path() / std::string("v_").append( vcfg.state_dir.filename().generic_string() );
 
          validating_node = std::make_unique<controller>(vcfg);
+         validating_node->add_indices();
          validating_node->startup();
 
          init(config);
@@ -404,6 +406,7 @@ namespace eosio { namespace testing {
 
         validating_node.reset();
         validating_node = std::make_unique<controller>(vcfg);
+        validating_node->add_indices();
         validating_node->startup();
 
         return ok;

libraries/testing/tester.cpp

@@ -17,7 +17,7 @@ namespace eosio { namespace testing {
       std::ifstream wast_file(fn);
       FC_ASSERT( wast_file.is_open(), "wast file cannot be found" );
       wast_file.seekg(0, std::ios::end);
-      std::vector<char> wast; 
+      std::vector<char> wast;
       int len = wast_file.tellg();
       FC_ASSERT( len >= 0, "wast file length is -1" );
       wast.resize(len+1);
@@ -32,7 +32,7 @@ namespace eosio { namespace testing {
       std::ifstream wasm_file(fn, std::ios::binary);
       FC_ASSERT( wasm_file.is_open(), "wasm file cannot be found" );
       wasm_file.seekg(0, std::ios::end);
-      std::vector<uint8_t> wasm; 
+      std::vector<uint8_t> wasm;
       int len = wasm_file.tellg();
       FC_ASSERT( len >= 0, "wasm file length is -1" );
       wasm.resize(len);
@@ -46,7 +46,7 @@ namespace eosio { namespace testing {
       std::ifstream abi_file(fn);
       FC_ASSERT( abi_file.is_open(), "abi file cannot be found" );
       abi_file.seekg(0, std::ios::end);
-      std::vector<char> abi; 
+      std::vector<char> abi;
       int len = abi_file.tellg();
       FC_ASSERT( len >= 0, "abi file length is -1" );
       abi.resize(len+1);
@@ -123,6 +123,7 @@ namespace eosio { namespace testing {
 
    void base_tester::open() {
       control.reset( new controller(cfg) );
+      control->add_indices();
       control->startup();
       chain_transactions.clear();
       control->accepted_block.connect([this]( const block_state_ptr& block_state ){

plugins/chain_plugin/chain_plugin.cpp

@@ -643,6 +643,8 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
             [this]( const header_confirmation& conf ) {
                my->accepted_confirmation_channel.publish( conf );
             } );
+
+      my->chain->add_indices();
    } FC_LOG_AND_RETHROW()
 
 }
@@ -906,11 +908,6 @@ bool chain_plugin::export_reversible_blocks( const fc::path& reversible_dir,
    return (end >= start) && ((end - start + 1) == num);
 }
 
-controller::config& chain_plugin::chain_config() {
-   // will trigger optional assert if called before/after plugin_initialize()
-   return *my->chain_config;
-}
-
 controller& chain_plugin::chain() { return *my->chain; }
 const controller& chain_plugin::chain() const { return *my->chain; }
 

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -642,11 +642,9 @@ public:
                                         const fc::path& reversible_blocks_file
                                        );
 
-   // Only call this in plugin_initialize() to modify controller constructor configuration
-   controller::config& chain_config();
-   // Only call this after plugin_startup()!
+   // Only call this after plugin_initialize()!
    controller& chain();
-   // Only call this after plugin_startup()!
+   // Only call this after plugin_initialize()!
    const controller& chain() const;
 
    chain::chain_id_type get_chain_id() const;

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -36,7 +36,7 @@ void db_size_api_plugin::plugin_startup() {
 }
 
 db_size_stats db_size_api_plugin::get() {
-   chainbase::database& db = app().get_plugin<chain_plugin>().chain().db();
+   const chainbase::database& db = app().get_plugin<chain_plugin>().chain().db();
    db_size_stats ret;
 
    ret.free_bytes = db.get_segment_manager()->get_free_memory();

plugins/history_plugin/history_plugin.cpp

@@ -206,7 +206,7 @@ namespace eosio {
 
          void record_account_action( account_name n, const base_action_trace& act ) {
             auto& chain = chain_plug->chain();
-            auto& db = chain.db();
+            chainbase::database& db = const_cast<chainbase::database&>( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!)
 
             const auto& idx = db.get_index<account_history_index, by_account_action_seq>();
             auto itr = idx.lower_bound( boost::make_tuple( name(n.value+1), 0 ) );
@@ -227,7 +227,7 @@ namespace eosio {
 
          void on_system_action( const action_trace& at ) {
             auto& chain = chain_plug->chain();
-            auto& db = chain.db();
+            chainbase::database& db = const_cast<chainbase::database&>( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!)
             if( at.act.name == N(newaccount) )
             {
                const auto create = at.act.data_as<chain::newaccount>();
@@ -256,7 +256,7 @@ namespace eosio {
             if( filter( at ) ) {
                //idump((fc::json::to_pretty_string(at)));
                auto& chain = chain_plug->chain();
-               auto& db = chain.db();
+               chainbase::database& db = const_cast<chainbase::database&>( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!)
 
                db.create<action_history_object>( [&]( auto& aho ) {
                   auto ps = fc::raw::pack_size( at );
@@ -344,10 +344,12 @@ namespace eosio {
          EOS_ASSERT( my->chain_plug, chain::missing_chain_plugin_exception, ""  );
          auto& chain = my->chain_plug->chain();
 
-         chain.db().add_index<account_history_index>();
-         chain.db().add_index<action_history_index>();
-         chain.db().add_index<account_control_history_multi_index>();
-         chain.db().add_index<public_key_history_multi_index>();
+         chainbase::database& db = const_cast<chainbase::database&>( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!)
+         // TODO: Use separate chainbase database for managing the state of the history_plugin (or remove deprecated history_plugin entirely) 
+         db.add_index<account_history_index>();
+         db.add_index<action_history_index>();
+         db.add_index<account_control_history_multi_index>();
+         db.add_index<public_key_history_multi_index>();
 
          my->applied_transaction_connection.emplace(
                chain.applied_transaction.connect( [&]( const transaction_trace_ptr& p ) {

plugins/producer_plugin/producer_plugin.cpp

@@ -514,9 +514,9 @@ void producer_plugin::set_program_options(
          ("greylist-account", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "account that can not access to extended CPU/NET virtual resources")
          ("produce-time-offset-us", boost::program_options::value<int32_t>()->default_value(0),
-          "offset of non last block producing time in micro second. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later")
+          "offset of non last block producing time in microseconds. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later")
          ("last-block-time-offset-us", boost::program_options::value<int32_t>()->default_value(0),
-          "offset of last block producing time in micro second. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later")
+          "offset of last block producing time in microseconds. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later")
          ("incoming-defer-ratio", bpo::value<double>()->default_value(1.0),
           "ratio between incoming transations and deferred transactions when both are exhausted")
          ;

plugins/txn_test_gen_plugin/README.md

@@ -56,23 +56,24 @@ $ ./nodeos -d ~/eos.data/producer_node --config-dir ~/eos.data/producer_node -l
 
 ### Launch non-producer that will generate transactions
 ```bash
-$ ./nodeos -d ~/eos.data/generator_node --config-dir ~/eos.data/generator_node -l ~/eos.data/logging.json --plugin eosio::txn_test_gen_plugin --plugin eosio::wallet_api_plugin --plugin eosio::chain_api_plugin --p2p-peer-address localhost:9876 --p2p-listen-endpoint localhost:5555
+$ ./nodeos -d ~/eos.data/generator_node --config-dir ~/eos.data/generator_node -l ~/eos.data/logging.json --plugin eosio::txn_test_gen_plugin --plugin eosio::chain_api_plugin --p2p-peer-address localhost:9876 --p2p-listen-endpoint localhost:5555
 ```
 
 ### Create a wallet on the non-producer and set bios contract
 ```bash
-$ ./cleos wallet create
+$ ./cleos wallet create --to-console
+$ ./cleos wallet import --private-key 5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3
 $ ./cleos set contract eosio ~/eos/build.release/contracts/eosio.bios/ 
 ```
 
 ### Initialize the accounts txn_test_gen_plugin uses
 ```bash
-$ curl --data-binary '["eosio", "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3"]' http://localhost:8888/v1/txn_test_gen/create_test_accounts
+$ curl --data-binary '["eosio", "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3"]' http://127.0.0.1:8888/v1/txn_test_gen/create_test_accounts
 ```
 
 ### Start transaction generation, this will submit 20 transactions evey 20ms (total of 1000TPS)
 ```bash
-$ curl --data-binary '["", 20, 20]' http://localhost:8888/v1/txn_test_gen/start_generation
+$ curl --data-binary '["", 20, 20]' http://127.0.0.1:8888/v1/txn_test_gen/start_generation
 ```
 
 ### Note the producer console prints
@@ -83,3 +84,6 @@ eosio generated block b243aeaa... #3221 @ 2018-04-25T16:07:48.000 with 500 trxs,
 ```
 
 Note in the console output there are 500 transactions in each of the blocks which are produced every 500 ms yielding 1,000 transactions / second.
+
+### Demonstration
+The following video provides a demo: https://vimeo.com/266585781

plugins/wallet_plugin/se_wallet.cpp

@@ -300,7 +300,7 @@ se_wallet::se_wallet() : my(new detail::se_wallet_impl()) {
       }
       unsigned int major, minor;
       if(sscanf(model, "MacBookPro%u,%u", &major, &minor) == 2) {
-         if(major >= 13 && minor >= 2) {
+         if((major >= 15) || (major >= 13 && minor >= 2)) {
             my->populate_existing_keys();
             return;
          }

programs/cleos/main.cpp

@@ -890,6 +890,7 @@ struct create_account_subcommand {
    string stake_net;
    string stake_cpu;
    uint32_t buy_ram_bytes_in_kbytes = 0;
+   uint32_t buy_ram_bytes = 0;
    string buy_ram_eos;
    bool transfer;
    bool simple;
@@ -907,7 +908,9 @@ struct create_account_subcommand {
          createAccount->add_option("--stake-cpu", stake_cpu,
                                    (localized("The amount of EOS delegated for CPU bandwidth")))->required();
          createAccount->add_option("--buy-ram-kbytes", buy_ram_bytes_in_kbytes,
-                                   (localized("The amount of RAM bytes to purchase for the new account in kibibytes (KiB), default is 8 KiB")));
+                                   (localized("The amount of RAM bytes to purchase for the new account in kibibytes (KiB)")));
+         createAccount->add_option("--buy-ram-bytes", buy_ram_bytes,
+                                   (localized("The amount of RAM bytes to purchase for the new account in bytes")));
          createAccount->add_option("--buy-ram", buy_ram_eos,
                                    (localized("The amount of RAM bytes to purchase for the new account in EOS")));
          createAccount->add_flag("--transfer", transfer,
@@ -928,12 +931,10 @@ struct create_account_subcommand {
             } EOS_RETHROW_EXCEPTIONS(public_key_type_exception, "Invalid active public key: ${public_key}", ("public_key", active_key_str));
             auto create = create_newaccount(creator, account_name, owner_key, active_key);
             if (!simple) {
-               if ( buy_ram_eos.empty() && buy_ram_bytes_in_kbytes == 0) {
-                  std::cerr << "ERROR: Either --buy-ram or --buy-ram-kbytes with non-zero value is required" << std::endl;
-                  return;
-               }
+               EOSC_ASSERT( buy_ram_eos.size() || buy_ram_bytes_in_kbytes || buy_ram_bytes, "ERROR: One of --buy-ram, --buy-ram-kbytes or --buy-ram-bytes should have non-zero value" );
+               EOSC_ASSERT( !buy_ram_bytes_in_kbytes || !buy_ram_bytes, "ERROR: --buy-ram-kbytes and --buy-ram-bytes cannot be set at the same time" );
                action buyram = !buy_ram_eos.empty() ? create_buyram(creator, account_name, to_asset(buy_ram_eos))
-                  : create_buyrambytes(creator, account_name, buy_ram_bytes_in_kbytes * 1024);
+                  : create_buyrambytes(creator, account_name, (buy_ram_bytes_in_kbytes) ? (buy_ram_bytes_in_kbytes * 1024) : buy_ram_bytes);
                auto net = to_asset(stake_net);
                auto cpu = to_asset(stake_cpu);
                if ( net.get_amount() != 0 || cpu.get_amount() != 0 ) {
@@ -1194,6 +1195,7 @@ struct delegate_bandwidth_subcommand {
    string stake_cpu_amount;
    string stake_storage_amount;
    string buy_ram_amount;
+   uint32_t buy_ram_bytes = 0;
    bool transfer = false;
 
    delegate_bandwidth_subcommand(CLI::App* actionRoot) {
@@ -1203,6 +1205,7 @@ struct delegate_bandwidth_subcommand {
       delegate_bandwidth->add_option("stake_net_quantity", stake_net_amount, localized("The amount of EOS to stake for network bandwidth"))->required();
       delegate_bandwidth->add_option("stake_cpu_quantity", stake_cpu_amount, localized("The amount of EOS to stake for CPU bandwidth"))->required();
       delegate_bandwidth->add_option("--buyram", buy_ram_amount, localized("The amount of EOS to buyram"));
+      delegate_bandwidth->add_option("--buy-ram-bytes", buy_ram_bytes, localized("The amount of RAM to buy in number of bytes"));
       delegate_bandwidth->add_flag("--transfer", transfer, localized("Transfer voting power and right to unstake EOS to receiver"));
       add_standard_transaction_options(delegate_bandwidth);
 
@@ -1214,12 +1217,11 @@ struct delegate_bandwidth_subcommand {
                   ("stake_cpu_quantity", to_asset(stake_cpu_amount))
                   ("transfer", transfer);
          std::vector<chain::action> acts{create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(delegatebw), act_payload)};
-         if (buy_ram_amount.length()) {
-            fc::variant act_payload2 = fc::mutable_variant_object()
-               ("payer", from_str)
-               ("receiver", receiver_str)
-               ("quant", to_asset(buy_ram_amount));
-            acts.push_back(create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyram), act_payload2));
+         EOSC_ASSERT( !(buy_ram_amount.size()) || !buy_ram_bytes, "ERROR: --buyram and --buy-ram-bytes cannot be set at the same time" );
+         if (buy_ram_amount.size()) {
+            acts.push_back( create_buyram(from_str, receiver_str, to_asset(buy_ram_amount)) );
+         } else if (buy_ram_bytes) {
+            acts.push_back( create_buyrambytes(from_str, receiver_str, buy_ram_bytes) );
          }
          send_actions(std::move(acts));
       });
@@ -1347,27 +1349,22 @@ struct buyram_subcommand {
    string receiver_str;
    string amount;
    bool kbytes = false;
+   bool bytes = false;
 
    buyram_subcommand(CLI::App* actionRoot) {
       auto buyram = actionRoot->add_subcommand("buyram", localized("Buy RAM"));
       buyram->add_option("payer", from_str, localized("The account paying for RAM"))->required();
       buyram->add_option("receiver", receiver_str, localized("The account receiving bought RAM"))->required();
-      buyram->add_option("amount", amount, localized("The amount of EOS to pay for RAM, or number of kbytes of RAM if --kbytes is set"))->required();
-      buyram->add_flag("--kbytes,-k", kbytes, localized("buyram in number of kbytes"));
+      buyram->add_option("amount", amount, localized("The amount of EOS to pay for RAM, or number of bytes/kibibytes of RAM if --bytes/--kbytes is set"))->required();
+      buyram->add_flag("--kbytes,-k", kbytes, localized("buyram in number of kibibytes (KiB)"));
+      buyram->add_flag("--bytes,-b", bytes, localized("buyram in number of bytes"));
       add_standard_transaction_options(buyram);
       buyram->set_callback([this] {
-         if (kbytes) {
-            fc::variant act_payload = fc::mutable_variant_object()
-                  ("payer", from_str)
-                  ("receiver", receiver_str)
-                  ("bytes", fc::to_uint64(amount) * 1024ull);
-            send_actions({create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyrambytes), act_payload)});
+         EOSC_ASSERT( !kbytes || !bytes, "ERROR: --kbytes and --bytes cannot be set at the same time" );
+         if (kbytes || bytes) {
+            send_actions( { create_buyrambytes(from_str, receiver_str, fc::to_uint64(amount) * ((kbytes) ? 1024ull : 1ull)) } );
          } else {
-            fc::variant act_payload = fc::mutable_variant_object()
-               ("payer", from_str)
-               ("receiver", receiver_str)
-               ("quant", to_asset(amount));
-            send_actions({create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyram), act_payload)});
+            send_actions( { create_buyram(from_str, receiver_str, to_asset(amount)) } );
          }
       });
    }

programs/eosio-launcher/main.cpp

@@ -26,6 +26,7 @@
 #include <fc/crypto/private_key.hpp>
 #include <fc/crypto/public_key.hpp>
 #include <fc/io/json.hpp>
+#include <fc/optional.hpp>
 #include <fc/network/ip.hpp>
 #include <fc/reflect/variant.hpp>
 #include <fc/log/logger_config.hpp>
@@ -387,63 +388,65 @@ string producer_names::producer_name(unsigned int producer_number) {
 }
 
 struct launcher_def {
-  bool force_overwrite;
-  size_t total_nodes;
-  size_t prod_nodes;
-  size_t producers;
-  size_t next_node;
-  string shape;
-  p2p_plugin p2p;
-  allowed_connection allowed_connections = PC_NONE;
-  bfs::path genesis;
-  bfs::path output;
-  bfs::path host_map_file;
-  bfs::path server_ident_file;
-  bfs::path stage;
-
-  string erd;
-  bfs::path config_dir_base;
-  bfs::path data_dir_base;
-  bool skip_transaction_signatures = false;
-  string eosd_extra_args;
-  std::map<uint,string> specific_nodeos_args;
-  testnet_def network;
-  string gelf_endpoint;
-  vector <string> aliases;
-  vector <host_def> bindings;
-  int per_host = 0;
-  last_run_def last_run;
-  int start_delay = 0;
-  bool gelf_enabled;
-  bool nogen;
-  bool boot;
-  bool add_enable_stale_production = false;
-  string launch_name;
-  string launch_time;
-  server_identities servers;
-  producer_set_def producer_set;
-   vector<string> genesis_block;
+   bool force_overwrite;
+   size_t total_nodes;
+   size_t prod_nodes;
+   size_t producers;
+   size_t next_node;
+   string shape;
+   p2p_plugin p2p;
+   allowed_connection allowed_connections = PC_NONE;
+   bfs::path genesis;
+   bfs::path output;
+   bfs::path host_map_file;
+   bfs::path server_ident_file;
+   bfs::path stage;
+
+   string erd;
+   bfs::path config_dir_base;
+   bfs::path data_dir_base;
+   bool skip_transaction_signatures = false;
+   string eosd_extra_args;
+   std::map<uint,string> specific_nodeos_args;
+   testnet_def network;
+   string gelf_endpoint;
+   vector <string> aliases;
+   vector <host_def> bindings;
+   int per_host = 0;
+   last_run_def last_run;
+   int start_delay = 0;
+   bool gelf_enabled;
+   bool nogen;
+   bool boot;
+   bool add_enable_stale_production = false;
+   string launch_name;
+   string launch_time;
+   server_identities servers;
+   producer_set_def producer_set;
    string start_temp;
    string start_script;
+   fc::optional<uint32_t> max_block_cpu_usage;
+   fc::optional<uint32_t> max_transaction_cpu_usage;
+   eosio::chain::genesis_state genesis_from_file;
 
    void assign_name (eosd_def &node, bool is_bios);
 
-  void set_options (bpo::options_description &cli);
-  void initialize (const variables_map &vmap);
+   void set_options (bpo::options_description &cli);
+   void initialize (const variables_map &vmap);
    void init_genesis ();
-  void load_servers ();
-  bool generate ();
-  void define_network ();
-  void bind_nodes ();
-  host_def *find_host (const string &name);
-  host_def *find_host_by_name_or_address (const string &name);
-  host_def *deploy_config_files (tn_node_def &node);
-  string compose_scp_command (const host_def &host, const bfs::path &source,
-                              const bfs::path &destination);
-  void write_config_file (tn_node_def &node);
-  void write_logging_config_file (tn_node_def &node);
-  void write_genesis_file (tn_node_def &node);
-  void write_setprods_file ();
+   void load_servers ();
+   bool generate ();
+   void define_network ();
+   void bind_nodes ();
+   host_def *find_host (const string &name);
+   host_def *find_host_by_name_or_address (const string &name);
+   host_def *deploy_config_files (tn_node_def &node);
+   string compose_scp_command (const host_def &host, const bfs::path &source,
+                               const bfs::path &destination);
+   void write_config_file (tn_node_def &node);
+   void write_logging_config_file (tn_node_def &node);
+   void write_genesis_file (tn_node_def &node);
+   void write_setprods_file ();
    void write_bios_boot ();
 
    bool   is_bios_ndx (size_t ndx);
@@ -451,25 +454,25 @@ struct launcher_def {
    bool   next_ndx(size_t &ndx);
    size_t skip_ndx (size_t from, size_t offset);
 
-  void make_ring ();
-  void make_star ();
-  void make_mesh ();
-  void make_custom ();
-  void write_dot_file ();
-  void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line);
-  void do_command(const host_def& host, const string& name, vector<pair<string, string>> env_pairs, const string& cmd);
-  bool do_ssh (const string &cmd, const string &host_name);
-  void prep_remote_config_dir (eosd_def &node, host_def *host);
-  void launch (eosd_def &node, string &gts);
-  void kill (launch_modes mode, string sig_opt);
-  static string get_node_num(uint16_t node_num);
-  pair<host_def, eosd_def> find_node(uint16_t node_num);
-  vector<pair<host_def, eosd_def>> get_nodes(const string& node_number_list);
-  void bounce (const string& node_numbers);
-  void down (const string& node_numbers);
-  void roll (const string& host_names);
-  void start_all (string &gts, launch_modes mode);
-  void ignite ();
+   void make_ring ();
+   void make_star ();
+   void make_mesh ();
+   void make_custom ();
+   void write_dot_file ();
+   void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line);
+   void do_command(const host_def& host, const string& name, vector<pair<string, string>> env_pairs, const string& cmd);
+   bool do_ssh (const string &cmd, const string &host_name);
+   void prep_remote_config_dir (eosd_def &node, host_def *host);
+   void launch (eosd_def &node, string &gts);
+   void kill (launch_modes mode, string sig_opt);
+   static string get_node_num(uint16_t node_num);
+   pair<host_def, eosd_def> find_node(uint16_t node_num);
+   vector<pair<host_def, eosd_def>> get_nodes(const string& node_number_list);
+   void bounce (const string& node_numbers);
+   void down (const string& node_numbers);
+   void roll (const string& host_names);
+   void start_all (string &gts, launch_modes mode);
+   void ignite ();
 };
 
 void
@@ -482,7 +485,7 @@ launcher_def::set_options (bpo::options_description &cfg) {
     ("mode,m",bpo::value<vector<string>>()->multitoken()->default_value({"any"}, "any"),"connection mode, combination of \"any\", \"producers\", \"specified\", \"none\"")
     ("shape,s",bpo::value<string>(&shape)->default_value("star"),"network topology, use \"star\" \"mesh\" or give a filename for custom")
     ("p2p-plugin", bpo::value<string>()->default_value("net"),"select a p2p plugin to use (either net or bnet). Defaults to net.")
-    ("genesis,g",bpo::value<bfs::path>(&genesis)->default_value("./genesis.json"),"set the path to genesis.json")
+    ("genesis,g",bpo::value<string>()->default_value("./genesis.json"),"set the path to genesis.json")
     ("skip-signature", bpo::bool_switch(&skip_transaction_signatures)->default_value(false), "nodeos does not require transaction signatures.")
     ("nodeos", bpo::value<string>(&eosd_extra_args), "forward nodeos command line argument(s) to each instance of nodeos, enclose arg(s) in quotes")
     ("specific-num", bpo::value<vector<uint>>()->composing(), "forward nodeos command line argument(s) (using \"--specific-nodeos\" flag) to this specific instance of nodeos. This parameter can be entered multiple times and requires a paired \"--specific-nodeos\" flag")
@@ -490,14 +493,16 @@ launcher_def::set_options (bpo::options_description &cfg) {
     ("delay,d",bpo::value<int>(&start_delay)->default_value(0),"seconds delay before starting each node after the first")
     ("boot",bpo::bool_switch(&boot)->default_value(false),"After deploying the nodes and generating a boot script, invoke it.")
     ("nogen",bpo::bool_switch(&nogen)->default_value(false),"launch nodes without writing new config files")
-    ("host-map",bpo::value<bfs::path>(&host_map_file)->default_value(""),"a file containing mapping specific nodes to hosts. Used to enhance the custom shape argument")
-    ("servers",bpo::value<bfs::path>(&server_ident_file)->default_value(""),"a file containing ip addresses and names of individual servers to deploy as producers or non-producers ")
+    ("host-map",bpo::value<string>(),"a file containing mapping specific nodes to hosts. Used to enhance the custom shape argument")
+    ("servers",bpo::value<string>(),"a file containing ip addresses and names of individual servers to deploy as producers or non-producers ")
     ("per-host",bpo::value<int>(&per_host)->default_value(0),"specifies how many nodeos instances will run on a single host. Use 0 to indicate all on one.")
     ("network-name",bpo::value<string>(&network.name)->default_value("testnet_"),"network name prefix used in GELF logging source")
     ("enable-gelf-logging",bpo::value<bool>(&gelf_enabled)->default_value(true),"enable gelf logging appender in logging configuration file")
     ("gelf-endpoint",bpo::value<string>(&gelf_endpoint)->default_value("10.160.11.21:12201"),"hostname:port or ip:port of GELF endpoint")
     ("template",bpo::value<string>(&start_temp)->default_value("testnet.template"),"the startup script template")
     ("script",bpo::value<string>(&start_script)->default_value("bios_boot.sh"),"the generated startup script name")
+    ("max-block-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-block-cpu-usage\" value to use in the genesis.json file")
+    ("max-transaction-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-transaction-cpu-usage\" value to use in the genesis.json file")
         ;
 }
 
@@ -529,6 +534,22 @@ launcher_def::initialize (const variables_map &vmap) {
     }
   }
 
+  if (vmap.count("max-block-cpu-usage")) {
+     max_block_cpu_usage = vmap["max-block-cpu-usage"].as<uint32_t>();
+  }
+
+  if (vmap.count("max-transaction-cpu-usage")) {
+     max_transaction_cpu_usage = vmap["max-transaction-cpu-usage"].as<uint32_t>();
+  }
+
+  genesis = vmap["genesis"].as<string>();
+  if (vmap.count("host-map")) {
+     host_map_file = vmap["host-map"].as<string>();
+  }
+  if (vmap.count("servers")) {
+     server_ident_file = vmap["servers"].as<string>();
+  }
+
   if (vmap.count("specific-num")) {
     const auto specific_nums = vmap["specific-num"].as<vector<uint>>();
     const auto specific_args = vmap["specific-nodeos"].as<vector<string>>();
@@ -1157,27 +1178,20 @@ launcher_def::write_logging_config_file(tn_node_def &node) {
 
 void
 launcher_def::init_genesis () {
-  bfs::path genesis_path = bfs::current_path() / "genesis.json";
-   bfs::ifstream src(genesis_path);
-   if (!src.good()) {
+   const bfs::path genesis_path = genesis.is_complete() ? genesis : bfs::current_path() / genesis;
+   if (!bfs::exists(genesis_path)) {
       cout << "generating default genesis file " << genesis_path << endl;
       eosio::chain::genesis_state default_genesis;
       fc::json::save_to_file( default_genesis, genesis_path, true );
-      src.open(genesis_path);
    }
    string bioskey = string(network.nodes["bios"].keys[0].get_public_key());
-   string str;
-   string prefix("initial_key");
-   while(getline(src,str)) {
-      size_t pos = str.find(prefix);
-      if (pos != string::npos) {
-         size_t cut = str.find("EOS",pos);
-         genesis_block.push_back(str.substr(0,cut) + bioskey + "\",");
-      }
-      else {
-         genesis_block.push_back(str);
-      }
-   }
+
+   fc::json::from_file(genesis_path).as<eosio::chain::genesis_state>(genesis_from_file);
+   genesis_from_file.initial_key = public_key_type(bioskey);
+   if (max_block_cpu_usage)
+      genesis_from_file.initial_configuration.max_block_cpu_usage = *max_block_cpu_usage;
+   if (max_transaction_cpu_usage)
+      genesis_from_file.initial_configuration.max_transaction_cpu_usage = *max_transaction_cpu_usage;
 }
 
 void
@@ -1191,10 +1205,7 @@ launcher_def::write_genesis_file(tn_node_def &node) {
   }
 
   filename = dd / "genesis.json";
-  bfs::ofstream gf ( dd / "genesis.json");
-  for (auto &line : genesis_block) {
-     gf << line << "\n";
-  }
+  fc::json::save_to_file( genesis_from_file, dd / "genesis.json", true );
 }
 
 void

tests/Node.py

@@ -31,7 +31,7 @@ class Node(object):
 
     # pylint: disable=too-many-instance-attributes
     # pylint: disable=too-many-arguments
-    def __init__(self, host, port, pid=None, cmd=None, enableMongo=False, mongoHost="localhost", mongoPort=27017, mongoDb="EOStest"):
+    def __init__(self, host, port, pid=None, cmd=None, walletMgr=None, enableMongo=False, mongoHost="localhost", mongoPort=27017, mongoDb="EOStest"):
         self.host=host
         self.port=port
         self.pid=pid
@@ -44,17 +44,18 @@ class Node(object):
         self.mongoDb=mongoDb
         self.endpointHttp="http://%s:%d" % (self.host, self.port)
         self.endpointArgs="--url %s" % (self.endpointHttp)
-        self.miscEosClientArgs="--no-auto-keosd"
         self.mongoEndpointArgs=""
         self.infoValid=None
         self.lastRetrievedHeadBlockNum=None
         self.lastRetrievedLIB=None
         self.transCache={}
+        self.walletMgr=walletMgr
         if self.enableMongo:
             self.mongoEndpointArgs += "--host %s --port %d %s" % (mongoHost, mongoPort, mongoDb)
 
     def eosClientArgs(self):
-        return self.endpointArgs + " " + self.miscEosClientArgs
+        walletArgs=" " + self.walletMgr.getWalletEndpointArgs() if self.walletMgr is not None else ""
+        return self.endpointArgs + walletArgs + " " + Utils.MiscEosClientArgs
 
     def __str__(self):
         #return "Host: %s, Port:%d, Pid:%s, Cmd:\"%s\"" % (self.host, self.port, self.pid, self.cmd)
@@ -226,9 +227,6 @@ class Node(object):
     def byteArrToStr(arr):
         return arr.decode("utf-8")
 
-    def setWalletEndpointArgs(self, args):
-        self.endpointArgs="--url http://%s:%d %s" % (self.host, self.port, args)
-
     def validateAccounts(self, accounts):
         assert(accounts)
         assert(isinstance(accounts, list))

tests/WalletMgr.py

@@ -12,8 +12,10 @@ from testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
-    __walletLogFile="test_keosd_output.log"
+    __walletLogOutFile="test_keosd_out.log"
+    __walletLogErrFile="test_keosd_err.log"
     __walletDataDir="test_wallet_0"
+    __MaxPort=9999
 
     # pylint: disable=too-many-arguments
     # walletd [True|False] True=Launch wallet(keosd) process; False=Manage launch process externally.
@@ -25,26 +27,53 @@ class WalletMgr(object):
         self.host=host
         self.wallets={}
         self.__walletPid=None
-        self.endpointArgs="--url http://%s:%d" % (self.nodeosHost, self.nodeosPort)
-        self.walletEndpointArgs=""
-        if self.walletd:
-            self.walletEndpointArgs += " --wallet-url http://%s:%d" % (self.host, self.port)
-            self.endpointArgs += self.walletEndpointArgs
+
+    def getWalletEndpointArgs(self):
+        if not self.walletd or not self.isLaunched():
+            return ""
+
+        return " --wallet-url http://%s:%d" % (self.host, self.port)
+
+    def getArgs(self):
+        return " --url http://%s:%d%s %s" % (self.nodeosHost, self.nodeosPort, self.getWalletEndpointArgs(), Utils.MiscEosClientArgs)
+
+    def isLaunched(self):
+        return self.__walletPid is not None
+
+    def isLocal(self):
+        return self.host=="localhost" or self.host=="127.0.0.1"
+
+    def findAvailablePort(self):
+        for i in range(WalletMgr.__MaxPort):
+            port=self.port+i
+            if port > WalletMgr.__MaxPort:
+                port-=WalletMgr.__MaxPort
+            if Utils.arePortsAvailable(port):
+                return port
+            if Utils.Debug: Utils.Print("Port %d not available for %s" % (port, Utils.EosWalletPath))
+
+        Utils.errorExit("Failed to find free port to use for %s" % (Utils.EosWalletPath))
 
     def launch(self):
         if not self.walletd:
             Utils.Print("ERROR: Wallet Manager wasn't configured to launch keosd")
             return False
 
+        if self.isLaunched():
+            return True
+
+        if self.isLocal():
+            self.port=self.findAvailablePort()
+
         if Utils.Debug:
             portStatus="N/A"
             portTaken=False
-            if self.host=="localhost" or self.host=="127.0.0.1":
+            if self.isLocal():
                 if Utils.arePortsAvailable(self.port):
                     portStatus="AVAILABLE"
-                    portTaken=True
                 else:
                     portStatus="NOT AVAILABLE"
+                    portTaken=True
             pgrepCmd=Utils.pgrepCmd(Utils.EosWalletName)
             psOut=Utils.checkOutput(pgrepCmd.split(), ignoreError=True)
             if psOut or portTaken:
@@ -58,7 +87,7 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --http-server-address=%s:%d --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
-        with open(WalletMgr.__walletLogFile, 'w') as sout, open(WalletMgr.__walletLogFile, 'w') as serr:
+        with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
 
@@ -80,7 +109,7 @@ class WalletMgr(object):
             return wallet
         p = re.compile(r'\n\"(\w+)\"\n', re.MULTILINE)
         cmdDesc="wallet create"
-        cmd="%s %s %s --name %s --to-console" % (Utils.EosClientPath, self.endpointArgs, cmdDesc, name)
+        cmd="%s %s %s --name %s --to-console" % (Utils.EosClientPath, self.getArgs(), cmdDesc, name)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         retStr=None
         maxRetryCount=4
@@ -96,7 +125,7 @@ class WalletMgr(object):
                     pgrepCmd=Utils.pgrepCmd(Utils.EosWalletName)
                     psOut=Utils.checkOutput(pgrepCmd.split())
                     portStatus="N/A"
-                    if self.host=="localhost" or self.host=="127.0.0.1":
+                    if self.isLocal():
                         if Utils.arePortsAvailable(self.port):
                             portStatus="AVAILABLE"
                         else:
@@ -139,7 +168,7 @@ class WalletMgr(object):
     def importKey(self, account, wallet, ignoreDupKeyWarning=False):
         warningMsg="Key already in wallet"
         cmd="%s %s wallet import --name %s --private-key %s" % (
-            Utils.EosClientPath, self.endpointArgs, wallet.name, account.ownerPrivateKey)
+            Utils.EosClientPath, self.getArgs(), wallet.name, account.ownerPrivateKey)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         try:
             Utils.checkOutput(cmd.split())
@@ -156,7 +185,7 @@ class WalletMgr(object):
             Utils.Print("WARNING: Active private key is not defined for account \"%s\"" % (account.name))
         else:
             cmd="%s %s wallet import --name %s  --private-key %s" % (
-                Utils.EosClientPath, self.endpointArgs, wallet.name, account.activePrivateKey)
+                Utils.EosClientPath, self.getArgs(), wallet.name, account.activePrivateKey)
             if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
             try:
                 Utils.checkOutput(cmd.split())
@@ -173,7 +202,7 @@ class WalletMgr(object):
         return True
 
     def lockWallet(self, wallet):
-        cmd="%s %s wallet lock --name %s" % (Utils.EosClientPath, self.endpointArgs, wallet.name)
+        cmd="%s %s wallet lock --name %s" % (Utils.EosClientPath, self.getArgs(), wallet.name)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             Utils.Print("ERROR: Failed to lock wallet %s." % (wallet.name))
@@ -182,7 +211,7 @@ class WalletMgr(object):
         return True
 
     def unlockWallet(self, wallet):
-        cmd="%s %s wallet unlock --name %s" % (Utils.EosClientPath, self.endpointArgs, wallet.name)
+        cmd="%s %s wallet unlock --name %s" % (Utils.EosClientPath, self.getArgs(), wallet.name)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         popen=subprocess.Popen(cmd.split(), stdout=Utils.FNull, stdin=subprocess.PIPE)
         _, errs = popen.communicate(input=wallet.password.encode("utf-8"))
@@ -193,7 +222,7 @@ class WalletMgr(object):
         return True
 
     def lockAllWallets(self):
-        cmd="%s %s wallet lock_all" % (Utils.EosClientPath, self.endpointArgs)
+        cmd="%s %s wallet lock_all" % (Utils.EosClientPath, self.getArgs())
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             Utils.Print("ERROR: Failed to lock all wallets.")
@@ -205,7 +234,7 @@ class WalletMgr(object):
         wallets=[]
 
         p = re.compile(r'\s+\"(\w+)\s\*\",?\n', re.MULTILINE)
-        cmd="%s %s wallet list" % (Utils.EosClientPath, self.endpointArgs)
+        cmd="%s %s wallet list" % (Utils.EosClientPath, self.getArgs())
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         retStr=None
         try:
@@ -227,7 +256,7 @@ class WalletMgr(object):
         keys=[]
 
         p = re.compile(r'\n\s+\"(\w+)\"\n', re.MULTILINE)
-        cmd="%s %s wallet private_keys --name %s --password %s " % (Utils.EosClientPath, self.endpointArgs, wallet.name, wallet.password)
+        cmd="%s %s wallet private_keys --name %s --password %s " % (Utils.EosClientPath, self.getArgs(), wallet.name, wallet.password)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         retStr=None
         try:
@@ -248,9 +277,13 @@ class WalletMgr(object):
     def dumpErrorDetails(self):
         Utils.Print("=================================================================")
         if self.__walletPid is not None:
-            Utils.Print("Contents of %s:" % (WalletMgr.__walletLogFile))
+            Utils.Print("Contents of %s:" % (WalletMgr.__walletLogOutFile))
+            Utils.Print("=================================================================")
+            with open(WalletMgr.__walletLogOutFile, "r") as f:
+                shutil.copyfileobj(f, sys.stdout)
+            Utils.Print("Contents of %s:" % (WalletMgr.__walletLogErrFile))
             Utils.Print("=================================================================")
-            with open(WalletMgr.__walletLogFile, "r") as f:
+            with open(WalletMgr.__walletLogErrFile, "r") as f:
                 shutil.copyfileobj(f, sys.stdout)
 
     def killall(self, allInstances=False):

tests/consensus-validation-malicious-producers.py

@@ -257,11 +257,6 @@ def myTest(transWillEnterBlock):
         currencyAccount=accounts[0]
         currencyAccount.name="currency0000"
 
-        Print("Stand up walletd")
-        if walletMgr.launch() is False:
-            error("Failed to stand up eos walletd.")
-            return False
-
         testWalletName="test"
         Print("Creating wallet \"%s\"." % (testWalletName))
         testWallet=walletMgr.create(testWalletName)

tests/distributed-transactions-test.py

@@ -47,11 +47,15 @@ try:
         if not cluster.initializeNodesFromJson(jsonStr):
             errorExit("Failed to initilize nodes from Json string.")
         total_nodes=len(cluster.getNodes())
+
+        walletMgr.killall(allInstances=killAll)
+        walletMgr.cleanup()
+        print("Stand up walletd")
+        if walletMgr.launch() is False:
+            errorExit("Failed to stand up keosd.")
     else:
         cluster.killall(allInstances=killAll)
         cluster.cleanup()
-        walletMgr.killall(allInstances=killAll)
-        walletMgr.cleanup()
 
         Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" %
                (pnodes, total_nodes-pnodes, topo, delay))
@@ -65,12 +69,6 @@ try:
         if not cluster.waitOnClusterBlockNumSync(3):
             errorExit("Cluster never stabilized")
 
-    Print("Stand up EOS wallet keosd")
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        errorExit("Failed to stand up keosd.")
-
     accountsCount=total_nodes
     walletName="MyWallet-%d" % (random.randrange(10000))
     Print("Creating wallet %s if one doesn't already exist." % walletName)

tests/launcher_test.py

@@ -46,20 +46,26 @@ Utils.setIrreversibleTimeout(timeout)
 try:
     TestHelper.printSystemInfo("BEGIN")
 
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
+    cluster.setWalletMgr(walletMgr)
 
     if not dontLaunch:
         cluster.killall(allInstances=killAll)
         cluster.cleanup()
         Print("Stand up cluster")
-        if cluster.launch(pnodes=4, dontKill=dontKill, p2pPlugin=p2pPlugin) is False:
+        if cluster.launch(pnodes=4, p2pPlugin=p2pPlugin) is False:
             cmdError("launcher")
             errorExit("Failed to stand up eos cluster.")
     else:
+        walletMgr.killall(allInstances=killAll)
+        walletMgr.cleanup()
         cluster.initializeNodes(defproduceraPrvtKey=defproduceraPrvtKey)
         killEosInstances=False
 
+        print("Stand up walletd")
+        if walletMgr.launch() is False:
+            cmdError("%s" % (WalletdName))
+            errorExit("Failed to stand up eos walletd.")
+
     Print("Validating system accounts after bootstrap")
     cluster.validateAccounts(None)
 
@@ -86,13 +92,6 @@ try:
     exchangeAccount.ownerPrivateKey=PRV_KEY2
     exchangeAccount.ownerPublicKey=PUB_KEY2
 
-    Print("Stand up %s" % (WalletdName))
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        cmdError("%s" % (WalletdName))
-        errorExit("Failed to stand up eos walletd.")
-
     testWalletName="test"
     Print("Creating wallet \"%s\"." % (testWalletName))
     testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,cluster.defproduceraAccount])

tests/nodeos_forked_chain_test.py

@@ -128,6 +128,7 @@ ClientName="cleos"
 try:
     TestHelper.printSystemInfo("BEGIN")
 
+    cluster.setWalletMgr(walletMgr)
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
@@ -141,7 +142,7 @@ try:
     # "bridge" shape connects defprocera through defproducerk (in node0) to each other and defproducerl through defproduceru (in node01)
     # and the only connection between those 2 groups is through the bridge node
 
-    if cluster.launch(prodCount=prodCount, onlyBios=False, dontKill=dontKill, topo="bridge", pnodes=totalProducerNodes,
+    if cluster.launch(prodCount=prodCount, onlyBios=False, topo="bridge", pnodes=totalProducerNodes,
                       totalNodes=totalNodes, totalProducers=totalProducers, p2pPlugin=p2pPlugin,
                       useBiosBootFile=False, specificExtraNodeosArgs=specificExtraNodeosArgs) is False:
         Utils.cmdError("launcher")
@@ -164,12 +165,6 @@ try:
     testWalletName="test"
 
     Print("Creating wallet \"%s\"." % (testWalletName))
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        Utils.cmdError("%s" % (WalletdName))
-        Utils.errorExit("Failed to stand up eos walletd.")
-
     testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4]])
 
     for _, account in cluster.defProducerAccounts.items():

tests/nodeos_run_remote_test.py

@@ -42,7 +42,7 @@ try:
     Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" %
            (pnodes, total_nodes-pnodes, topo, delay))
     Print("Stand up cluster")
-    if cluster.launch(pnodes, total_nodes, prodCount, topo, delay, onlyBios=onlyBios, dontKill=dontKill) is False:
+    if cluster.launch(pnodes, total_nodes, prodCount, topo, delay, onlyBios=onlyBios) is False:
         errorExit("Failed to stand up eos cluster.")
 
     Print ("Wait for Cluster stabilization")

tests/nodeos_run_test.py

@@ -55,25 +55,30 @@ Utils.setIrreversibleTimeout(timeout)
 
 try:
     TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
     Print("SERVER: %s" % (server))
     Print("PORT: %d" % (port))
 
     if enableMongo and not cluster.isMongodDbRunning():
         errorExit("MongoDb doesn't seem to be running.")
 
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-
     if localTest and not dontLaunch:
         cluster.killall(allInstances=killAll)
         cluster.cleanup()
         Print("Stand up cluster")
-        if cluster.launch(prodCount=prodCount, onlyBios=onlyBios, dontKill=dontKill, dontBootstrap=dontBootstrap, p2pPlugin=p2pPlugin) is False:
+        if cluster.launch(prodCount=prodCount, onlyBios=onlyBios, dontBootstrap=dontBootstrap, p2pPlugin=p2pPlugin) is False:
             cmdError("launcher")
             errorExit("Failed to stand up eos cluster.")
     else:
         cluster.initializeNodes(defproduceraPrvtKey=defproduceraPrvtKey, defproducerbPrvtKey=defproducerbPrvtKey)
         killEosInstances=False
+        Print("Stand up %s" % (WalletdName))
+        walletMgr.killall(allInstances=killAll)
+        walletMgr.cleanup()
+        print("Stand up walletd")
+        if walletMgr.launch() is False:
+            cmdError("%s" % (WalletdName))
+            errorExit("Failed to stand up eos walletd.")
 
     if sanityTest:
         testSuccessful=True
@@ -105,13 +110,6 @@ try:
     exchangeAccount.ownerPrivateKey=PRV_KEY2
     exchangeAccount.ownerPublicKey=PUB_KEY2
 
-    Print("Stand up %s" % (WalletdName))
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        cmdError("%s" % (WalletdName))
-        errorExit("Failed to stand up eos walletd.")
-
     testWalletName="test"
     Print("Creating wallet \"%s\"." % (testWalletName))
     testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,cluster.defproduceraAccount,cluster.defproducerbAccount])

tests/nodeos_under_min_avail_ram.py

@@ -73,6 +73,7 @@ ClientName="cleos"
 
 try:
     TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
 
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
@@ -82,7 +83,7 @@ try:
     maxRAMFlag="--chain-state-db-size-mb"
     maxRAMValue=1010
     extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue)
-    if cluster.launch(onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False:
+    if cluster.launch(onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False:
         Utils.cmdError("launcher")
         errorExit("Failed to stand up eos cluster.")
 
@@ -96,12 +97,6 @@ try:
     testWalletName="test"
 
     Print("Creating wallet \"%s\"." % (testWalletName))
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        Utils.cmdError("%s" % (WalletdName))
-        errorExit("Failed to stand up eos walletd.")
-
     testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount])
 
     for _, account in cluster.defProducerAccounts.items():

tests/nodeos_voting_test.py

@@ -161,11 +161,12 @@ ClientName="cleos"
 
 try:
     TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
 
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
-    if cluster.launch(prodCount=prodCount, onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin, useBiosBootFile=False) is False:
+    if cluster.launch(prodCount=prodCount, onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin, useBiosBootFile=False) is False:
         Utils.cmdError("launcher")
         Utils.errorExit("Failed to stand up eos cluster.")
 
@@ -184,12 +185,6 @@ try:
     testWalletName="test"
 
     Print("Creating wallet \"%s\"." % (testWalletName))
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        Utils.cmdError("%s" % (WalletdName))
-        Utils.errorExit("Failed to stand up eos walletd.")
-
     testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4]])
 
     for _, account in cluster.defProducerAccounts.items():

tests/restart-scenarios-test.py

@@ -52,6 +52,7 @@ walletMgr=WalletMgr(True)
 
 try:
     TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
 
     cluster.setChainStrategy(chainSyncStrategyStr)
     cluster.setWalletMgr(walletMgr)
@@ -74,11 +75,6 @@ try:
         errorExit("Cluster never stabilized")
 
     Print("Stand up EOS wallet keosd")
-    walletMgr.killall(allInstances=killAll)
-    walletMgr.cleanup()
-    if walletMgr.launch() is False:
-        errorExit("Failed to stand up keosd.")
-
     accountsCount=total_nodes
     walletName="MyWallet"
     Print("Creating wallet %s if one doesn't already exist." % walletName)

tests/testUtils.py

+import errno
 import subprocess
 import time
 import os
@@ -18,6 +19,7 @@ class Utils:
     FNull = open(os.devnull, 'w')
 
     EosClientPath="programs/cleos/cleos"
+    MiscEosClientArgs="--no-auto-keosd"
 
     EosWalletName="keosd"
     EosWalletPath="programs/keosd/"+ EosWalletName

unittests/api_tests.cpp

@@ -1964,23 +1964,9 @@ BOOST_FIXTURE_TEST_CASE(new_api_feature_tests, TESTER) { try {
       });
 
    // change privilege
-   {
-      chainbase::database &db = control->db();
-      const account_object &account = db.get<account_object, by_name>(N(testapi));
-      db.modify(account, [&](account_object &v) {
-         v.privileged = true;
-      });
-   }
-
-#ifndef NON_VALIDATING_TEST
-   {
-      chainbase::database &db = validating_node->db();
-      const account_object &account = db.get<account_object, by_name>(N(testapi));
-      db.modify(account, [&](account_object &v) {
-         v.privileged = true;
-      });
-   }
-#endif
+   push_action(config::system_account_name, N(setpriv), config::system_account_name,  mutable_variant_object()
+                                                       ("account", "testapi")
+                                                       ("is_priv", 1));
 
    CALL_TEST_FUNCTION( *this, "test_transaction", "new_feature", {} );
 

unittests/auth_tests.cpp

@@ -383,7 +383,7 @@ try {
    chain.create_account(acc1a);
    chain.produce_block();
 
-   chainbase::database &db = chain.control->db();
+   const chainbase::database &db = chain.control->db();
 
    using resource_usage_object = eosio::chain::resource_limits::resource_usage_object;
    using by_owner = eosio::chain::resource_limits::by_owner;

unittests/database_tests.cpp

@@ -26,7 +26,9 @@ BOOST_AUTO_TEST_SUITE(database_tests)
    BOOST_AUTO_TEST_CASE(undo_test) {
       try {
          TESTER test;
-         auto &db = test.control->db();
+
+         // Bypass read-only restriction on state DB access for this unit test which really needs to mutate the DB to properly conduct its test.
+         eosio::chain::database& db = const_cast<eosio::chain::database&>( test.control->db() );
 
          auto ses = db.start_undo_session(true);
 

unittests/delay_tests.cpp

@@ -2317,16 +2317,10 @@ BOOST_AUTO_TEST_CASE( max_transaction_delay_execute ) { try {
    chain.produce_blocks();
 
    //change max_transaction_delay to 60 sec
-   chain.control->db().modify( chain.control->get_global_properties(),
-                              [&]( auto& gprops ) {
-                                 gprops.configuration.max_transaction_delay = 60;
-                              });
-#ifndef NON_VALIDATING_TEST
-   chain.validating_node->db().modify( chain.validating_node->get_global_properties(),
-                              [&]( auto& gprops ) {
-                                 gprops.configuration.max_transaction_delay = 60;
-                              });
-#endif
+   auto params = chain.control->get_global_properties().configuration;
+   params.max_transaction_delay = 60;
+   chain.push_action( config::system_account_name, N(setparams), config::system_account_name, mutable_variant_object()
+                        ("params", params) );
 
    chain.produce_blocks();
    //should be able to create transaction with delay 60 sec, despite permission delay being 30 days, because max_transaction_delay is 60 sec

unittests/special_accounts_tests.cpp

@@ -35,7 +35,7 @@ BOOST_FIXTURE_TEST_CASE(accounts_exists, tester)
 
       tester test;
       chain::controller *control = test.control.get();
-      chain::database &chain1_db = control->db();
+      const chain::database& chain1_db = control->db();
 
       auto nobody = chain1_db.find<account_object, by_name>(config::null_account_name);
       BOOST_CHECK(nobody != nullptr);