move validation of ram usage to an explicit sync call. This is called...

move validation of ram usage to an explicit sync call.  This is called per-cycle when applying a block and per-transaction when generating a block.  This means that block generation is currently more strict but in a way that should never fail the parallelizable block validation EOSIO/eos#2098

libraries/chain/apply_context.cpp

@@ -321,7 +321,7 @@ const contracts::table_id_object& apply_context::find_or_create_table( name code
 
    require_write_lock(scope);
 
-   update_db_usage(payer, config::billable_size_v<contracts::table_id_object>, "New Table ${c},${s},${t}", _V("c",code)("s",scope)("t",table));
+   update_db_usage(payer, config::billable_size_v<contracts::table_id_object>);
 
    return mutable_db.create<contracts::table_id_object>([&](contracts::table_id_object &t_id){
       t_id.code = code;
@@ -368,14 +368,14 @@ const bytes& apply_context::get_packed_transaction() {
    return trx_meta.packed_trx;
 }
 
-void apply_context::update_db_usage( const account_name& payer, int64_t delta, const char* use_format, const fc::variant_object& args ) {
+void apply_context::update_db_usage( const account_name& payer, int64_t delta ) {
    require_write_lock( payer );
    if( (delta > 0) ) {
       if (!(privileged || payer == account_name(receiver))) {
          require_authorization( payer );
       }
 
-      mutable_controller.get_mutable_resource_limits_manager().add_account_ram_usage(payer, delta, use_format, args);
+      mutable_controller.get_mutable_resource_limits_manager().add_pending_account_ram_usage(payer, delta);
    }
 }
 
@@ -452,7 +452,7 @@ int apply_context::db_store_i64( uint64_t code, uint64_t scope, uint64_t table,
    });
 
    int64_t billable_size = (int64_t)(buffer_size + config::billable_size_v<key_value_object>);
-   update_db_usage( payer, billable_size, "New Row ${id} in (${c},${s},${t})", _V("id", obj.primary_key)("c",receiver)("s",scope)("t",table));
+   update_db_usage( payer, billable_size);
 
    keyval_cache.cache_table( tab );
    return keyval_cache.add( obj );
@@ -474,10 +474,10 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
       // refund the existing payer
       update_db_usage( obj.payer,  -(old_size) );
       // charge the new payer
-      update_db_usage( payer,  (new_size), "Transfer Row ${id} in (${c},${s},${t})", _V("id", obj.primary_key)("c",tab.code)("s",tab.scope)("t",tab.table));
+      update_db_usage( payer,  (new_size));
    } else if(old_size != new_size) {
       // charge/refund the existing payer the difference
-      update_db_usage( obj.payer, new_size - old_size, "Update Row ${id} in (${c},${s},${t})", _V("id", obj.primary_key)("c",tab.code)("s",tab.scope)("t",tab.table));
+      update_db_usage( obj.payer, new_size - old_size);
    }
 
    mutable_db.modify( obj, [&]( auto& o ) {

libraries/chain/chain_controller.cpp

@@ -920,6 +920,7 @@ void chain_controller::__apply_block(const signed_block& next_block)
             c_trace.shard_traces.emplace_back(move(s_trace));
          } /// for each shard
 
+         _resource_limits.synchronize_account_ram_usage();
          _apply_cycle_trace(c_trace);
          r_trace.cycle_traces.emplace_back(move(c_trace));
       } /// for each cycle
@@ -2028,17 +2029,16 @@ transaction_trace chain_controller::_apply_error( transaction_metadata& meta ) {
 
 void chain_controller::_destroy_generated_transaction( const generated_transaction_object& gto ) {
    auto& generated_transaction_idx = _db.get_mutable_index<generated_transaction_multi_index>();
-   _resource_limits.add_account_ram_usage(gto.payer, -( config::billable_size_v<generated_transaction_object> + gto.packed_trx.size()));
+   _resource_limits.add_pending_account_ram_usage(gto.payer, -( config::billable_size_v<generated_transaction_object> + gto.packed_trx.size()));
    generated_transaction_idx.remove(gto);
 
 }
 
 void chain_controller::_create_generated_transaction( const deferred_transaction& dto ) {
    size_t trx_size = fc::raw::pack_size(dto);
-   _resource_limits.add_account_ram_usage(
+   _resource_limits.add_pending_account_ram_usage(
       dto.payer,
-      (config::billable_size_v<generated_transaction_object> + (int64_t)trx_size),
-      "Generated Transaction ${id} from ${s}", _V("id", dto.sender_id)("s",dto.sender)
+      (config::billable_size_v<generated_transaction_object> + (int64_t)trx_size)
    );
 
    _db.create<generated_transaction_object>([&](generated_transaction_object &obj) {
@@ -2157,6 +2157,7 @@ transaction_trace chain_controller::wrap_transaction_processing( transaction_met
    // for now apply the transaction serially but schedule it according to those invariants
 
    auto result = trx_processing(data);
+   _resource_limits.synchronize_account_ram_usage();
 
    auto& bcycle = _pending_block->regions.back().cycles_summary.back();
    auto& bshard = bcycle.front();

libraries/chain/contracts/eosio_contract.cpp

@@ -75,10 +75,9 @@ void apply_eosio_newaccount(apply_context& context) {
       a.creation_date = context.controller.head_block_time();
    });
    resources.initialize_account(create.name);
-   resources.add_account_ram_usage(
+   resources.add_pending_account_ram_usage(
       create.creator,
-      (int64_t)config::overhead_per_account_ram_bytes,
-      "New Account ${n}", _V("n", create.name)
+      (int64_t)config::overhead_per_account_ram_bytes
    );
 
    auto create_permission = [owner=create.name, &db, &context, &resources](const permission_name& name, permission_object::id_type parent, authority &&auth) {
@@ -89,10 +88,9 @@ void apply_eosio_newaccount(apply_context& context) {
          p.auth = std::move(auth);
       });
 
-      resources.add_account_ram_usage(
+      resources.add_pending_account_ram_usage(
          owner,
-         (int64_t)(config::billable_size_v<permission_object> + result.auth.get_billable_size()),
-         "New Permission ${a}@${p}", _V("a", owner)("p",name)
+         (int64_t)(config::billable_size_v<permission_object> + result.auth.get_billable_size())
       );
 
       return result;
@@ -141,10 +139,9 @@ void apply_eosio_setcode(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      resources.add_account_ram_usage(
+      resources.add_pending_account_ram_usage(
          act.account,
-         new_size - old_size,
-         "Update Contract Code to ${v} [new size=${new}, old size=${old}]", _V("v",account.code_version)("new", new_size)("old",old_size)
+         new_size - old_size
       );
    }
 }
@@ -175,10 +172,9 @@ void apply_eosio_setabi(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      resources.add_account_ram_usage(
+      resources.add_pending_account_ram_usage(
          act.account,
-         new_size - old_size,
-         "Update Contract ABI [new size=${new}, old size=${old}]", _V("new", new_size)("old",old_size)
+         new_size - old_size
       );
    }
 }
@@ -261,10 +257,9 @@ void apply_eosio_updateauth(apply_context& context) {
 
       int64_t new_size = (int64_t)(config::billable_size_v<permission_object> + permission->auth.get_billable_size());
 
-      resources.add_account_ram_usage(
+      resources.add_pending_account_ram_usage(
          permission->owner,
-         new_size - old_size,
-         "Update Permission ${a}@${p} [new size=${new}, old size=${old}] ", _V("a", permission->owner)("p",permission->name)
+         new_size - old_size
       );
    } else {
       // TODO/QUESTION: If we are creating a new permission, should we check if the message declared
@@ -278,10 +273,9 @@ void apply_eosio_updateauth(apply_context& context) {
          po.delay = fc::seconds(update.delay);
       });
 
-      resources.add_account_ram_usage(
+      resources.add_pending_account_ram_usage(
          p.owner,
-         (int64_t)(config::billable_size_v<permission_object> + p.auth.get_billable_size()),
-         "New Permission ${a}@${p}", _V("a", p.owner)("p",p.name)
+         (int64_t)(config::billable_size_v<permission_object> + p.auth.get_billable_size())
       );
 
    }
@@ -315,7 +309,7 @@ void apply_eosio_deleteauth(apply_context& context) {
                  "Cannot delete a linked authority. Unlink the authority first");
    }
 
-   resources.add_account_ram_usage(
+   resources.add_pending_account_ram_usage(
       permission.owner,
       -(int64_t)(config::billable_size_v<permission_object> + permission.auth.get_billable_size())
    );
@@ -360,10 +354,9 @@ void apply_eosio_linkauth(apply_context& context) {
             link.required_permission = requirement.requirement;
          });
 
-         resources.add_account_ram_usage(
+         resources.add_pending_account_ram_usage(
             l.account,
-            (int64_t)(config::billable_size_v<permission_link_object>),
-            "New Permission Link ${code}::${act} -> ${a}@${p}", _V("code", l.code)("act",l.message_type)("a", l.account)("p",l.required_permission)
+            (int64_t)(config::billable_size_v<permission_link_object>)
          );
       }
   } FC_CAPTURE_AND_RETHROW((requirement))
@@ -379,7 +372,7 @@ void apply_eosio_unlinkauth(apply_context& context) {
    auto link_key = boost::make_tuple(unlink.account, unlink.code, unlink.type);
    auto link = db.find<permission_link_object, by_action_name>(link_key);
    EOS_ASSERT(link != nullptr, action_validate_exception, "Attempting to unlink authority, but no link found");
-   resources.add_account_ram_usage(
+   resources.add_pending_account_ram_usage(
       link->account,
       -(int64_t)(config::billable_size_v<permission_link_object>)
    );

libraries/chain/include/eosio/chain/apply_context.hpp

@@ -556,7 +556,7 @@ class apply_context {
       int get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const;
       int get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const;
 
-      void update_db_usage( const account_name& payer, int64_t delta, const char* use_format = "Unspecified", const fc::variant_object& args = fc::variant_object() );
+      void update_db_usage( const account_name& payer, int64_t delta );
       int  db_store_i64( uint64_t scope, uint64_t table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size );
       void db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size );
       void db_remove_i64( int iterator );

libraries/chain/include/eosio/chain/resource_limits.hpp

@@ -38,7 +38,9 @@ namespace eosio { namespace chain { namespace resource_limits {
 
          void add_transaction_usage( const vector<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal );
 
-         void add_account_ram_usage( const account_name account, int64_t ram_delta, const char* use_format = "Unspecified", const fc::variant_object& args = fc::variant_object() );
+
+         void add_pending_account_ram_usage( const account_name account, int64_t ram_delta );
+         void synchronize_account_ram_usage( );
 
          void set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight);
          void get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight) const;

libraries/chain/include/eosio/chain/resource_limits_private.hpp

@@ -92,6 +92,7 @@ namespace eosio { namespace chain { namespace resource_limits {
    };
 
    struct by_owner;
+   struct by_dirty;
 
    using resource_limits_index = chainbase::shared_multi_index_container<
       resource_limits_object,
@@ -116,13 +117,25 @@ namespace eosio { namespace chain { namespace resource_limits {
       usage_accumulator        cpu_usage;
 
       uint64_t                 ram_usage = 0;
+      uint64_t                 pending_ram_usage = 0;
+
+      bool is_dirty() const {
+         // checks for ram_usage overflowing a signed int are maintained in the update step
+         return ram_usage != pending_ram_usage;
+      }
    };
 
    using resource_usage_index = chainbase::shared_multi_index_container<
       resource_usage_object,
       indexed_by<
          ordered_unique<tag<by_id>, member<resource_usage_object, resource_usage_object::id_type, &resource_usage_object::id>>,
-         ordered_unique<tag<by_owner>, member<resource_usage_object, account_name, &resource_usage_object::owner> >
+         ordered_unique<tag<by_owner>, member<resource_usage_object, account_name, &resource_usage_object::owner> >,
+         ordered_unique<tag<by_dirty>,
+            composite_key<resource_usage_object,
+               BOOST_MULTI_INDEX_CONST_MEM_FUN(resource_usage_object, bool, is_dirty),
+               BOOST_MULTI_INDEX_MEMBER(resource_usage_object, resource_usage_object::id_type, id)
+            >
+         >
       >
    >;
 

libraries/chain/resource_limits.cpp

@@ -120,22 +120,45 @@ void resource_limits_manager::add_transaction_usage(const vector<account_name>&
    EOS_ASSERT( state.pending_net_usage <= config.net_limit_parameters.max, block_resource_exhausted, "Block has insufficient net resources" );
 }
 
-void resource_limits_manager::add_account_ram_usage( const account_name account, int64_t ram_delta, const char* use_format, const fc::variant_object& args ) {
+void resource_limits_manager::add_pending_account_ram_usage( const account_name account, int64_t ram_delta ) {
+   if (ram_delta == 0) {
+      return;
+   }
+
    const auto& usage = _db.get<resource_usage_object,by_owner>( account );
-   const auto& limits = _db.get<resource_limits_object,by_owner>( boost::make_tuple(false, account));
 
-   if (limits.ram_bytes >= 0 && usage.ram_usage + ram_delta > limits.ram_bytes) {
-      tx_resource_exhausted e(FC_LOG_MESSAGE(error, "account ${a} has insufficient ram bytes", ("a", account)));
-      e.append_log(fc::log_message( FC_LOG_CONTEXT(error), use_format, args ));
-      e.append_log(FC_LOG_MESSAGE(error, "needs ${d} has ${m}", ("d",ram_delta)("m",limits.ram_bytes)));
-      throw e;
-   }
+   EOS_ASSERT(ram_delta < 0 || UINT64_MAX - usage.pending_ram_usage >= (uint64_t)ram_delta, transaction_exception, "Ram usage delta would overflow UINT64_MAX");
+   EOS_ASSERT(ram_delta > 0 || usage.pending_ram_usage >= (uint64_t)(-ram_delta), transaction_exception, "Ram usage delta would underflow UINT64_MAX");
 
    _db.modify(usage, [&](resource_usage_object& o){
-      o.ram_usage += ram_delta;
+      o.pending_ram_usage += ram_delta;
    });
 }
 
+void resource_limits_manager::synchronize_account_ram_usage( ) {
+   auto& multi_index = _db.get_mutable_index<resource_usage_index>();
+   auto& by_dirty_index = multi_index.indices().get<by_dirty>();
+
+   while(!by_dirty_index.empty()) {
+      const auto& itr = by_dirty_index.lower_bound(boost::make_tuple(true));
+      if (itr == by_dirty_index.end() || itr->is_dirty() != true) {
+         break;
+      }
+
+      const auto& limits = _db.get<resource_limits_object,by_owner>( boost::make_tuple(false, itr->owner));
+
+      if (limits.ram_bytes >= 0 && itr->pending_ram_usage > limits.ram_bytes) {
+         tx_resource_exhausted e(FC_LOG_MESSAGE(error, "account ${a} has insufficient ram bytes", ("a", itr->owner)));
+         e.append_log(FC_LOG_MESSAGE(error, "needs ${d} has ${m}", ("d",itr->pending_ram_usage)("m",limits.ram_bytes)));
+         throw e;
+      }
+
+      _db.modify(*itr, [&](resource_usage_object& o){
+         o.ram_usage = o.pending_ram_usage;
+      });
+   }
+}
+
 void resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) {
    const auto& usage = _db.get<resource_usage_object,by_owner>( account );
    /*

tests/library_tests/chain/resource_limits_test.cpp

@@ -245,10 +245,12 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test)
       process_account_limit_updates();
 
       for (int idx = 0; idx < expected_iterations - 1; idx++) {
-         add_account_ram_usage(account, increment);
+         add_pending_account_ram_usage(account, increment);
+         synchronize_account_ram_usage( );
       }
 
-      BOOST_REQUIRE_THROW(add_account_ram_usage(account, increment), tx_resource_exhausted);
+      add_pending_account_ram_usage(account, increment);
+      BOOST_REQUIRE_THROW(synchronize_account_ram_usage( ), tx_resource_exhausted);
    } FC_LOG_AND_RETHROW();
 
    BOOST_FIXTURE_TEST_CASE(enforce_account_ram_commitment, resource_limits_fixture) try {
@@ -262,7 +264,8 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test)
       initialize_account(account);
       set_account_limits(account, limit, -1, -1 );
       process_account_limit_updates();
-      add_account_ram_usage(account, commit);
+      add_pending_account_ram_usage(account, commit);
+      synchronize_account_ram_usage( );
 
       for (int idx = 0; idx < expected_iterations - 1; idx++) {
          set_account_limits(account, limit - increment * idx, -1, -1);