From bdb295b502b0e1595a640cef6a98ea887a626fc5 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Fri, 13 Apr 2018 22:46:37 -0400 Subject: [PATCH] progress --- libraries/chain/block_header_state.cpp | 7 +- libraries/chain/controller.cpp | 195 ++++++++++-------- libraries/chain/fork_database.cpp | 41 +++- .../eosio/chain/block_header_state.hpp | 2 +- .../chain/include/eosio/chain/block_state.hpp | 15 +- .../chain/include/eosio/chain/context.hpp | 52 +++++ .../chain/include/eosio/chain/controller.hpp | 18 +- .../include/eosio/chain/fork_database.hpp | 2 +- .../eosio/chain/incremental_merkle.hpp | 4 +- .../eosio/chain/transaction_metadata.hpp | 2 + libraries/chain/main.cpp | 26 +-- 11 files changed, 243 insertions(+), 121 deletions(-) create mode 100644 libraries/chain/include/eosio/chain/context.hpp diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index 02531610b..6928269f9 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -123,9 +123,10 @@ namespace eosio { namespace chain { result.set_new_producers( *h.new_producers ); } - result.header.action_mroot = h.action_mroot; - result.header.transaction_mroot = h.transaction_mroot; - result.id = h.id(); + result.header.action_mroot = h.action_mroot; + result.header.transaction_mroot = h.transaction_mroot; + result.header.producer_signature = h.producer_signature; + result.id = h.id(); return result; } /// next diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index e05fbba18..ffbfea62e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -17,6 +17,7 @@ #include +#include #include namespace eosio { namespace chain { @@ -33,6 +34,9 @@ struct pending_state { block_state_ptr _pending_block_state; block_state_ptr _expected_block_state; + vector _actions; + vector _transaction_receipts; + void push() { _db_session.push(); } @@ -70,11 +74,13 @@ struct controller_impl { cfg.read_only ? database::read_only : database::read_write, cfg.shared_memory_size ), blog( cfg.block_log_dir ), + fork_db( cfg.shared_memory_dir ), //wasmif( cfg.wasm_runtime ), resource_limits( db ), conf( cfg ), self( s ) { + head = fork_db.head(); } void init() { @@ -89,7 +95,7 @@ struct controller_impl { * unwind that pending state. This state will be regenerated * when we catch up to the head block later. */ - clear_all_undo(); + //clear_all_undo(); } ~controller_impl() { @@ -145,21 +151,25 @@ struct controller_impl { * it would be the genesis state. */ void initialize_fork_db() { - wlog( " Initializing new blockchain with genesis state " ); - producer_schedule_type initial_schedule{ 0, {{N(eosio), conf.genesis.initial_key}} }; - - block_header_state genheader; - genheader.active_schedule = initial_schedule; - genheader.pending_schedule = initial_schedule; - wdump((genheader.active_schedule)); - genheader.pending_schedule_hash = fc::sha256::hash(initial_schedule); - genheader.header.timestamp = conf.genesis.initial_timestamp; - genheader.header.action_mroot = conf.genesis.compute_chain_id(); - genheader.id = genheader.header.id(); - genheader.block_num = genheader.header.block_num(); - - head = std::make_shared( genheader ); - fork_db.set( head ); + head = fork_db.head(); + if( !head ) { + wlog( " Initializing new blockchain with genesis state " ); + producer_schedule_type initial_schedule{ 0, {{N(eosio), conf.genesis.initial_key}} }; + + block_header_state genheader; + genheader.active_schedule = initial_schedule; + genheader.pending_schedule = initial_schedule; + genheader.pending_schedule_hash = fc::sha256::hash(initial_schedule); + genheader.header.timestamp = conf.genesis.initial_timestamp; + genheader.header.action_mroot = conf.genesis.compute_chain_id(); + genheader.id = genheader.header.id(); + genheader.block_num = genheader.header.block_num(); + + head = std::make_shared( genheader ); + db.set_revision( head->block_num ); + fork_db.set( head ); + } + FC_ASSERT( db.revision() == head->block_num, "fork database is inconsistant with shared memory" ); } @@ -204,68 +214,39 @@ struct controller_impl { }); } - transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx ) { + + + void apply_transaction( const transaction_metadata_ptr& trx, bool implicit = false ) { + transaction_context trx_context( self, trx ); + trx_context.exec(); + + auto& acts = pending->_actions; + fc::move_append( acts, move(trx_context.executed) ); + + if( !implicit ) { + pending->_applied_transaction_metas.emplace_back( trx ); + } + } + + void push_transaction( const transaction_metadata_ptr& trx ) { return db.with_write_lock( [&](){ return apply_transaction( trx ); }); } + void record_transaction( const transaction_metadata_ptr& trx ) { try { - _db.create([&](transaction_object& transaction) { + db.create([&](transaction_object& transaction) { transaction.trx_id = trx->id; transaction.expiration = trx->trx.expiration; }); } catch ( ... ) { EOS_ASSERT( false, transaction_exception, - "duplicate transaction ${id}", - ("id", trx.id() ) ); + "duplicate transaction ${id}", ("id", trx->id ) ); } } - void start_transaction( const transaction_metadata_ptr& trx ) { - record_transaction( trx ); - } - - transaction_trace_ptr apply_transaction( const transaction_metadata_ptr& trx, bool implicit = false ) { - auto trx_session = db.start_undo_session(); - // TODO: notify pre transaction - - start_transaction( trx ); - - - for( const auto& act : trx->trx.context_free_actions ) { - apply_action( act, ... ); - /* - apply_context context( self, db, act, trx ); - context.context_free = true; - context.exec(); - receipt.kcpu_usage += context.kcpu_usage - */ - - } - - for( const auto& act : trx->trx.actions ) { - auto act_session = db.start_undo_session(); - // TODO: notify pre action - - /* - apply_context context( self, db, act, trx ); - context.exec(); - receipt.kcpu_usage += context.kcpu_usage - */ - - // TODO: notify post action - act_session.squash(); - } - - finalize_transaction(); - - // TODO: notify post transaction - trx_session.squash(); - - push_transaction_receipt( trx, type ); - } @@ -337,21 +318,36 @@ struct controller_impl { */ + void set_action_merkle() { + vector action_digests; + action_digests.reserve( pending->_actions.size() ); + for( const auto& a : pending->_actions ) + action_digests.emplace_back( a.digest() ); + + pending->_pending_block_state->header.action_mroot = merkle( move(action_digests) ); + } - transaction_trace_ptr apply_transaction( const transaction_metadata_ptr& trx ) { - auto trx_trace = std::make_shared(); - trx_trace->action_traces.reserve( trx->trx.actions.size() ); - trx_trace->action_receipts.reserve( trx->trx.actions.size() ); + void set_trx_merkle() { + vector trx_digests; + trx_digests.reserve( pending->_transaction_receipts.size() ); + for( const auto& a : pending->_transaction_receipts) + trx_digests.emplace_back( a.digest() ); - for( const auto& act : trx->trx.actions ) { -// trx_trace->action_receipts.emplace_back( apply_action( act - } - return trx_trace; + pending->_pending_block_state->header.transaction_mroot = merkle( move(trx_digests) ); } - void finalize_block( const block_trace& trace ) + void finalize_block() { try { + ilog( "finalize block" ); + + set_action_merkle(); + set_trx_merkle(); + + auto p = pending->_pending_block_state; + p->id = p->header.id(); + + /* TODO RESTORE const auto& b = trace.block; @@ -412,7 +408,7 @@ struct controller_impl { * At the start of each block we notify the system contract with a transaction that passes in * the block header of the prior block (which is currently our head block) */ - transaction get_on_block_transaction() + signed_transaction get_on_block_transaction() { action on_block_act; on_block_act.account = config::system_account_name; @@ -420,7 +416,7 @@ struct controller_impl { on_block_act.authorization = vector{{config::system_account_name, config::active_name}}; on_block_act.data = fc::raw::pack(head_block_header()); - transaction trx; + signed_transaction trx; trx.actions.emplace_back(std::move(on_block_act)); trx.set_reference_block(head_block_id()); trx.expiration = head_block_time() + fc::seconds(1); @@ -511,29 +507,33 @@ void controller::startup() { */ } -const chainbase::database& controller::db()const { return my->db; } +chainbase::database& controller::db()const { return my->db; } void controller::start_block( block_timestamp_type when ) { wlog( "start_block" ); FC_ASSERT( !my->pending ); + idump( (my->db.revision())(my->head->block_num) ); + my->pending = my->db.start_undo_session(true); my->pending->_pending_block_state = std::make_shared( *my->head, when ); - auto t = my->get_on_block_transaction(); - // apply_transaction( - // idump((t)); + idump( (my->db.revision())(my->pending->_pending_block_state->block_num) ); + + idump((my->head->id)); + idump((my->pending->_pending_block_state->header)); + + auto onbtrx = std::make_shared( my->get_on_block_transaction() ); + my->apply_transaction( onbtrx, true ); } void controller::finalize_block() { - auto p = my->pending->_pending_block_state; - - /// set trx mroot and act mroot on pending.header and pending.block (which also has copy of header) - // my->pending->_pending_block_state->id = my->pending->_pending_block_state->header.id(); + my->finalize_block(); } void controller::sign_block( std::function signer_callback ) { + ilog( "sign block" ); auto p = my->pending->_pending_block_state; p->header.sign( signer_callback, p->pending_schedule_hash ); FC_ASSERT( p->block_signing_key == p->header.signee( p->pending_schedule_hash ), @@ -542,10 +542,12 @@ void controller::sign_block( std::function } void controller::commit_block() { + ilog( "commit block ${b}", ("b", my->pending->_pending_block_state->header) ); my->head = my->fork_db.add( my->pending->_pending_block_state ); my->pending->push(); my->pending.reset(); edump((my->head->header.block_num())(my->head->block_num)); + idump((my->head->id)); } block_state_ptr controller::head_block_state()const { @@ -554,24 +556,37 @@ block_state_ptr controller::head_block_state()const { void controller::push_block( const signed_block_ptr& b ) { + wdump((my->head->header.block_num())(my->head->block_num)); my->head = my->fork_db.add( b ); - //return my->push_block( b ); + wdump((my->head->header.block_num())(my->head->block_num)); } -transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx ) { - return my->push_transaction(); +void controller::push_transaction( const transaction_metadata_ptr& trx ) { + my->push_transaction(trx); } -transaction_trace_ptr controller::push_transaction() { - return transaction_trace_ptr(); +void controller::push_transaction() { } -transaction_trace_ptr controller::push_transaction( const transaction_id_type& trxid ) { - return transaction_trace_ptr(); +void controller::push_transaction( const transaction_id_type& trxid ) { /// lookup scheduled trx and then apply it... } uint32_t controller::head_block_num()const { return my->head->block_num; } + +uint64_t controller::next_global_sequence() { + return 0; +} +uint64_t controller::next_recv_sequence( account_name receiver ) { + return 0; +} +uint64_t controller::next_auth_sequence( account_name actor ) { + return 0; +} +void controller::record_transaction( const transaction_metadata_ptr& trx ) { + my->record_transaction( trx ); +} + } } /// eosio::chain diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index a15a4fb9c..bbf200f3c 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace eosio { namespace chain { using boost::multi_index_container; @@ -25,13 +27,50 @@ namespace eosio { namespace chain { struct fork_database_impl { fork_multi_index_type index; block_state_ptr head; + fc::path datadir; }; - fork_database::fork_database():my( new fork_database_impl() ) { + fork_database::fork_database( const fc::path& data_dir ):my( new fork_database_impl() ) { + my->datadir = data_dir; + + if (!fc::is_directory(my->datadir)) + fc::create_directories(my->datadir); + + auto fork_db_dat = my->datadir / "forkdb.dat"; + if( fc::exists( fork_db_dat ) ) { + string content; + fc::read_file_contents( fork_db_dat, content ); + + fc::datastream ds( content.data(), content.size() ); + vector states; + fc::raw::unpack( ds, states ); + + for( auto& s : states ) { + set( std::make_shared( move( s ) ) ); + } + block_id_type head_id; + fc::raw::unpack( ds, head_id ); + + my->head = get_block( head_id ); + } } fork_database::~fork_database() { + fc::datastream ps; + vector states; + states.reserve( my->index.size() ); + for( const auto& s : my->index ) { + states.push_back( *s ); + } + + auto fork_db_dat = my->datadir / "forkdb.dat"; + std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc ); + fc::raw::pack( out, states ); + if( my->head ) + fc::raw::pack( out, my->head->id ); + else + fc::raw::pack( out, block_id_type() ); } void fork_database::set( block_state_ptr s ) { diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index eb641f3db..d7ec0ca86 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -39,7 +39,7 @@ struct block_header_state { FC_REFLECT( eosio::chain::block_header_state, (id)(block_num)(header)(dpos_last_irreversible_blocknum) (pending_schedule_lib_num)(pending_schedule_hash) - (pending_schedule)(active_schedule)//(blockroot_merkle) + (pending_schedule)(active_schedule)(blockroot_merkle) (producer_to_last_produced) ) diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index fae3667b5..7839c9db2 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -9,13 +9,13 @@ namespace eosio { namespace chain { * For each action dispatched this receipt is generated */ struct action_receipt { - account_name receiver; - action act; - uint32_t block_sequence; /// block num - uint32_t sender_sequence; - transaction_id_type trx_id; /// the trx that started this action - uint16_t trx_action_dispatch_seq; ///< the relative order for implied trx - /// TODO: add code/scope/rw sequence numbers + account_name receiver; + action act; + uint64_t global_sequence; ///< total number of actions dispatched since genesis + uint64_t recv_sequence; ///< total number of actions with this receiver since genesis + flat_map auth_sequence; + + digest_type digest()const { return digest_type::hash(*this); } }; struct action_trace { @@ -61,4 +61,5 @@ namespace eosio { namespace chain { } } /// namespace eosio::chain +FC_REFLECT( eosio::chain::action_receipt, (receiver)(act)(global_sequence)(recv_sequence)(auth_sequence) ) FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), ) diff --git a/libraries/chain/include/eosio/chain/context.hpp b/libraries/chain/include/eosio/chain/context.hpp new file mode 100644 index 000000000..0ece2706d --- /dev/null +++ b/libraries/chain/include/eosio/chain/context.hpp @@ -0,0 +1,52 @@ +#pragma once +#include + +namespace eosio { namespace chain { + + + class transaction_context { + public: + transaction_context( controller& c, const transaction_metadata_ptr& trx ) + :control(c), + trx_meta(trx), + undo_session(c.db().start_undo_session(true)) + { + executed.reserve( trx_meta->total_actions() ); + } + + void exec() { + control.record_transaction( trx_meta ); + + for( const auto& act : trx_meta->trx.context_free_actions ) { + dispatch_action( act, act.account, true ); + } + + for( const auto& act : trx_meta->trx.actions ) { + dispatch_action( act, act.account, false ); + } + + undo_session.squash(); + } + + void dispatch_action( const action& a, account_name receiver, bool context_free = false ) { + action_receipt r; + r.receiver = receiver; + r.act = a; + r.global_sequence = control.next_global_sequence(); + r.recv_sequence = control.next_recv_sequence( receiver ); + + for( const auto& auth : a.authorization ) { + r.auth_sequence[auth.actor] = control.next_auth_sequence( auth.actor ); + } + + executed.emplace_back( move(r) ); + } + + vector executed; + + controller& control; + const transaction_metadata_ptr& trx_meta; + chainbase::database::session undo_session; + }; + +} } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 9bca4a2d4..1a371530b 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -46,9 +46,9 @@ namespace eosio { namespace chain { vector abort_block(); - transaction_trace_ptr push_transaction( const transaction_metadata_ptr& trx = transaction_metadata_ptr() ); - transaction_trace_ptr push_transaction( const transaction_id_type& scheduled ); - transaction_trace_ptr push_transaction(); + void push_transaction( const transaction_metadata_ptr& trx = transaction_metadata_ptr() ); + void push_transaction( const transaction_id_type& scheduled ); + void push_transaction(); void finalize_block(); void sign_block( std::function signer_callback ); @@ -56,12 +56,22 @@ namespace eosio { namespace chain { void push_block( const signed_block_ptr& b ); - const chainbase::database& db()const; + chainbase::database& db()const; uint32_t head_block_num()const; block_state_ptr head_block_state()const; + + + + + + uint64_t next_global_sequence(); + uint64_t next_recv_sequence( account_name receiver ); + uint64_t next_auth_sequence( account_name actor ); + void record_transaction( const transaction_metadata_ptr& trx ); + /* signal pre_apply_block; signal post_apply_block; diff --git a/libraries/chain/include/eosio/chain/fork_database.hpp b/libraries/chain/include/eosio/chain/fork_database.hpp index 6913da466..89a4c8020 100644 --- a/libraries/chain/include/eosio/chain/fork_database.hpp +++ b/libraries/chain/include/eosio/chain/fork_database.hpp @@ -22,7 +22,7 @@ namespace eosio { namespace chain { class fork_database { public: - fork_database(); + fork_database( const fc::path& data_dir ); ~fork_database(); block_state_ptr get_block(const block_id_type& id)const; diff --git a/libraries/chain/include/eosio/chain/incremental_merkle.hpp b/libraries/chain/include/eosio/chain/incremental_merkle.hpp index 3b3d16b9f..15c3fc439 100644 --- a/libraries/chain/include/eosio/chain/incremental_merkle.hpp +++ b/libraries/chain/include/eosio/chain/incremental_merkle.hpp @@ -233,7 +233,7 @@ class incremental_merkle_impl { } } - private: +// private: uint64_t _node_count; Container _active_nodes; }; @@ -242,3 +242,5 @@ typedef incremental_merkle_impl incremental_merkle; typedef incremental_merkle_impl shared_incremental_merkle; } } /// eosio::chain + +FC_REFLECT( eosio::chain::incremental_merkle, (_active_nodes)(_node_count) ); diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index b1969c55b..b41f8e26c 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -35,6 +35,8 @@ class transaction_metadata { } + uint32_t total_actions()const { return trx.context_free_actions.size() + trx.actions.size(); } + /* transaction_metadata( const transaction& t, diff --git a/libraries/chain/main.cpp b/libraries/chain/main.cpp index 968ced2a7..d23b77e51 100644 --- a/libraries/chain/main.cpp +++ b/libraries/chain/main.cpp @@ -9,19 +9,19 @@ int main( int argc, char** argv ) { controller c( {} ); controller c2( {.shared_memory_dir = "c2dir", .block_log_dir = "c2dir" } ); -for( uint32_t i = 0; i < 4; ++i ) { - c.start_block(); - c.finalize_block(); - c.sign_block( []( digest_type d ) { - auto priv = fc::variant("5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3").as(); - return priv.sign(d); - }); - c.commit_block(); - - c2.push_block( c.head_block_state()->block ); - idump((c.head_block_state()->header)); - idump((c2.head_block_state()->header)); -} + for( uint32_t i = 0; i < 4; ++i ) { + c.start_block(); + c.finalize_block(); + c.sign_block( []( digest_type d ) { + auto priv = fc::variant("5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3").as(); + return priv.sign(d); + }); + c.commit_block(); + + c2.push_block( c.head_block_state()->block ); + idump((c.head_block_state()->header)); + idump((c2.head_block_state()->header)); + } } FC_CAPTURE_AND_RETHROW() -- GitLab