Merge branch 'master' into contracts-console

libraries/chain/controller.cpp

@@ -92,14 +92,14 @@ struct controller_impl {
 
    controller_impl( const controller::config& cfg, controller& s  )
    :self(s),
-    db( cfg.shared_memory_dir,
+    db( cfg.state_dir,
         cfg.read_only ? database::read_only : database::read_write,
-        cfg.shared_memory_size ),
-    reversible_blocks( cfg.block_log_dir/"reversible",
+        cfg.state_size ),
+    reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name,
         cfg.read_only ? database::read_only : database::read_write,
         cfg.reversible_cache_size ),
-    blog( cfg.block_log_dir ),
-    fork_db( cfg.shared_memory_dir ),
+    blog( cfg.blocks_dir ),
+    fork_db( cfg.state_dir ),
     wasmif( cfg.wasm_runtime ),
     resource_limits( db ),
     authorization( s, db ),

libraries/chain/fork_database.cpp

@@ -55,7 +55,7 @@ namespace eosio { namespace chain {
       if (!fc::is_directory(my->datadir))
          fc::create_directories(my->datadir);
 
-      auto fork_db_dat = my->datadir / "forkdb.dat";
+      auto fork_db_dat = my->datadir / config::forkdb_filename;
       if( fc::exists( fork_db_dat ) ) {
          string content;
          fc::read_file_contents( fork_db_dat, content );
@@ -86,7 +86,7 @@ namespace eosio { namespace chain {
          states.push_back( *s );
       }
 
-      auto fork_db_dat = my->datadir / "forkdb.dat";
+      auto fork_db_dat = my->datadir / config::forkdb_filename;
       std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
       fc::raw::pack( out, states );
       if( my->head )

libraries/chain/include/eosio/chain/config.hpp

@@ -12,10 +12,14 @@ namespace eosio { namespace chain { namespace config {
 
 typedef __uint128_t uint128_t;
 
-const static auto default_block_log_dir     = "block_log";
-const static auto default_shared_memory_dir = "shared_mem";
-const static auto default_shared_memory_size = 1*1024*1024*1024ll;
-const static auto default_reversible_cache_size = 320*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
+const static auto default_blocks_dir_name    = "blocks";
+const static auto reversible_blocks_dir_name = "reversible";
+const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
+
+const static auto default_state_dir_name     = "state";
+const static auto forkdb_filename            = "forkdb.dat";
+const static auto default_state_size            = 1*1024*1024*1024ll;
+
 
 const static uint64_t system_account_name    = N(eosio);
 const static uint64_t null_account_name      = N(eosio.null);

libraries/chain/include/eosio/chain/controller.hpp

@@ -36,13 +36,13 @@ namespace eosio { namespace chain {
    class controller {
       public:
          struct config {
-            path         block_log_dir           =  chain::config::default_block_log_dir;
-            uint64_t     reversible_cache_size  =  chain::config::default_reversible_cache_size;;
-            path         shared_memory_dir   =  chain::config::default_shared_memory_dir;
-            uint64_t     shared_memory_size  =  chain::config::default_shared_memory_size;
-            bool         read_only           =  false;
-            bool         force_all_checks    =  false;
-            bool         contracts_console   =  false;
+            path         blocks_dir             =  chain::config::default_blocks_dir_name;
+            path         state_dir              =  chain::config::default_state_dir_name;
+            uint64_t     state_size             =  chain::config::default_state_size;
+            uint64_t     reversible_cache_size  =  chain::config::default_reversible_cache_size;
+            bool         read_only              =  false;
+            bool         force_all_checks       =  false;
+            bool         contracts_console      =  false;
 
             genesis_state                  genesis;
             wasm_interface::vm_type        wasm_runtime = chain::config::default_wasm_runtime;
@@ -204,9 +204,11 @@ namespace eosio { namespace chain {
 } }  /// eosio::chain
 
 FC_REFLECT( eosio::chain::controller::config,
-            (block_log_dir)
+            (blocks_dir)
+            (state_dir)
+            (state_size)
             (reversible_cache_size)
-            (shared_memory_dir)(shared_memory_size)(read_only)
+            (read_only)
             (force_all_checks)
             (contracts_console)
             (genesis)

chainbase @ b40a1ec9

-Subproject commit a8c07fca66fd56405ca29140b504123b6f360854
+Subproject commit b40a1ec99ee0eb16e28ee1c1a9577f1af7d48b84

libraries/testing/include/eosio/testing/tester.hpp

@@ -310,9 +310,10 @@ namespace eosio { namespace testing {
       controller::config vcfg;
 
       validating_tester() {
-         vcfg.block_log_dir      = tempdir.path() / "vblocklog";
-         vcfg.shared_memory_dir  = tempdir.path() / "vshared";
-         vcfg.shared_memory_size = 1024*1024*8;
+         vcfg.blocks_dir      = tempdir.path() / std::string("v_").append(config::default_blocks_dir_name);
+         vcfg.state_dir  = tempdir.path() /  std::string("v_").append(config::default_state_dir_name);
+         vcfg.state_size = 1024*1024*8;
+         vcfg.reversible_cache_size = 1024*1024*8;
 
          vcfg.genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000");
          vcfg.genesis.initial_key = get_public_key( config::system_account_name, "active" );

libraries/testing/tester.cpp

@@ -37,9 +37,9 @@ namespace eosio { namespace testing {
    }
 
    void base_tester::init(bool push_genesis) {
-      cfg.block_log_dir      = tempdir.path() / "blocklog";
-      cfg.shared_memory_dir  = tempdir.path() / "shared";
-      cfg.shared_memory_size = 1024*1024*8;
+      cfg.blocks_dir      = tempdir.path() / config::default_blocks_dir_name;
+      cfg.state_dir  = tempdir.path() / config::default_state_dir_name;
+      cfg.state_size = 1024*1024*8;
       cfg.reversible_cache_size = 1024*1024*8;
       cfg.contracts_console = true;
 

plugins/chain_plugin/chain_plugin.cpp

@@ -56,11 +56,10 @@ public:
    ,incoming_transaction_sync_method(app().get_method<incoming::methods::transaction_sync>())
    {}
 
-   bfs::path                        block_log_dir;
+   bfs::path                        blocks_dir;
    bfs::path                        genesis_file;
    time_point                       genesis_timestamp;
    bool                             readonly = false;
-   uint64_t                         shared_memory_size;
    flat_map<uint32_t,block_id_type> loaded_checkpoints;
 
    fc::optional<fork_database>      fork_db;
@@ -89,7 +88,7 @@ public:
    methods::get_block_by_id::method_type::handle                     get_block_by_id_provider;
    methods::get_head_block_id::method_type::handle                   get_head_block_id_provider;
    methods::get_last_irreversible_block_number::method_type::handle  get_last_irreversible_block_number_provider;
-   
+
    // scoped connections for chain controller
    fc::optional<scoped_connection>                                   accepted_block_header_connection;
    fc::optional<scoped_connection>                                   accepted_block_connection;
@@ -112,11 +111,13 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
    cfg.add_options()
          ("genesis-json", bpo::value<bfs::path>()->default_value("genesis.json"), "File to read Genesis State from")
          ("genesis-timestamp", bpo::value<string>(), "override the initial timestamp in the Genesis State file")
-         ("block-log-dir", bpo::value<bfs::path>()->default_value("blocks"),
-          "the location of the block log (absolute path or relative to application data dir)")
-         ("checkpoint,c", bpo::value<vector<string>>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.")
+         ("blocks-dir", bpo::value<bfs::path>()->default_value("blocks"),
+          "the location of the blocks directory (absolute path or relative to application data dir)")
+         ("checkpoint", bpo::value<vector<string>>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.")
          ("wasm-runtime", bpo::value<eosio::chain::wasm_interface::vm_type>()->value_name("wavm/binaryen"), "Override default WASM runtime")
-         ("shared-memory-size-mb", bpo::value<uint64_t>()->default_value(config::default_shared_memory_size / (1024  * 1024)), "Maximum size MB of database shared memory file")
+         ("chain-state-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_state_size / (1024  * 1024)), "Maximum size (in MB) of the chain state database")
+         ("reversible-blocks-db-size-mb", bpo::value<uint64_t>()->default_value(config::default_reversible_cache_size / (1024  * 1024)), "Maximum size (in MB) of the reversible blocks database")
+
 
 #warning TODO: rate limiting
          /*("per-authorized-account-transaction-msg-rate-limit-time-frame-sec", bpo::value<uint32_t>()->default_value(default_per_auth_account_time_frame_seconds),
@@ -134,11 +135,11 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("force-all-checks", bpo::bool_switch()->default_value(false),
           "do not skip any checks that can be skipped while replaying irreversible blocks")
          ("replay-blockchain", bpo::bool_switch()->default_value(false),
-          "clear chain database and replay all blocks")
+          "clear chain state database and replay all blocks")
          ("hard-replay-blockchain", bpo::bool_switch()->default_value(false),
-          "clear chain database, recover as many blocks as possible from the block log, and then replay those blocks")
-         ("resync-blockchain", bpo::bool_switch()->default_value(false),
-          "clear chain database and block log")
+          "clear chain state database, recover as many blocks as possible from the block log, and then replay those blocks")
+         ("delete-all-blocks", bpo::bool_switch()->default_value(false),
+          "clear chain state database and block log")
          ("contracts-console", bpo::bool_switch()->default_value(false),
           "print contract's output to console")
          ;
@@ -170,15 +171,12 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
        my->genesis_timestamp = time_point::from_iso_string (tstr);
      }
    }
-   if (options.count("block-log-dir")) {
-      auto bld = options.at("block-log-dir").as<bfs::path>();
+   if (options.count("blocks-dir")) {
+      auto bld = options.at("blocks-dir").as<bfs::path>();
       if(bld.is_relative())
-         my->block_log_dir = app().data_dir() / bld;
+         my->blocks_dir = app().data_dir() / bld;
       else
-         my->block_log_dir = bld;
-   }
-   if (options.count("shared-memory-size-mb")) {
-      my->shared_memory_size = options.at("shared-memory-size-mb").as<uint64_t>() * 1024 * 1024;
+         my->blocks_dir = bld;
    }
 
    if(options.count("checkpoint"))
@@ -195,10 +193,15 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
    if(options.count("wasm-runtime"))
       my->wasm_runtime = options.at("wasm-runtime").as<vm_type>();
 
-   my->chain_config->block_log_dir = my->block_log_dir;
-   my->chain_config->shared_memory_dir = app().data_dir() / default_shared_memory_dir;
+   my->chain_config->blocks_dir = my->blocks_dir;
+   my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name;
    my->chain_config->read_only = my->readonly;
-   my->chain_config->shared_memory_size = my->shared_memory_size;
+
+   if (options.count("chain-state-db-size-mb"))
+      my->chain_config->state_size = options.at("chain-state-db-size-mb").as<uint64_t>() * 1024 * 1024;
+
+   if (options.count("reversible-blocks-db-size-mb"))
+      my->chain_config->reversible_cache_size = options.at("reversible-blocks-db-size-mb").as<uint64_t>() * 1024 * 1024;
 
    if( my->wasm_runtime )
       my->chain_config->wasm_runtime = *my->wasm_runtime;
@@ -206,36 +209,36 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
    my->chain_config->force_all_checks  = options.at("force-all-checks").as<bool>();
    my->chain_config->contracts_console = options.at("contracts-console").as<bool>();
 
-   if( options.at("resync-blockchain").as<bool>() ) {
-      ilog("Resync requested: wiping database and blocks");
-      fc::remove_all(app().data_dir() / default_shared_memory_dir);
-      fc::remove_all(my->block_log_dir);
+   if( options.at("delete-all-blocks").as<bool>() ) {
+      ilog("Deleting state database and blocks");
+      fc::remove_all( my->chain_config->state_dir );
+      fc::remove_all(my->blocks_dir);
    } else if( options.at("hard-replay-blockchain").as<bool>() ) {
-      ilog("Hard replay requested: wiping database");
-      fc::remove_all(app().data_dir() / default_shared_memory_dir);
-      auto backup_dir = block_log::repair_log( my->block_log_dir );
-      if( fc::exists(backup_dir/"reversible") || options.at("fix-reversible-blocks").as<bool>() ) {
+      ilog("Hard replay requested: deleting state database");
+      fc::remove_all( my->chain_config->state_dir );
+      auto backup_dir = block_log::repair_log( my->blocks_dir );
+      if( fc::exists(backup_dir/config::reversible_blocks_dir_name) || options.at("fix-reversible-blocks").as<bool>() ) {
          // Do not try to recover reversible blocks if the directory does not exist, unless the option was explicitly provided.
-         if( !recover_reversible_blocks( backup_dir/"reversible",
+         if( !recover_reversible_blocks( backup_dir/config::reversible_blocks_dir_name,
                                           my->chain_config->reversible_cache_size,
-                                          my->chain_config->block_log_dir/"reversible" ) ) {
+                                          my->chain_config->blocks_dir/config::reversible_blocks_dir_name ) ) {
             ilog("Reversible blocks database was not corrupted. Copying from backup to blocks directory.");
-            fc::copy( backup_dir/"reversible", my->chain_config->block_log_dir/"reversible" );
-            fc::copy( backup_dir/"reversible/shared_memory.bin", my->chain_config->block_log_dir/"reversible/shared_memory.bin" );
-            fc::copy( backup_dir/"reversible/shared_memory.meta", my->chain_config->block_log_dir/"reversible/shared_memory.meta" );
+            fc::copy( backup_dir/config::reversible_blocks_dir_name, my->chain_config->blocks_dir/config::reversible_blocks_dir_name );
+            fc::copy( backup_dir/"reversible/shared_memory.bin", my->chain_config->blocks_dir/"reversible/shared_memory.bin" );
+            fc::copy( backup_dir/"reversible/shared_memory.meta", my->chain_config->blocks_dir/"reversible/shared_memory.meta" );
          }
       }
    } else if( options.at("replay-blockchain").as<bool>() ) {
-      ilog("Replay requested: wiping database");
-      fc::remove_all(app().data_dir() / default_shared_memory_dir);
+      ilog("Replay requested: deleting state database");
+      fc::remove_all( my->chain_config->state_dir );
       if( options.at("fix-reversible-blocks").as<bool>() ) {
-         if( !recover_reversible_blocks( my->chain_config->block_log_dir/"reversible",
+         if( !recover_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name,
                                           my->chain_config->reversible_cache_size ) ) {
             ilog("Reversible blocks database was not corrupted.");
          }
       }
    } else if( options.at("fix-reversible-blocks").as<bool>() ) {
-      if( !recover_reversible_blocks( my->chain_config->block_log_dir/"reversible",
+      if( !recover_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name,
                                        my->chain_config->reversible_cache_size ) ) {
          ilog("Reversible blocks database verified to not be corrupted. Now exiting...");
       } else {
@@ -409,7 +412,7 @@ bool chain_plugin::recover_reversible_blocks( const fc::path& db_dir, uint32_t c
 
    if( num == 0 )
       ilog( "There were no recoverable blocks in the reversible block database" );
-   else if( num == 0 )
+   else if( num == 1 )
       ilog( "Recovered 1 block from reversible block database: block ${start}", ("start", start) );
    else
       ilog( "Recovered ${num} blocks from reversible block database: blocks ${start} to ${end}",

programs/nodeos/main.cpp

@@ -114,6 +114,14 @@ int main(int argc, char** argv)
    } catch( const boost::exception& e ) {
       elog("${e}", ("e",boost::diagnostic_information(e)));
       return OTHER_FAIL;
+   } catch( const std::runtime_error& e ) {
+      if( std::string(e.what()) == "database dirty flag set" ) {
+         elog( "database dirty flag set (likely due to unclean shutdown): replay required" );
+      } else if( std::string(e.what()) == "database metadata dirty flag set" ) {
+         elog( "database metadata dirty flag set (likely due to unclean shutdown): replay required" );
+      } else {
+         elog( "${e}", ("e",e.what()));
+      }
    } catch( const std::exception& e ) {
       elog("${e}", ("e",e.what()));
       return OTHER_FAIL;

tests/chain_tests/block_tests.cpp

@@ -869,8 +869,8 @@ BOOST_FIXTURE_TEST_CASE(reindex, validating_tester)
       // Create shared configuration, so the new chain can be recreated from existing block log
       chain_controller::controller_config cfg;
       fc::temp_directory tempdir;
-      cfg.block_log_dir      = tempdir.path() / "blocklog";
-      cfg.shared_memory_dir  = tempdir.path() / "shared";
+      cfg.blocks_dir      = tempdir.path() / config::default_blocks_dir_name;
+      cfg.shared_memory_dir  = tempdir.path() / config::default_state_dir_name;
       cfg.genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000");
       cfg.genesis.initial_key = get_public_key( config::system_account_name, "active" );