From a23c6052c805ae39494fa536bb00ac93ffb2be85 Mon Sep 17 00:00:00 2001 From: sdong Date: Mon, 11 Apr 2016 11:39:51 -0700 Subject: [PATCH] Don't run DBOptionsAllFieldsSettable under valgrind Summary: Test DBOptionsAllFieldsSettable sometimes fails under valgrind. Move option settable tests to a separate test file and disable it in valgrind.. Test Plan: Run valgrind test and make sure the test doesn't run. Reviewers: andrewkr, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: kradhakrishnan, yiwu, yhchiang, leveldb, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D56529 --- CMakeLists.txt | 1 + Makefile | 6 +- util/options_settable_test.cc | 448 ++++++++++++++++++++++++++++++++++ util/options_test.cc | 388 ----------------------------- 4 files changed, 454 insertions(+), 389 deletions(-) create mode 100644 util/options_settable_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 0af644764..350ec95f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -399,6 +399,7 @@ set(TESTS util/iostats_context_test.cc util/memenv_test.cc util/mock_env_test.cc + util/options_settable_test.cc util/options_test.cc util/rate_limiter_test.cc util/slice_transform_test.cc diff --git a/Makefile b/Makefile index 9396798b4..193c13b21 100644 --- a/Makefile +++ b/Makefile @@ -330,6 +330,7 @@ TESTS = \ rate_limiter_test \ delete_scheduler_test \ options_test \ + options_settable_test \ options_util_test \ event_logger_test \ cuckoo_table_builder_test \ @@ -637,7 +638,7 @@ ubsan_crash_test: $(MAKE) clean valgrind_check: $(TESTS) - for t in $(filter-out %skiplist_test,$(TESTS)); do \ + for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \ $(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \ ret_code=$$?; \ if [ $$ret_code -ne 0 ]; then \ @@ -1024,6 +1025,9 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS) options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/util/options_settable_test.cc b/util/options_settable_test.cc new file mode 100644 index 000000000..0cbcdaee3 --- /dev/null +++ b/util/options_settable_test.cc @@ -0,0 +1,448 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include +#include +#include +#include + +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/leveldb_options.h" +#include "util/options_helper.h" +#include "util/options_parser.h" +#include "util/options_sanity_check.h" +#include "util/random.h" +#include "util/stderr_logger.h" +#include "util/testharness.h" +#include "util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include +using GFLAGS::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace rocksdb { + +// Verify options are settable from options strings. +// We take the approach that depends on compiler behavior that copy constructor +// won't touch implicit padding bytes, so that the test is fragile. +// As a result, we only run the tests to verify new fields in options are +// settable through string on limited platforms as it depends on behavior of +// compilers. +#ifndef ROCKSDB_LITE +#ifdef OS_LINUX +#ifndef __clang__ + +class OptionsSettableTest : public testing::Test { + public: + OptionsSettableTest() {} +}; + +const char kSpecialChar = 'z'; +typedef std::vector> OffsetGap; + +void FillWithSpecialChar(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { + size_t offset = 0; + for (auto& pair : blacklist) { + std::memset(start_ptr + offset, kSpecialChar, pair.first - offset); + offset = pair.first + pair.second; + } + std::memset(start_ptr + offset, kSpecialChar, total_size - offset); +} + +int NumUnsetBytes(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { + int total_unset_bytes_base = 0; + size_t offset = 0; + for (auto& pair : blacklist) { + for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + offset = pair.first + pair.second; + } + } + for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + } + return total_unset_bytes_base; +} + +// If the test fails, likely a new option is added to BlockBasedTableOptions +// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetBlockBasedTableOptionsFromString() and add the option to the input string +// passed to the GetBlockBasedTableOptionsFromString() in this test. +// If it is a complicated type, you also need to add the field to +// kBbtoBlacklist, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) { + // Items in the form of . Need to be in ascending order + // and not overlapping. Need to updated if new pointer-option is added. + const OffsetGap kBbtoBlacklist = { + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, block_cache), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, block_cache_compressed), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, filter_policy), + sizeof(std::shared_ptr)}, + }; + + // In this test, we catch a new option of BlockBasedTableOptions that is not + // settable through GetBlockBasedTableOptionsFromString(). + // We count padding bytes of the option struct, and assert it to be the same + // as unset bytes of an option struct initialized by + // GetBlockBasedTableOptionsFromString(). + + char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *bbto = BlockBasedTableOptions(); + int unset_bytes_base = + NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + ASSERT_GT(unset_bytes_base, 0); + bbto->~BlockBasedTableOptions(); + + // Construct the base option passed into + // GetBlockBasedTableOptionsFromString(). + bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + // This option is not setable: + bbto->use_delta_encoding = true; + + char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + BlockBasedTableOptions* new_bbto = + new (new_bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + *bbto, + "cache_index_and_filter_blocks=1;" + "pin_l0_filter_and_index_blocks_in_cache=1;" + "index_type=kHashSearch;" + "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4; " + "index_block_restart_interval=4;" + "filter_policy=bloomfilter:4:true;whole_key_filtering=1;" + "skip_table_builder_flush=1;format_version=1;" + "hash_index_allow_collision=false;", + new_bbto)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist)); + + ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); + ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr); + ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr); + + bbto->~BlockBasedTableOptions(); + new_bbto->~BlockBasedTableOptions(); + + delete[] bbto_ptr; + delete[] new_bbto_ptr; +} + +// If the test fails, likely a new option is added to DBOptions +// but it cannot be set through GetDBOptionsFromString(), or the test is not +// updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetDBOptionsFromString() and add the option to the input string passed to +// DBOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kDBOptionsBlacklist, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { + const OffsetGap kDBOptionsBlacklist = { + {offsetof(struct DBOptions, env), sizeof(Env*)}, + {offsetof(struct DBOptions, rate_limiter), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, sst_file_manager), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, statistics), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, db_paths), sizeof(std::vector)}, + {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, listeners), + sizeof(std::vector>)}, + {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, + }; + + char* options_ptr = new char[sizeof(DBOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + DBOptions* options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = DBOptions(); + int unset_bytes_base = + NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~DBOptions(); + + options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + char* new_options_ptr = new char[sizeof(DBOptions)]; + DBOptions* new_options = new (new_options_ptr) DBOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK( + GetDBOptionsFromString(*options, + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "base_background_compactions=3;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "new_table_reader_for_compaction_inputs=false;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "db_log_dir=path/to/db_log_dir;" + "skip_log_error_on_recovery=true;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "enable_thread_tracking=false;" + "disable_data_sync=false;" + "recycle_log_file_num=0;" + "disableDataSync=false;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "allow_os_buffer=false;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "max_log_file_size=4607;" + "random_access_max_buffer_size=1048576;" + "advise_random_on_open=true;" + "fail_if_options_file_error=false;" + "allow_concurrent_memtable_write=true;" + "wal_recovery_mode=kPointInTimeRecovery;" + "enable_write_thread_adaptive_yield=true;" + "write_thread_slow_yield_usec=5;" + "write_thread_max_yield_usec=1000;" + "access_hint_on_compaction_start=NONE;" + "info_log_level=DEBUG_LEVEL;", + new_options)); + + ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), + kDBOptionsBlacklist)); + + options->~DBOptions(); + new_options->~DBOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} + +// If the test fails, likely a new option is added to ColumnFamilyOptions +// but it cannot be set through GetColumnFamilyOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetColumnFamilyOptionsFromString() and add the option to the input +// string passed to GetColumnFamilyOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kColumnFamilyOptionsBlacklist, and maybe add customized verification +// for it. +TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { + const OffsetGap kColumnFamilyOptionsBlacklist = { + {offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)}, + {offsetof(struct ColumnFamilyOptions, merge_operator), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter), + sizeof(const CompactionFilter*)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compression_per_level), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, prefix_extractor), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, + max_bytes_for_level_multiplier_additional), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, memtable_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, table_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, + table_properties_collector_factories), + sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, + {offsetof(struct ColumnFamilyOptions, inplace_callback), + sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))}, + }; + + char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = ColumnFamilyOptions(); + + // Deprecatd option which is not initialized. Need to set it to avoid + // Valgrind error + options->max_mem_compaction_level = 0; + + int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~ColumnFamilyOptions(); + + options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Following options are not settable through + // GetColumnFamilyOptionsFromString(): + options->rate_limit_delay_max_milliseconds = 33; + options->compaction_pri = CompactionPri::kOldestSmallestSeqFirst; + options->compaction_options_universal = CompactionOptionsUniversal(); + options->compression_opts = CompressionOptions(); + options->hard_rate_limit = 0; + options->soft_rate_limit = 0; + options->compaction_options_fifo = CompactionOptionsFIFO(); + options->max_mem_compaction_level = 0; + + char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; + ColumnFamilyOptions* new_options = + new (new_options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK(GetColumnFamilyOptionsFromString( + *options, + "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" + "table_factory=PlainTable;" + "prefix_extractor=rocksdb.CappedPrefix.13;" + "comparator=leveldb.BytewiseComparator;" + "compression_per_level=kBZip2Compression:kBZip2Compression:" + "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" + "kSnappyCompression;" + "max_bytes_for_level_base=986;" + "bloom_locality=8016;" + "target_file_size_base=4294976376;" + "memtable_prefix_bloom_huge_page_tlb_size=2557;" + "max_successive_merges=5497;" + "max_sequential_skip_in_iterations=4294971408;" + "arena_block_size=1893;" + "target_file_size_multiplier=35;" + "source_compaction_factor=54;" + "min_write_buffer_number_to_merge=9;" + "max_write_buffer_number=84;" + "write_buffer_size=1653;" + "max_grandparent_overlap_factor=64;" + "max_bytes_for_level_multiplier=60;" + "memtable_factory=SkipListFactory;" + "compression=kNoCompression;" + "min_partial_merge_operands=7576;" + "level0_stop_writes_trigger=33;" + "num_levels=99;" + "level0_slowdown_writes_trigger=22;" + "level0_file_num_compaction_trigger=14;" + "expanded_compaction_factor=34;" + "compaction_filter=urxcqstuwnCompactionFilter;" + "soft_rate_limit=530.615385;" + "soft_pending_compaction_bytes_limit=0;" + "max_write_buffer_number_to_maintain=84;" + "verify_checksums_in_compaction=false;" + "merge_operator=aabcxehazrMergeOperator;" + "memtable_prefix_bloom_bits=4642;" + "paranoid_file_checks=true;" + "inplace_update_num_locks=7429;" + "optimize_filters_for_hits=false;" + "level_compaction_dynamic_level_bytes=false;" + "inplace_update_support=false;" + "compaction_style=kCompactionStyleFIFO;" + "memtable_prefix_bloom_probes=2511;" + "purge_redundant_kvs_while_flush=true;" + "filter_deletes=false;" + "hard_pending_compaction_bytes_limit=0;" + "disable_auto_compactions=false;" + "compaction_measure_io_stats=true;", + new_options)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist)); + + options->~ColumnFamilyOptions(); + new_options->~ColumnFamilyOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} +#endif // !__clang__ +#endif // OS_LINUX +#endif // !ROCKSDB_LITE + +} // namespace rocksdb + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/util/options_test.cc b/util/options_test.cc index 65c478757..43671b6b8 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -1497,395 +1497,7 @@ TEST_F(OptionsParserTest, EscapeOptionString) { "Escape \\# and # comment together ."), "Escape \\# and"); } - -// Only run the tests to verify new fields in options are settable through -// string on limited platforms as it depends on behavior of compilers. -#ifdef OS_LINUX -#ifndef __clang__ -const char kSpecialChar = 'z'; -typedef std::vector> OffsetGap; - -void FillWithSpecialChar(char* start_ptr, size_t total_size, - const OffsetGap& blacklist) { - size_t offset = 0; - for (auto& pair : blacklist) { - std::memset(start_ptr + offset, kSpecialChar, pair.first - offset); - offset = pair.first + pair.second; - } - std::memset(start_ptr + offset, kSpecialChar, total_size - offset); -} - -int NumUnsetBytes(char* start_ptr, size_t total_size, - const OffsetGap& blacklist) { - int total_unset_bytes_base = 0; - size_t offset = 0; - for (auto& pair : blacklist) { - for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { - if (*ptr == kSpecialChar) { - total_unset_bytes_base++; - } - offset = pair.first + pair.second; - } - } - for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { - if (*ptr == kSpecialChar) { - total_unset_bytes_base++; - } - } - return total_unset_bytes_base; -} - -// If the test fails, likely a new option is added to BlockBasedTableOptions -// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the -// test is not updated accordingly. -// After adding an option, we need to make sure it is settable by -// GetBlockBasedTableOptionsFromString() and add the option to the input string -// passed to the GetBlockBasedTableOptionsFromString() in this test. -// If it is a complicated type, you also need to add the field to -// kBbtoBlacklist, and maybe add customized verification for it. -TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) { - // Items in the form of . Need to be in ascending order - // and not overlapping. Need to updated if new pointer-option is added. - const OffsetGap kBbtoBlacklist = { - {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, block_cache), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, block_cache_compressed), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, filter_policy), - sizeof(std::shared_ptr)}, - }; - - // In this test, we catch a new option of BlockBasedTableOptions that is not - // settable through GetBlockBasedTableOptionsFromString(). - // We count padding bytes of the option struct, and assert it to be the same - // as unset bytes of an option struct initialized by - // GetBlockBasedTableOptionsFromString(). - - char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; - - // Count padding bytes by setting all bytes in the memory to a special char, - // copy a well constructed struct to this memory and see how many special - // bytes left. - BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); - FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); - // It based on the behavior of compiler that padding bytes are not changed - // when copying the struct. It's prone to failure when compiler behavior - // changes. We verify there is unset bytes to detect the case. - *bbto = BlockBasedTableOptions(); - int unset_bytes_base = - NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); - ASSERT_GT(unset_bytes_base, 0); - bbto->~BlockBasedTableOptions(); - - // Construct the base option passed into - // GetBlockBasedTableOptionsFromString(). - bbto = new (bbto_ptr) BlockBasedTableOptions(); - FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); - // This option is not setable: - bbto->use_delta_encoding = true; - - char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; - BlockBasedTableOptions* new_bbto = - new (new_bbto_ptr) BlockBasedTableOptions(); - FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), - kBbtoBlacklist); - - // Need to update the option string if a new option is added. - ASSERT_OK(GetBlockBasedTableOptionsFromString( - *bbto, - "cache_index_and_filter_blocks=1;" - "pin_l0_filter_and_index_blocks_in_cache=1;" - "index_type=kHashSearch;" - "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" - "block_cache=1M;block_cache_compressed=1k;block_size=1024;" - "block_size_deviation=8;block_restart_interval=4; " - "index_block_restart_interval=4;" - "filter_policy=bloomfilter:4:true;whole_key_filtering=1;" - "skip_table_builder_flush=1;format_version=1;" - "hash_index_allow_collision=false;", - new_bbto)); - - ASSERT_EQ(unset_bytes_base, - NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), - kBbtoBlacklist)); - - ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); - ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr); - ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr); - - bbto->~BlockBasedTableOptions(); - new_bbto->~BlockBasedTableOptions(); - - delete[] bbto_ptr; - delete[] new_bbto_ptr; -} - -// If the test fails, likely a new option is added to DBOptions -// but it cannot be set through GetDBOptionsFromString(), or the test is not -// updated accordingly. -// After adding an option, we need to make sure it is settable by -// GetDBOptionsFromString() and add the option to the input string passed to -// DBOptionsFromString()in this test. -// If it is a complicated type, you also need to add the field to -// kDBOptionsBlacklist, and maybe add customized verification for it. -TEST_F(OptionsParserTest, DBOptionsAllFieldsSettable) { - const OffsetGap kDBOptionsBlacklist = { - {offsetof(struct DBOptions, env), sizeof(Env*)}, - {offsetof(struct DBOptions, rate_limiter), - sizeof(std::shared_ptr)}, - {offsetof(struct DBOptions, sst_file_manager), - sizeof(std::shared_ptr)}, - {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr)}, - {offsetof(struct DBOptions, statistics), - sizeof(std::shared_ptr)}, - {offsetof(struct DBOptions, db_paths), sizeof(std::vector)}, - {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, - {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, - {offsetof(struct DBOptions, listeners), - sizeof(std::vector>)}, - {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr)}, - {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, - }; - - char* options_ptr = new char[sizeof(DBOptions)]; - - // Count padding bytes by setting all bytes in the memory to a special char, - // copy a well constructed struct to this memory and see how many special - // bytes left. - DBOptions* options = new (options_ptr) DBOptions(); - FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); - // It based on the behavior of compiler that padding bytes are not changed - // when copying the struct. It's prone to failure when compiler behavior - // changes. We verify there is unset bytes to detect the case. - *options = DBOptions(); - int unset_bytes_base = - NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); - ASSERT_GT(unset_bytes_base, 0); - options->~DBOptions(); - - options = new (options_ptr) DBOptions(); - FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); - - char* new_options_ptr = new char[sizeof(DBOptions)]; - DBOptions* new_options = new (new_options_ptr) DBOptions(); - FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); - - // Need to update the option string if a new option is added. - ASSERT_OK( - GetDBOptionsFromString(*options, - "wal_bytes_per_sync=4295048118;" - "delete_obsolete_files_period_micros=4294967758;" - "WAL_ttl_seconds=4295008036;" - "WAL_size_limit_MB=4295036161;" - "wal_dir=path/to/wal_dir;" - "db_write_buffer_size=2587;" - "max_subcompactions=64330;" - "table_cache_numshardbits=28;" - "max_open_files=72;" - "max_file_opening_threads=35;" - "base_background_compactions=3;" - "max_background_compactions=33;" - "use_fsync=true;" - "use_adaptive_mutex=false;" - "max_total_wal_size=4295005604;" - "compaction_readahead_size=0;" - "new_table_reader_for_compaction_inputs=false;" - "keep_log_file_num=4890;" - "skip_stats_update_on_db_open=false;" - "max_manifest_file_size=4295009941;" - "db_log_dir=path/to/db_log_dir;" - "skip_log_error_on_recovery=true;" - "writable_file_max_buffer_size=1048576;" - "paranoid_checks=true;" - "is_fd_close_on_exec=false;" - "bytes_per_sync=4295013613;" - "enable_thread_tracking=false;" - "disable_data_sync=false;" - "recycle_log_file_num=0;" - "disableDataSync=false;" - "create_missing_column_families=true;" - "log_file_time_to_roll=3097;" - "max_background_flushes=35;" - "create_if_missing=false;" - "error_if_exists=true;" - "allow_os_buffer=false;" - "delayed_write_rate=4294976214;" - "manifest_preallocation_size=1222;" - "allow_mmap_writes=false;" - "stats_dump_period_sec=70127;" - "allow_fallocate=true;" - "allow_mmap_reads=false;" - "max_log_file_size=4607;" - "random_access_max_buffer_size=1048576;" - "advise_random_on_open=true;" - "fail_if_options_file_error=false;" - "allow_concurrent_memtable_write=true;" - "wal_recovery_mode=kPointInTimeRecovery;" - "enable_write_thread_adaptive_yield=true;" - "write_thread_slow_yield_usec=5;" - "write_thread_max_yield_usec=1000;" - "access_hint_on_compaction_start=NONE;" - "info_log_level=DEBUG_LEVEL;", - new_options)); - - ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), - kDBOptionsBlacklist)); - - options->~DBOptions(); - new_options->~DBOptions(); - - delete[] options_ptr; - delete[] new_options_ptr; -} - -// If the test fails, likely a new option is added to ColumnFamilyOptions -// but it cannot be set through GetColumnFamilyOptionsFromString(), or the -// test is not updated accordingly. -// After adding an option, we need to make sure it is settable by -// GetColumnFamilyOptionsFromString() and add the option to the input -// string passed to GetColumnFamilyOptionsFromString()in this test. -// If it is a complicated type, you also need to add the field to -// kColumnFamilyOptionsBlacklist, and maybe add customized verification -// for it. -TEST_F(OptionsParserTest, ColumnFamilyOptionsAllFieldsSettable) { - const OffsetGap kColumnFamilyOptionsBlacklist = { - {offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)}, - {offsetof(struct ColumnFamilyOptions, merge_operator), - sizeof(std::shared_ptr)}, - {offsetof(struct ColumnFamilyOptions, compaction_filter), - sizeof(const CompactionFilter*)}, - {offsetof(struct ColumnFamilyOptions, compaction_filter_factory), - sizeof(std::shared_ptr)}, - {offsetof(struct ColumnFamilyOptions, compression_per_level), - sizeof(std::vector)}, - {offsetof(struct ColumnFamilyOptions, prefix_extractor), - sizeof(std::shared_ptr)}, - {offsetof(struct ColumnFamilyOptions, - max_bytes_for_level_multiplier_additional), - sizeof(std::vector)}, - {offsetof(struct ColumnFamilyOptions, memtable_factory), - sizeof(std::shared_ptr)}, - {offsetof(struct ColumnFamilyOptions, table_factory), - sizeof(std::shared_ptr)}, - {offsetof(struct ColumnFamilyOptions, - table_properties_collector_factories), - sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, - {offsetof(struct ColumnFamilyOptions, inplace_callback), - sizeof(UpdateStatus (*)(char*, uint32_t*, Slice, std::string*))}, - }; - - char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; - - // Count padding bytes by setting all bytes in the memory to a special char, - // copy a well constructed struct to this memory and see how many special - // bytes left. - ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); - FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), - kColumnFamilyOptionsBlacklist); - // It based on the behavior of compiler that padding bytes are not changed - // when copying the struct. It's prone to failure when compiler behavior - // changes. We verify there is unset bytes to detect the case. - *options = ColumnFamilyOptions(); - - // Deprecatd option which is not initialized. Need to set it to avoid - // Valgrind error - options->max_mem_compaction_level = 0; - - int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), - kColumnFamilyOptionsBlacklist); - ASSERT_GT(unset_bytes_base, 0); - options->~ColumnFamilyOptions(); - - options = new (options_ptr) ColumnFamilyOptions(); - FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), - kColumnFamilyOptionsBlacklist); - - // Following options are not settable through - // GetColumnFamilyOptionsFromString(): - options->rate_limit_delay_max_milliseconds = 33; - options->compaction_pri = CompactionPri::kOldestSmallestSeqFirst; - options->compaction_options_universal = CompactionOptionsUniversal(); - options->compression_opts = CompressionOptions(); - options->hard_rate_limit = 0; - options->soft_rate_limit = 0; - options->compaction_options_fifo = CompactionOptionsFIFO(); - options->max_mem_compaction_level = 0; - - char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; - ColumnFamilyOptions* new_options = - new (new_options_ptr) ColumnFamilyOptions(); - FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), - kColumnFamilyOptionsBlacklist); - - // Need to update the option string if a new option is added. - ASSERT_OK(GetColumnFamilyOptionsFromString( - *options, - "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" - "table_factory=PlainTable;" - "prefix_extractor=rocksdb.CappedPrefix.13;" - "comparator=leveldb.BytewiseComparator;" - "compression_per_level=kBZip2Compression:kBZip2Compression:" - "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" - "kSnappyCompression;" - "max_bytes_for_level_base=986;" - "bloom_locality=8016;" - "target_file_size_base=4294976376;" - "memtable_prefix_bloom_huge_page_tlb_size=2557;" - "max_successive_merges=5497;" - "max_sequential_skip_in_iterations=4294971408;" - "arena_block_size=1893;" - "target_file_size_multiplier=35;" - "source_compaction_factor=54;" - "min_write_buffer_number_to_merge=9;" - "max_write_buffer_number=84;" - "write_buffer_size=1653;" - "max_grandparent_overlap_factor=64;" - "max_bytes_for_level_multiplier=60;" - "memtable_factory=SkipListFactory;" - "compression=kNoCompression;" - "min_partial_merge_operands=7576;" - "level0_stop_writes_trigger=33;" - "num_levels=99;" - "level0_slowdown_writes_trigger=22;" - "level0_file_num_compaction_trigger=14;" - "expanded_compaction_factor=34;" - "compaction_filter=urxcqstuwnCompactionFilter;" - "soft_rate_limit=530.615385;" - "soft_pending_compaction_bytes_limit=0;" - "max_write_buffer_number_to_maintain=84;" - "verify_checksums_in_compaction=false;" - "merge_operator=aabcxehazrMergeOperator;" - "memtable_prefix_bloom_bits=4642;" - "paranoid_file_checks=true;" - "inplace_update_num_locks=7429;" - "optimize_filters_for_hits=false;" - "level_compaction_dynamic_level_bytes=false;" - "inplace_update_support=false;" - "compaction_style=kCompactionStyleFIFO;" - "memtable_prefix_bloom_probes=2511;" - "purge_redundant_kvs_while_flush=true;" - "filter_deletes=false;" - "hard_pending_compaction_bytes_limit=0;" - "disable_auto_compactions=false;" - "compaction_measure_io_stats=true;", - new_options)); - - ASSERT_EQ(unset_bytes_base, - NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), - kColumnFamilyOptionsBlacklist)); - - options->~ColumnFamilyOptions(); - new_options->~ColumnFamilyOptions(); - - delete[] options_ptr; - delete[] new_options_ptr; -} -#endif // !__clang__ -#endif // OS_LINUX #endif // !ROCKSDB_LITE - } // namespace rocksdb int main(int argc, char** argv) { -- GitLab