提交 8ee2b2b9 编写于 作者: W wangyi.ywq 提交者: 奏之章

[feature] add test to cover test table propertites after compaction

上级 65696a3d
......@@ -1025,6 +1025,7 @@ if(WITH_TESTS)
table/merger_test.cc
table/sst_file_reader_test.cc
table/table_test.cc
table/terark_zip_table_row_ttl_test.cc
tools/ldb_cmd_test.cc
tools/reduce_levels_test.cc
tools/sst_dump_test.cc
......
......@@ -132,6 +132,7 @@
#include <iostream>
namespace TERARKDB_NAMESPACE {
const std::string kDefaultColumnFamilyName("default");
const uint64_t kDumpStatsWaitMicroseconds = 10000;
const std::string kPersistentStatsColumnFamilyName(
......@@ -895,6 +896,8 @@ void DBImpl::ScheduleGCTTL() {
// ROCKS_LOG_INFO(immutable_db_options_.info_log,
// "SST Table property info:%" PRIu64 ",%" PRIu64 ",%"
// PRIu64, ratio_expire_time, scan_gap_expire_time, now);
printf("SST Table property info:%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n",
ratio_expire_time, scan_gap_expire_time, now);
return (std::min(ratio_expire_time, scan_gap_expire_time) <= now);
};
ROCKS_LOG_INFO(immutable_db_options_.info_log, "Start ScheduleGCTTL");
......@@ -905,6 +908,8 @@ void DBImpl::ScheduleGCTTL() {
VersionStorageInfo* vsi = cfd->current()->storage_info();
for (int l = 0; l < vsi->num_levels(); l++) {
for (auto sst : vsi->LevelFiles(l)) {
TEST_SYNC_POINT("DBImpl:Exist-SST");
cnt++;
if (sst->marked_for_compaction) {
marked_count++;
} else if (should_marked_for_compacted(sst->prop.ratio_expire_time,
......
......@@ -4,25 +4,24 @@
#include "db/db_impl.h"
#include "db/db_test_util.h"
#include "db/periodic_work_scheduler.h"
#include "rocksdb/terark_namespace.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/testharness.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
class DBImplGCTTL_Test : public DBTestBase {
public:
DBImplGCTTL_Test()
: DBTestBase("./db_GC_ttl_test"),
mock_env_(new MockTimeEnv(Env::Default())) {}
~DBImplGCTTL_Test(){
Close();
}
~DBImplGCTTL_Test() { Close(); }
void init() {
dbname = test::PerThreadDBPath("ttl_gc_test");
DestroyDB(dbname, options);
options.create_if_missing = true;
options.ttl_garbage_collection_percentage = 50.0;
options.ttl_garbage_collection_percentage = 0.50;
options.ttl_scan_gap = 10;
options.ttl_extractor_factory.reset(new test::TestTtlExtractorFactory());
options.level0_file_num_compaction_trigger = 8;
......@@ -31,9 +30,8 @@ class DBImplGCTTL_Test : public DBTestBase {
new BlockBasedTableFactory(BlockBasedTableOptions()));
}
void run(){
void run() {
int L0FilesNums = 4;
uint64_t ttl = 200;
options.env = mock_env_.get();
SetUp();
Reopen(options);
......@@ -51,18 +49,29 @@ class DBImplGCTTL_Test : public DBTestBase {
}
dbfull()->Flush(FlushOptions());
}
// dbfull()->StartPeriodicWorkScheduler();
dbfull()->TEST_WaitForStatsDumpRun([&] { mock_env_->set_current_time(ttl); });
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(ttl); });
ASSERT_TRUE(flag);
ASSERT_EQ(L0FilesNums, mark);
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
dbfull()->TEST_WaitForCompact();
}
void read() {
mark = 0;
cnt = 0;
dbfull()->TEST_WaitForStatsDumpRun(
[&] { mock_env_->set_current_time(ttl + ttl); });
ASSERT_EQ(mark, cnt);
}
protected:
std::unique_ptr<rocksdb::MockTimeEnv> mock_env_;
std::unique_ptr<TERARKDB_NAMESPACE::MockTimeEnv> mock_env_;
Options options;
std::string dbname;
bool flag = false;
int mark = 0;
int cnt = 0;
uint64_t ttl = 200;
void SetUp() override {
mock_env_->InstallTimedWaitFixCallback();
......@@ -73,33 +82,33 @@ class DBImplGCTTL_Test : public DBTestBase {
*periodic_work_scheduler_ptr =
PeriodicWorkTestScheduler::Default(mock_env_.get());
});
rocksdb::SyncPoint::GetInstance()->SetCallBack("DBImpl:ScheduleGCTTL",
[&](void* /*arg*/) {
mark = 0;
flag = true;
});
rocksdb::SyncPoint::GetInstance()->SetCallBack(
TERARKDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"DBImpl:ScheduleGCTTL", [&](void* /*arg*/) {
mark = 0;
flag = true;
});
TERARKDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"DBImpl:ScheduleGCTTL-mark", [&](void* /*arg*/) { mark++; });
TERARKDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"DBImpl:Exist-SST", [&](void* /*arg*/) { cnt++; });
}
};
TEST_F(DBImplGCTTL_Test, BlockBasedTableTest) {
init();
run();
dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
dbfull()->TEST_WaitForCompact();
dbfull()->ScheduleGCTTL();
read();
}
TEST_F(DBImplGCTTL_Test, TerarkTableTest) {
init();
TerarkZipTableOptions terarkziptableoptions;
options.table_factory.reset(TERARKDB_NAMESPACE::NewTerarkZipTableFactory(
terarkziptableoptions, options.table_factory));
terarkziptableoptions, options.table_factory));
run();
read();
}
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
......
......@@ -369,6 +369,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
// incorrect.
options.stats_dump_period_sec = 0;
options.stats_persist_period_sec = 0;
options.env = env_;
int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec
......
......@@ -6,10 +6,11 @@
#include "db/periodic_work_scheduler.h"
#include "db/db_impl.h"
#include "rocksdb/terark_namespace.h"
#include "util/cast_util.h"
#ifndef ROCKSDB_LITE
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
PeriodicWorkScheduler::PeriodicWorkScheduler(Env* env) {
timer = std::unique_ptr<Timer>(new Timer(env));
......@@ -42,7 +43,7 @@ void PeriodicWorkScheduler::Register(DBImpl* dbi,
timer->Add([dbi]() { dbi->ScheduleGCTTL(); },
GetTaskName(dbi, "schedule_gc_ttl"),
initial_delay.fetch_add(1) % kDefaultScheduleGCTTLPeriodSec *
kMicrosInSecond,
kMicrosInSecond,
kDefaultScheduleGCTTLPeriodSec * kMicrosInSecond);
}
......@@ -109,6 +110,6 @@ PeriodicWorkTestScheduler::PeriodicWorkTestScheduler(Env* env)
: PeriodicWorkScheduler(env) {}
#endif // !NDEBUG
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
#endif // ROCKSDB_LITE
......@@ -8,9 +8,10 @@
#ifndef ROCKSDB_LITE
#include "db/db_impl.h"
#include "rocksdb/terark_namespace.h"
#include "util/timer.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// PeriodicWorkScheduler is a singleton object, which is scheduling/running
// DumpStats(), PersistStats(), and FlushInfoLog() for all DB instances. All DB
......@@ -66,6 +67,6 @@ class PeriodicWorkTestScheduler : public PeriodicWorkScheduler {
};
#endif // !NDEBUG
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
#endif // ROCKSDB_LITE
......@@ -11,11 +11,11 @@
#include <map>
#include <string>
// #include "db/db_impl.h"
#include "rocksdb/statistics.h"
#include "rocksdb/status.h"
#include "rocksdb/terark_namespace.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
class DBImpl;
......@@ -48,4 +48,4 @@ class StatsHistoryIterator {
virtual Status status() const = 0;
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -7,10 +7,11 @@
#include <string>
#include "rocksdb/status.h"
#include "rocksdb/terark_namespace.h"
#include "rocksdb/types.h"
#include "utilities/util/factory.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
class Slice;
class Status;
......@@ -49,4 +50,4 @@ class TtlExtractorFactory
}
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -582,7 +582,7 @@ int main(int argc, char** argv) {
factory.reset(new TERARKDB_NAMESPACE::SkipListFactory);
#ifndef ROCKSDB_LITE
} else if (FLAGS_memtablerep == "patricia_trie") {
factory.reset(rocksdb::NewPatriciaTrieRepFactory());
factory.reset(TERARKDB_NAMESPACE::NewPatriciaTrieRepFactory());
} else if (FLAGS_memtablerep == "vector") {
factory.reset(new TERARKDB_NAMESPACE::VectorRepFactory);
} else if (FLAGS_memtablerep == "hashskiplist") {
......
......@@ -6,10 +6,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/db_impl.h"
#include "monitoring/in_memory_stats_history.h"
namespace rocksdb {
#include "db/db_impl.h"
#include "rocksdb/terark_namespace.h"
namespace TERARKDB_NAMESPACE {
InMemoryStatsHistoryIterator::~InMemoryStatsHistoryIterator() {}
......@@ -42,4 +44,4 @@ void InMemoryStatsHistoryIterator::AdvanceIteratorByTime(uint64_t start_time,
}
}
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -9,8 +9,9 @@
#pragma once
#include "rocksdb/stats_history.h"
#include "rocksdb/terark_namespace.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
class InMemoryStatsHistoryIterator final : public StatsHistoryIterator {
public:
......@@ -54,4 +55,4 @@ class InMemoryStatsHistoryIterator final : public StatsHistoryIterator {
DBImpl* db_impl_;
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -11,11 +11,13 @@
#include <cstring>
#include <string>
#include <utility>
#include "db/db_impl.h"
#include "port/likely.h"
#include "rocksdb/terark_namespace.h"
#include "util/string_util.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// 10 digit seconds timestamp => [Sep 9, 2001 ~ Nov 20, 2286]
const int kNowSecondsStringLength = 10;
const std::string kFormatVersionKeyString =
......@@ -69,7 +71,7 @@ void OptimizeForPersistentStats(ColumnFamilyOptions* cfo) {
cfo->write_buffer_size = 2 << 20;
cfo->target_file_size_base = 2 * 1048576;
cfo->max_bytes_for_level_base = 10 * 1048576;
// cfo->snap_refresh_nanos = 0;
// cfo->snap_refresh_nanos = 0;
cfo->soft_pending_compaction_bytes_limit = 256 * 1048576;
cfo->hard_pending_compaction_bytes_limit = 1073741824ul;
cfo->compression = kNoCompression;
......@@ -168,4 +170,4 @@ void PersistentStatsHistoryIterator::AdvanceIteratorByTime(uint64_t start_time,
}
}
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -10,8 +10,9 @@
#include "db/db_impl.h"
#include "rocksdb/stats_history.h"
#include "rocksdb/terark_namespace.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
extern const std::string kFormatVersionKeyString;
extern const std::string kCompatibleVersionKeyString;
......@@ -80,4 +81,4 @@ class PersistentStatsHistoryIterator final : public StatsHistoryIterator {
DBImpl* db_impl_;
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -8,9 +8,10 @@
#ifndef ROCKSDB_LITE
#include "db/db_impl.h"
#include "rocksdb/terark_namespace.h"
#include "util/timer.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// StatsDumpScheduler is a singleton object, which is scheduling/running
// DumpStats() and PersistStats() for all DB instances. All DB instances uses
......@@ -59,6 +60,6 @@ class StatsDumpTestScheduler : public StatsDumpScheduler {
};
#endif // !NDEBUG
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
#endif // ROCKSDB_LITE
......@@ -22,11 +22,12 @@
#include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
#include "rocksdb/rate_limiter.h"
#include "rocksdb/terark_namespace.h"
#include "util/random.h"
#include "util/sync_point.h"
#include "util/testutil.h"
#include "util/random.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
#ifndef ROCKSDB_LITE
class StatsHistoryTest : public DBTestBase {
......@@ -642,7 +643,7 @@ TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
}
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
int main(int argc, char** argv) {
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
......
......@@ -20,13 +20,14 @@ int main() {
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "rocksdb/terark_namespace.h"
#include "rocksdb/trace_reader_writer.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "tools/block_cache_trace_analyzer.h"
#include "trace_replay/block_cache_tracer.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
namespace {
const uint64_t kBlockSize = 1024;
......@@ -45,7 +46,7 @@ class BlockCacheTracerTest : public testing::Test {
public:
BlockCacheTracerTest() {
test_path_ = test::PerThreadDBPath("block_cache_tracer_test");
env_ = rocksdb::Env::Default();
env_ = TERARKDB_NAMESPACE::Env::Default();
EXPECT_OK(env_->CreateDir(test_path_));
trace_file_path_ = test_path_ + "/block_cache_trace";
block_cache_sim_config_path_ = test_path_ + "/block_cache_sim_config";
......@@ -176,7 +177,8 @@ class BlockCacheTracerTest : public testing::Test {
argv[argc++] = arg_buffer + cursor;
cursor += static_cast<int>(arg.size()) + 1;
}
ASSERT_EQ(0, rocksdb::block_cache_trace_analyzer_tool(argc, argv));
ASSERT_EQ(0,
TERARKDB_NAMESPACE::block_cache_trace_analyzer_tool(argc, argv));
}
Env* env_;
......@@ -235,9 +237,9 @@ TEST_F(BlockCacheTracerTest, BlockCacheAnalyzer) {
ASSERT_EQ("lru", result_strs[0]); // cache_name
ASSERT_EQ("1", result_strs[1]); // num_shard_bits
ASSERT_EQ(std::to_string(expected_capacities[config_index]),
result_strs[2]); // cache_capacity
result_strs[2]); // cache_capacity
ASSERT_EQ("100.0000", result_strs[3]); // miss_ratio
ASSERT_EQ("50", result_strs[4]); // number of accesses.
ASSERT_EQ("50", result_strs[4]); // number of accesses.
config_index++;
}
ASSERT_EQ(expected_capacities.size(), config_index);
......@@ -423,7 +425,7 @@ TEST_F(BlockCacheTracerTest, MixedBlocks) {
}
}
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
......
......@@ -1083,12 +1083,13 @@ DEFINE_uint64(stats_dump_period_sec,
TERARKDB_NAMESPACE::Options().stats_dump_period_sec,
"Gap between printing stats to log in seconds");
DEFINE_uint64(stats_persist_period_sec,
rocksdb::Options().stats_persist_period_sec,
TERARKDB_NAMESPACE::Options().stats_persist_period_sec,
"Gap between persisting stats in seconds");
DEFINE_bool(persist_stats_to_disk, rocksdb::Options().persist_stats_to_disk,
DEFINE_bool(persist_stats_to_disk,
TERARKDB_NAMESPACE::Options().persist_stats_to_disk,
"whether to persist stats to disk");
DEFINE_uint64(stats_history_buffer_size,
rocksdb::Options().stats_history_buffer_size,
TERARKDB_NAMESPACE::Options().stats_history_buffer_size,
"Max number of stats snapshots to keep in memory");
enum RepFactory {
......
......@@ -7,11 +7,12 @@
#include "db/db_impl/db_impl.h"
#include "rocksdb/slice.h"
#include "rocksdb/terark_namespace.h"
#include "util/coding.h"
#include "util/hash.h"
#include "util/string_util.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
namespace {
const unsigned int kCharSize = 1;
......@@ -266,4 +267,4 @@ Status BlockCacheTracer::WriteBlockAccess(const BlockCacheTraceRecord& record,
referenced_key);
}
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -10,10 +10,11 @@
#include "monitoring/instrumented_mutex.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/terark_namespace.h"
#include "rocksdb/trace_reader_writer.h"
#include "util/trace_replay.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
extern const uint64_t kMicrosInSecond;
......@@ -37,8 +38,8 @@ extern const uint64_t kMicrosInSecond;
// 6. BlockBasedTable::ApproximateOffsetOf. (kCompaction or
// kUserApproximateSize).
struct BlockCacheLookupContext {
BlockCacheLookupContext(const TableReaderCaller& _caller) : caller(_caller) {}
const TableReaderCaller caller;
BlockCacheLookupContext(const TableReaderCaller& _caller) : caller(_caller) {}
const TableReaderCaller caller;
// These are populated when we perform lookup/insert on block cache. The block
// cache tracer uses these inforation when logging the block access at
// BlockBasedTable::GET and BlockBasedTable::MultiGet.
......@@ -213,4 +214,4 @@ class BlockCacheTracer {
std::atomic<BlockCacheTraceWriter*> writer_;
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
......@@ -5,9 +5,10 @@
#include "util/mock_time_env.h"
#include "rocksdb/terark_namespace.h"
#include "util/sync_point.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// TODO: this is a workaround for the different behavior on different platform
// for timedwait timeout. Ideally timedwait API should be moved to env.
......@@ -35,4 +36,4 @@ void MockTimeEnv::InstallTimedWaitFixCallback() {
#endif // !NDEBUG
}
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
......@@ -15,11 +15,12 @@
#include "monitoring/instrumented_mutex.h"
#include "rocksdb/env.h"
#include "util/sync_point.h"
#include "rocksdb/terark_namespace.h"
#include "util/mutexlock.h"
#include "util/sync_point.h"
#include "util/sync_point_impl.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// A Timer class to handle repeated work.
//
......@@ -51,10 +52,8 @@ class Timer {
// the function takes 1000us to run. If it starts at time [now]us, then it
// finishes at [now]+1000us, 2nd run starting time will be at [now]+3000us.
// repeat_every_us == 0 means do not repeat.
void Add(std::function<void()> fn,
const std::string& fn_name,
uint64_t start_after_us,
uint64_t repeat_every_us) {
void Add(std::function<void()> fn, const std::string& fn_name,
uint64_t start_after_us, uint64_t repeat_every_us) {
std::unique_ptr<FunctionInfo> fn_info(
new FunctionInfo(std::move(fn), fn_name,
env_->NowMicros() + start_after_us, repeat_every_us));
......@@ -182,7 +181,6 @@ class Timer {
#endif // NDEBUG
private:
void Run() {
InstrumentedMutexLock l(&mutex_);
......@@ -223,8 +221,8 @@ class Timer {
// current_fn may be cancelled already.
if (current_fn->IsValid() && current_fn->repeat_every_us > 0) {
assert(running_);
current_fn->next_run_time_us = env_->NowMicros() +
current_fn->repeat_every_us;
current_fn->next_run_time_us =
env_->NowMicros() + current_fn->repeat_every_us;
// Schedule new work into the heap with new time.
heap_.push(current_fn);
......@@ -282,9 +280,7 @@ class Timer {
repeat_every_us(_repeat_every_us),
valid(true) {}
void Cancel() {
valid = false;
}
void Cancel() { valid = false; }
bool IsValid() const { return valid; }
};
......@@ -298,8 +294,7 @@ class Timer {
}
struct RunTimeOrder {
bool operator()(const FunctionInfo* f1,
const FunctionInfo* f2) {
bool operator()(const FunctionInfo* f1, const FunctionInfo* f2) {
return f1->next_run_time_us > f2->next_run_time_us;
}
};
......@@ -313,13 +308,12 @@ class Timer {
bool running_;
bool executing_task_;
std::priority_queue<FunctionInfo*,
std::vector<FunctionInfo*>,
RunTimeOrder> heap_;
std::priority_queue<FunctionInfo*, std::vector<FunctionInfo*>, RunTimeOrder>
heap_;
// In addition to providing a mapping from a function name to a function,
// it is also responsible for memory management.
std::unordered_map<std::string, std::unique_ptr<FunctionInfo>> map_;
};
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
......@@ -6,8 +6,9 @@
#include "util/timer.h"
#include "db/db_test_util.h"
#include "rocksdb/terark_namespace.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
class TimerTest : public testing::Test {
public:
......@@ -342,7 +343,7 @@ TEST_F(TimerTest, RepeatIntervalWithFuncRunningTime) {
ASSERT_TRUE(timer.Shutdown());
}
} // namespace ROCKSDB_NAMESPACE
} // namespace TERARKDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
......
......@@ -5,10 +5,11 @@
#pragma once
#include "rocksdb/terark_namespace.h"
#include "rocksdb/utilities/sim_cache.h"
#include "trace_replay/block_cache_tracer.h"
namespace rocksdb {
namespace TERARKDB_NAMESPACE {
// A cache configuration provided by user.
struct CacheConfiguration {
......@@ -93,4 +94,4 @@ class BlockCacheTraceSimulator {
uint64_t trace_start_time_ = 0;
};
} // namespace rocksdb
} // namespace TERARKDB_NAMESPACE
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册