db_test.cc 358.6 KB
Newer Older
1 2 3 4 5
//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
J
jorlow@chromium.org 已提交
6 7 8 9
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

S
sdong 已提交
10 11
// Introduction of SyncPoint effectively disabled building and running this test
// in Release build.
D
Dmitri Smirnov 已提交
12
// which is a pity, it is a good test
13
#include <algorithm>
14
#include <iostream>
15
#include <set>
16
#include <thread>
17
#include <unordered_set>
S
Stanislau Hlebik 已提交
18
#include <utility>
K
krad 已提交
19
#include <fcntl.h>
S
sdong 已提交
20 21 22
#ifndef OS_WIN
#include <unistd.h>
#endif
23

24
#include "db/filename.h"
25
#include "db/dbformat.h"
J
jorlow@chromium.org 已提交
26
#include "db/db_impl.h"
27
#include "db/db_test_util.h"
I
Igor Canadi 已提交
28
#include "db/job_context.h"
J
jorlow@chromium.org 已提交
29 30
#include "db/version_set.h"
#include "db/write_batch_internal.h"
31
#include "memtable/hash_linklist_rep.h"
I
Igor Canadi 已提交
32
#include "port/stack_trace.h"
33 34
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
A
agiardullo 已提交
35
#include "rocksdb/convenience.h"
K
Kai Liu 已提交
36
#include "rocksdb/db.h"
I
Islam AbdelRahman 已提交
37
#include "rocksdb/delete_scheduler.h"
38
#include "rocksdb/env.h"
39
#include "rocksdb/experimental.h"
K
Kai Liu 已提交
40
#include "rocksdb/filter_policy.h"
A
agiardullo 已提交
41
#include "rocksdb/options.h"
42
#include "rocksdb/perf_context.h"
43 44
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
A
agiardullo 已提交
45
#include "rocksdb/snapshot.h"
46
#include "rocksdb/sst_file_writer.h"
S
Siying Dong 已提交
47
#include "rocksdb/table.h"
48
#include "rocksdb/table_properties.h"
Y
Yueh-Hsuan Chiang 已提交
49
#include "rocksdb/thread_status.h"
50
#include "rocksdb/wal_filter.h"
51
#include "rocksdb/utilities/write_batch_with_index.h"
52
#include "rocksdb/utilities/checkpoint.h"
A
agiardullo 已提交
53
#include "rocksdb/utilities/optimistic_transaction_db.h"
K
kailiu 已提交
54
#include "table/block_based_table_factory.h"
55
#include "table/mock_table.h"
56
#include "table/plain_table_factory.h"
S
sdong 已提交
57
#include "table/scoped_arena_iterator.h"
58
#include "util/file_reader_writer.h"
S
Sanjay Ghemawat 已提交
59
#include "util/hash.h"
60
#include "utilities/merge_operators.h"
J
jorlow@chromium.org 已提交
61
#include "util/logging.h"
I
Igor Canadi 已提交
62
#include "util/compression.h"
63
#include "util/mutexlock.h"
L
Lei Jin 已提交
64
#include "util/rate_limiter.h"
K
Kai Liu 已提交
65
#include "util/statistics.h"
J
jorlow@chromium.org 已提交
66
#include "util/testharness.h"
67
#include "util/sync_point.h"
J
jorlow@chromium.org 已提交
68
#include "util/testutil.h"
69
#include "util/mock_env.h"
70
#include "util/string_util.h"
71
#include "util/thread_status_util.h"
72
#include "util/xfunc.h"
J
jorlow@chromium.org 已提交
73

74
namespace rocksdb {
J
jorlow@chromium.org 已提交
75

I
Igor Canadi 已提交
76 77 78 79
static long TestGetTickerCount(const Options& options, Tickers ticker_type) {
  return options.statistics->getTickerCount(ticker_type);
}

I
Islam AbdelRahman 已提交
80
#ifndef ROCKSDB_LITE
81 82
// A helper function that ensures the table properties returned in
// `GetPropertiesOfAllTablesTest` is correct.
C
clark.kang 已提交
83
// This test assumes entries size is different for each of the tables.
I
Igor Canadi 已提交
84
namespace {
85

I
Igor Canadi 已提交
86 87 88 89 90 91 92 93 94 95
uint64_t GetNumberOfSstFilesForColumnFamily(DB* db,
                                            std::string column_family_name) {
  std::vector<LiveFileMetaData> metadata;
  db->GetLiveFilesMetaData(&metadata);
  uint64_t result = 0;
  for (auto& fileMetadata : metadata) {
    result += (fileMetadata.column_family_name == column_family_name);
  }
  return result;
}
96

I
Igor Canadi 已提交
97
}  // namespace
I
Islam AbdelRahman 已提交
98
#endif  // ROCKSDB_LITE
99

100 101 102 103 104
class DBTest : public DBTestBase {
 public:
  DBTest() : DBTestBase("/db_test") {}
};

105 106 107
class DBTestWithParam : public DBTest,
                        public testing::WithParamInterface<uint32_t> {
 public:
108
  DBTestWithParam() { max_subcompactions_ = GetParam(); }
109 110 111 112 113

  // Required if inheriting from testing::WithParamInterface<>
  static void SetUpTestCase() {}
  static void TearDownTestCase() {}

114
  uint32_t max_subcompactions_;
115 116
};

I
Islam AbdelRahman 已提交
117
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
118
TEST_F(DBTest, Empty) {
S
Sanjay Ghemawat 已提交
119
  do {
120
    Options options;
K
kailiu 已提交
121 122
    options.env = env_;
    options.write_buffer_size = 100000;  // Small write buffer
123
    options = CurrentOptions(options);
L
Lei Jin 已提交
124
    CreateAndReopenWithCF({"pikachu"}, options);
J
jorlow@chromium.org 已提交
125

126 127 128 129 130
    std::string num;
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ("0", num);

131 132
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_EQ("v1", Get(1, "foo"));
133 134 135
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ("1", num);
K
kailiu 已提交
136

I
Igor Canadi 已提交
137 138
    // Block sync calls
    env_->delay_sstable_sync_.store(true, std::memory_order_release);
139
    Put(1, "k1", std::string(100000, 'x'));         // Fill memtable
140 141 142 143
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ("2", num);

144
    Put(1, "k2", std::string(100000, 'y'));         // Trigger compaction
145 146 147 148
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ("1", num);

149
    ASSERT_EQ("v1", Get(1, "foo"));
I
Igor Canadi 已提交
150 151
    // Release sync calls
    env_->delay_sstable_sync_.store(false, std::memory_order_release);
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176

    ASSERT_OK(db_->DisableFileDeletions());
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("1", num);

    ASSERT_OK(db_->DisableFileDeletions());
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("2", num);

    ASSERT_OK(db_->DisableFileDeletions());
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("3", num);

    ASSERT_OK(db_->EnableFileDeletions(false));
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("2", num);

    ASSERT_OK(db_->EnableFileDeletions());
    ASSERT_TRUE(
        dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
    ASSERT_EQ("0", num);
S
Sanjay Ghemawat 已提交
177
  } while (ChangeOptions());
J
jorlow@chromium.org 已提交
178
}
I
Islam AbdelRahman 已提交
179
#endif  // ROCKSDB_LITE
J
jorlow@chromium.org 已提交
180

I
Igor Sugak 已提交
181
TEST_F(DBTest, WriteEmptyBatch) {
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
  Options options;
  options.env = env_;
  options.write_buffer_size = 100000;
  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  ASSERT_OK(Put(1, "foo", "bar"));
  env_->sync_counter_.store(0);
  WriteOptions wo;
  wo.sync = true;
  wo.disableWAL = false;
  WriteBatch empty_batch;
  ASSERT_OK(dbfull()->Write(wo, &empty_batch));
  ASSERT_GE(env_->sync_counter_.load(), 1);

  // make sure we can re-open it.
  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
  ASSERT_EQ("bar", Get(1, "foo"));
}

I
Islam AbdelRahman 已提交
202
#ifndef ROCKSDB_LITE
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
TEST_F(DBTest, GetAggregatedIntPropertyTest) {
  const int kKeySize = 100;
  const int kValueSize = 500;
  const int kKeyNum = 100;

  Options options;
  options.env = env_;
  options.create_if_missing = true;
  options.write_buffer_size = (kKeySize + kValueSize) * kKeyNum / 10;
  // Make them never flush
  options.min_write_buffer_number_to_merge = 1000;
  options.max_write_buffer_number = 1000;
  options = CurrentOptions(options);
  CreateAndReopenWithCF({"one", "two", "three", "four"}, options);

  Random rnd(301);
  for (auto* handle : handles_) {
    for (int i = 0; i < kKeyNum; ++i) {
      db_->Put(WriteOptions(), handle, RandomString(&rnd, kKeySize),
               RandomString(&rnd, kValueSize));
    }
  }

  uint64_t manual_sum = 0;
  uint64_t api_sum = 0;
  uint64_t value = 0;
  for (auto* handle : handles_) {
    ASSERT_TRUE(
        db_->GetIntProperty(handle, DB::Properties::kSizeAllMemTables, &value));
    manual_sum += value;
  }
  ASSERT_TRUE(db_->GetAggregatedIntProperty(DB::Properties::kSizeAllMemTables,
                                            &api_sum));
  ASSERT_GT(manual_sum, 0);
  ASSERT_EQ(manual_sum, api_sum);

  ASSERT_FALSE(db_->GetAggregatedIntProperty(DB::Properties::kDBStats, &value));

  uint64_t before_flush_trm;
  uint64_t after_flush_trm;
  for (auto* handle : handles_) {
    ASSERT_TRUE(db_->GetAggregatedIntProperty(
        DB::Properties::kEstimateTableReadersMem, &before_flush_trm));

    // Issue flush and expect larger memory usage of table readers.
    db_->Flush(FlushOptions(), handle);

    ASSERT_TRUE(db_->GetAggregatedIntProperty(
        DB::Properties::kEstimateTableReadersMem, &after_flush_trm));
    ASSERT_GT(after_flush_trm, before_flush_trm);
  }
}

I
Igor Sugak 已提交
256
TEST_F(DBTest, ReadOnlyDB) {
257 258 259 260 261
  ASSERT_OK(Put("foo", "v1"));
  ASSERT_OK(Put("bar", "v2"));
  ASSERT_OK(Put("foo", "v3"));
  Close();

262 263 264
  auto options = CurrentOptions();
  assert(options.env = env_);
  ASSERT_OK(ReadOnlyReopen(options));
265 266 267 268 269 270 271 272 273 274
  ASSERT_EQ("v3", Get("foo"));
  ASSERT_EQ("v2", Get("bar"));
  Iterator* iter = db_->NewIterator(ReadOptions());
  int count = 0;
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    ASSERT_OK(iter->status());
    ++count;
  }
  ASSERT_EQ(count, 2);
  delete iter;
275 276 277
  Close();

  // Reopen and flush memtable.
L
Lei Jin 已提交
278
  Reopen(options);
279 280 281
  Flush();
  Close();
  // Now check keys in read only mode.
282
  ASSERT_OK(ReadOnlyReopen(options));
283 284
  ASSERT_EQ("v3", Get("foo"));
  ASSERT_EQ("v2", Get("bar"));
285
  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
286 287
}

I
Igor Sugak 已提交
288
TEST_F(DBTest, CompactedDB) {
L
Lei Jin 已提交
289 290 291 292 293 294 295
  const uint64_t kFileSize = 1 << 20;
  Options options;
  options.disable_auto_compactions = true;
  options.write_buffer_size = kFileSize;
  options.target_file_size_base = kFileSize;
  options.max_bytes_for_level_base = 1 << 30;
  options.compression = kNoCompression;
296
  options = CurrentOptions(options);
L
Lei Jin 已提交
297
  Reopen(options);
L
Lei Jin 已提交
298 299 300 301
  // 1 L0 file, use CompactedDB if max_open_files = -1
  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
  Flush();
  Close();
302
  ASSERT_OK(ReadOnlyReopen(options));
L
Lei Jin 已提交
303 304 305 306 307 308
  Status s = Put("new", "value");
  ASSERT_EQ(s.ToString(),
            "Not implemented: Not supported operation in read only mode.");
  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
  Close();
  options.max_open_files = -1;
309
  ASSERT_OK(ReadOnlyReopen(options));
L
Lei Jin 已提交
310 311 312 313 314
  s = Put("new", "value");
  ASSERT_EQ(s.ToString(),
            "Not implemented: Not supported in compacted db mode.");
  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
  Close();
L
Lei Jin 已提交
315
  Reopen(options);
L
Lei Jin 已提交
316 317 318 319 320 321
  // Add more L0 files
  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2')));
  Flush();
  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a')));
  Flush();
  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b')));
322
  ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e')));
L
Lei Jin 已提交
323 324 325
  Flush();
  Close();

326
  ASSERT_OK(ReadOnlyReopen(options));
L
Lei Jin 已提交
327 328 329 330 331 332 333
  // Fallback to read-only DB
  s = Put("new", "value");
  ASSERT_EQ(s.ToString(),
            "Not implemented: Not supported operation in read only mode.");
  Close();

  // Full compaction
L
Lei Jin 已提交
334
  Reopen(options);
L
Lei Jin 已提交
335 336 337 338 339
  // Add more keys
  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
  ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h')));
  ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i')));
  ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j')));
340
  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
341 342 343 344
  ASSERT_EQ(3, NumTableFilesAtLevel(1));
  Close();

  // CompactedDB
345
  ASSERT_OK(ReadOnlyReopen(options));
L
Lei Jin 已提交
346 347 348 349 350 351 352 353 354 355 356 357 358 359
  s = Put("new", "value");
  ASSERT_EQ(s.ToString(),
            "Not implemented: Not supported in compacted db mode.");
  ASSERT_EQ("NOT_FOUND", Get("abc"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb"));
  ASSERT_EQ("NOT_FOUND", Get("ccc"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff"));
  ASSERT_EQ("NOT_FOUND", Get("ggg"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii"));
  ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj"));
  ASSERT_EQ("NOT_FOUND", Get("kkk"));
360 361 362 363 364 365 366

  // MultiGet
  std::vector<std::string> values;
  std::vector<Status> status_list = dbfull()->MultiGet(ReadOptions(),
      std::vector<Slice>({Slice("aaa"), Slice("ccc"), Slice("eee"),
                          Slice("ggg"), Slice("iii"), Slice("kkk")}),
      &values);
367 368
  ASSERT_EQ(status_list.size(), static_cast<uint64_t>(6));
  ASSERT_EQ(values.size(), static_cast<uint64_t>(6));
369 370 371 372 373 374 375 376 377
  ASSERT_OK(status_list[0]);
  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]);
  ASSERT_TRUE(status_list[1].IsNotFound());
  ASSERT_OK(status_list[2]);
  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]);
  ASSERT_TRUE(status_list[3].IsNotFound());
  ASSERT_OK(status_list[4]);
  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]);
  ASSERT_TRUE(status_list[5].IsNotFound());
L
Lei Jin 已提交
378 379
}

K
Kai Liu 已提交
380 381
// Make sure that when options.block_cache is set, after a new table is
// created its index/filter blocks are added to block cache.
I
Igor Sugak 已提交
382
TEST_F(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
K
Kai Liu 已提交
383 384 385
  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
386 387
  BlockBasedTableOptions table_options;
  table_options.cache_index_and_filter_blocks = true;
388
  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
389
  options.table_factory.reset(new BlockBasedTableFactory(table_options));
L
Lei Jin 已提交
390
  CreateAndReopenWithCF({"pikachu"}, options);
K
Kai Liu 已提交
391

392 393 394
  ASSERT_OK(Put(1, "key", "val"));
  // Create a new table.
  ASSERT_OK(Flush(1));
K
Kai Liu 已提交
395 396

  // index/filter blocks added to block cache right after table creation.
I
Igor Canadi 已提交
397 398
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
K
Kai Liu 已提交
399
  ASSERT_EQ(2, /* only index/filter were added */
I
Igor Canadi 已提交
400 401
            TestGetTickerCount(options, BLOCK_CACHE_ADD));
  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
402 403 404 405
  uint64_t int_num;
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
  ASSERT_EQ(int_num, 0U);
K
Kai Liu 已提交
406 407 408 409

  // Make sure filter block is in cache.
  std::string value;
  ReadOptions ropt;
410
  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
K
Kai Liu 已提交
411 412

  // Miss count should remain the same.
I
Igor Canadi 已提交
413 414
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
K
Kai Liu 已提交
415

416
  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
I
Igor Canadi 已提交
417 418
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
K
Kai Liu 已提交
419 420

  // Make sure index block is in cache.
I
Igor Canadi 已提交
421
  auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
422
  value = Get(1, "key");
I
Igor Canadi 已提交
423
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
K
Kai Liu 已提交
424
  ASSERT_EQ(index_block_hit + 1,
I
Igor Canadi 已提交
425
            TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
K
Kai Liu 已提交
426

427
  value = Get(1, "key");
I
Igor Canadi 已提交
428
  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
K
Kai Liu 已提交
429
  ASSERT_EQ(index_block_hit + 2,
I
Igor Canadi 已提交
430
            TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
K
Kai Liu 已提交
431 432
}

433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
TEST_F(DBTest, ParanoidFileChecks) {
  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
  options.level0_file_num_compaction_trigger = 2;
  options.paranoid_file_checks = true;
  BlockBasedTableOptions table_options;
  table_options.cache_index_and_filter_blocks = false;
  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
  options.table_factory.reset(new BlockBasedTableFactory(table_options));
  CreateAndReopenWithCF({"pikachu"}, options);

  ASSERT_OK(Put(1, "1_key", "val"));
  ASSERT_OK(Put(1, "9_key", "val"));
  // Create a new table.
  ASSERT_OK(Flush(1));
  ASSERT_EQ(1, /* read and cache data block */
            TestGetTickerCount(options, BLOCK_CACHE_ADD));

  ASSERT_OK(Put(1, "1_key2", "val2"));
  ASSERT_OK(Put(1, "9_key2", "val2"));
  // Create a new SST file. This will further trigger a compaction
  // and generate another file.
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(3, /* Totally 3 files created up to now */
            TestGetTickerCount(options, BLOCK_CACHE_ADD));

  // After disabling options.paranoid_file_checks. NO further block
  // is added after generating a new file.
  ASSERT_OK(
      dbfull()->SetOptions(handles_[1], {{"paranoid_file_checks", "false"}}));

  ASSERT_OK(Put(1, "1_key3", "val3"));
  ASSERT_OK(Put(1, "9_key3", "val3"));
  ASSERT_OK(Flush(1));
  ASSERT_OK(Put(1, "1_key4", "val4"));
  ASSERT_OK(Put(1, "9_key4", "val4"));
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(3, /* Totally 3 files created up to now */
            TestGetTickerCount(options, BLOCK_CACHE_ADD));
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
namespace {
void ResetTableProperties(TableProperties* tp) {
  tp->data_size = 0;
  tp->index_size = 0;
  tp->filter_size = 0;
  tp->raw_key_size = 0;
  tp->raw_value_size = 0;
  tp->num_data_blocks = 0;
  tp->num_entries = 0;
}

void ParseTablePropertiesString(std::string tp_string, TableProperties* tp) {
  double dummy_double;
  std::replace(tp_string.begin(), tp_string.end(), ';', ' ');
  std::replace(tp_string.begin(), tp_string.end(), '=', ' ');
  ResetTableProperties(tp);

  sscanf(tp_string.c_str(), "# data blocks %" SCNu64
                            " # entries %" SCNu64
                            " raw key size %" SCNu64
                            " raw average key size %lf "
                            " raw value size %" SCNu64
                            " raw average value size %lf "
                            " data block size %" SCNu64
                            " index block size %" SCNu64
                            " filter block size %" SCNu64,
         &tp->num_data_blocks, &tp->num_entries, &tp->raw_key_size,
         &dummy_double, &tp->raw_value_size, &dummy_double, &tp->data_size,
         &tp->index_size, &tp->filter_size);
}

void VerifySimilar(uint64_t a, uint64_t b, double bias) {
  ASSERT_EQ(a == 0U, b == 0U);
  if (a == 0) {
    return;
  }
  double dbl_a = static_cast<double>(a);
  double dbl_b = static_cast<double>(b);
  if (dbl_a > dbl_b) {
    ASSERT_LT(static_cast<double>(dbl_a - dbl_b) / (dbl_a + dbl_b), bias);
  } else {
    ASSERT_LT(static_cast<double>(dbl_b - dbl_a) / (dbl_a + dbl_b), bias);
  }
}

void VerifyTableProperties(const TableProperties& base_tp,
                           const TableProperties& new_tp,
                           double filter_size_bias = 0.1,
                           double index_size_bias = 0.1,
                           double data_size_bias = 0.1,
                           double num_data_blocks_bias = 0.05) {
  VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias);
  VerifySimilar(base_tp.index_size, new_tp.index_size, index_size_bias);
  VerifySimilar(base_tp.filter_size, new_tp.filter_size, filter_size_bias);
  VerifySimilar(base_tp.num_data_blocks, new_tp.num_data_blocks,
                num_data_blocks_bias);
  ASSERT_EQ(base_tp.raw_key_size, new_tp.raw_key_size);
  ASSERT_EQ(base_tp.raw_value_size, new_tp.raw_value_size);
  ASSERT_EQ(base_tp.num_entries, new_tp.num_entries);
}

void GetExpectedTableProperties(TableProperties* expected_tp,
                                const int kKeySize, const int kValueSize,
                                const int kKeysPerTable, const int kTableCount,
                                const int kBloomBitsPerKey,
                                const size_t kBlockSize) {
  const int kKeyCount = kTableCount * kKeysPerTable;
  const int kAvgSuccessorSize = kKeySize / 2;
  const int kEncodingSavePerKey = kKeySize / 4;
  expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
  expected_tp->raw_value_size = kKeyCount * kValueSize;
  expected_tp->num_entries = kKeyCount;
  expected_tp->num_data_blocks =
      kTableCount *
      (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
      kBlockSize;
  expected_tp->data_size =
      kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
  expected_tp->index_size =
      expected_tp->num_data_blocks * (kAvgSuccessorSize + 12);
  expected_tp->filter_size =
      kTableCount * (kKeysPerTable * kBloomBitsPerKey / 8);
}
}  // namespace

TEST_F(DBTest, AggregatedTableProperties) {
  for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) {
    const int kKeysPerTable = 100;
    const int kKeySize = 80;
    const int kValueSize = 200;
    const int kBloomBitsPerKey = 20;

    Options options = CurrentOptions();
    options.level0_file_num_compaction_trigger = 8;
    options.compression = kNoCompression;
    options.create_if_missing = true;

    BlockBasedTableOptions table_options;
    table_options.filter_policy.reset(
        NewBloomFilterPolicy(kBloomBitsPerKey, false));
    table_options.block_size = 1024;
    options.table_factory.reset(new BlockBasedTableFactory(table_options));

    DestroyAndReopen(options);

    Random rnd(5632);
    for (int table = 1; table <= kTableCount; ++table) {
      for (int i = 0; i < kKeysPerTable; ++i) {
        db_->Put(WriteOptions(), RandomString(&rnd, kKeySize),
                 RandomString(&rnd, kValueSize));
      }
      db_->Flush(FlushOptions());
    }
    std::string property;
    db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property);

    TableProperties expected_tp;
    GetExpectedTableProperties(&expected_tp, kKeySize, kValueSize,
                               kKeysPerTable, kTableCount, kBloomBitsPerKey,
                               table_options.block_size);

    TableProperties output_tp;
    ParseTablePropertiesString(property, &output_tp);

    VerifyTableProperties(expected_tp, output_tp);
  }
}

605 606 607
TEST_F(DBTest, ReadLatencyHistogramByLevel) {
  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;
608
  options.level0_file_num_compaction_trigger = 6;
609 610
  options.num_levels = 4;
  options.compression = kNoCompression;
611
  options.max_bytes_for_level_base = 4500 << 10;
612 613 614 615 616 617 618 619 620
  options.target_file_size_base = 98 << 10;
  options.max_write_buffer_number = 2;
  options.statistics = rocksdb::CreateDBStatistics();
  options.max_open_files = 100;

  BlockBasedTableOptions table_options;
  table_options.no_block_cache = true;

  DestroyAndReopen(options);
621
  int key_index = 0;
622
  Random rnd(301);
623
  for (int num = 0; num < 8; num++) {
624
    Put("foo", "bar");
625
    GenerateNewFile(&rnd, &key_index);
626
    dbfull()->TEST_WaitForCompact();
627
  }
628
  dbfull()->TEST_WaitForCompact();
629 630 631 632 633

  std::string prop;
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));

  // Get() after flushes, See latency histogram tracked.
634
  for (int key = 0; key < key_index; key++) {
635 636 637 638 639 640 641 642 643
    Get(Key(key));
  }
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));

  // Reopen and issue Get(). See thee latency tracked
  Reopen(options);
644
  dbfull()->TEST_WaitForCompact();
645
  for (int key = 0; key < key_index; key++) {
646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
    Get(Key(key));
  }
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));

  // Reopen and issue iterating. See thee latency tracked
  Reopen(options);
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
  {
    unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
    for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
    }
  }
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));

  // options.max_open_files preloads table readers.
  options.max_open_files = -1;
  Reopen(options);
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
676
  for (int key = 0; key < key_index; key++) {
677 678 679 680 681 682 683 684
    Get(Key(key));
  }
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
  ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
}

685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
TEST_F(DBTest, AggregatedTablePropertiesAtLevel) {
  const int kTableCount = 100;
  const int kKeysPerTable = 10;
  const int kKeySize = 50;
  const int kValueSize = 400;
  const int kMaxLevel = 7;
  const int kBloomBitsPerKey = 20;
  Random rnd(301);
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 8;
  options.compression = kNoCompression;
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.target_file_size_base = 8192;
  options.max_bytes_for_level_base = 10000;
  options.max_bytes_for_level_multiplier = 2;
  // This ensures there no compaction happening when we call GetProperty().
  options.disable_auto_compactions = true;

  BlockBasedTableOptions table_options;
  table_options.filter_policy.reset(
      NewBloomFilterPolicy(kBloomBitsPerKey, false));
  table_options.block_size = 1024;
  options.table_factory.reset(new BlockBasedTableFactory(table_options));

  DestroyAndReopen(options);

  std::string level_tp_strings[kMaxLevel];
  std::string tp_string;
  TableProperties level_tps[kMaxLevel];
  TableProperties tp, sum_tp, expected_tp;
  for (int table = 1; table <= kTableCount; ++table) {
    for (int i = 0; i < kKeysPerTable; ++i) {
      db_->Put(WriteOptions(), RandomString(&rnd, kKeySize),
               RandomString(&rnd, kValueSize));
    }
    db_->Flush(FlushOptions());
    db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
    ResetTableProperties(&sum_tp);
    for (int level = 0; level < kMaxLevel; ++level) {
      db_->GetProperty(
          DB::Properties::kAggregatedTablePropertiesAtLevel + ToString(level),
          &level_tp_strings[level]);
      ParseTablePropertiesString(level_tp_strings[level], &level_tps[level]);
      sum_tp.data_size += level_tps[level].data_size;
      sum_tp.index_size += level_tps[level].index_size;
      sum_tp.filter_size += level_tps[level].filter_size;
      sum_tp.raw_key_size += level_tps[level].raw_key_size;
      sum_tp.raw_value_size += level_tps[level].raw_value_size;
      sum_tp.num_data_blocks += level_tps[level].num_data_blocks;
      sum_tp.num_entries += level_tps[level].num_entries;
    }
    db_->GetProperty(DB::Properties::kAggregatedTableProperties, &tp_string);
    ParseTablePropertiesString(tp_string, &tp);
    ASSERT_EQ(sum_tp.data_size, tp.data_size);
    ASSERT_EQ(sum_tp.index_size, tp.index_size);
    ASSERT_EQ(sum_tp.filter_size, tp.filter_size);
    ASSERT_EQ(sum_tp.raw_key_size, tp.raw_key_size);
    ASSERT_EQ(sum_tp.raw_value_size, tp.raw_value_size);
    ASSERT_EQ(sum_tp.num_data_blocks, tp.num_data_blocks);
    ASSERT_EQ(sum_tp.num_entries, tp.num_entries);
    if (table > 3) {
      GetExpectedTableProperties(&expected_tp, kKeySize, kValueSize,
                                 kKeysPerTable, table, kBloomBitsPerKey,
                                 table_options.block_size);
      // Gives larger bias here as index block size, filter block size,
      // and data block size become much harder to estimate in this test.
      VerifyTableProperties(tp, expected_tp, 0.5, 0.4, 0.4, 0.25);
    }
  }
}
I
Islam AbdelRahman 已提交
756
#endif  // ROCKSDB_LITE
757

758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
class CoutingUserTblPropCollector : public TablePropertiesCollector {
 public:
  const char* Name() const override { return "CoutingUserTblPropCollector"; }

  Status Finish(UserCollectedProperties* properties) override {
    std::string encoded;
    PutVarint32(&encoded, count_);
    *properties = UserCollectedProperties{
        {"CoutingUserTblPropCollector", message_}, {"Count", encoded},
    };
    return Status::OK();
  }

  Status AddUserKey(const Slice& user_key, const Slice& value, EntryType type,
                    SequenceNumber seq, uint64_t file_size) override {
    ++count_;
    return Status::OK();
  }

  virtual UserCollectedProperties GetReadableProperties() const override {
    return UserCollectedProperties{};
  }

 private:
  std::string message_ = "Rocksdb";
  uint32_t count_ = 0;
};

class CoutingUserTblPropCollectorFactory
    : public TablePropertiesCollectorFactory {
 public:
789 790 791 792 793 794 795 796
  explicit CoutingUserTblPropCollectorFactory(
      uint32_t expected_column_family_id)
      : expected_column_family_id_(expected_column_family_id),
        num_created_(0) {}
  virtual TablePropertiesCollector* CreateTablePropertiesCollector(
      TablePropertiesCollectorFactory::Context context) override {
    EXPECT_EQ(expected_column_family_id_, context.column_family_id);
    num_created_++;
797 798 799 800 801
    return new CoutingUserTblPropCollector();
  }
  const char* Name() const override {
    return "CoutingUserTblPropCollectorFactory";
  }
802 803 804 805 806
  void set_expected_column_family_id(uint32_t v) {
    expected_column_family_id_ = v;
  }
  uint32_t expected_column_family_id_;
  uint32_t num_created_;
807 808
};

I
Islam AbdelRahman 已提交
809
#ifndef ROCKSDB_LITE
810
TEST_F(DBTest, GetUserDefinedTableProperties) {
811
  Options options = CurrentOptions();
812
  options.level0_file_num_compaction_trigger = (1<<30);
813 814
  options.max_background_flushes = 0;
  options.table_properties_collector_factories.resize(1);
815 816 817
  std::shared_ptr<CoutingUserTblPropCollectorFactory> collector_factory =
      std::make_shared<CoutingUserTblPropCollectorFactory>(0);
  options.table_properties_collector_factories[0] = collector_factory;
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
  Reopen(options);
  // Create 4 tables
  for (int table = 0; table < 4; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      db_->Put(WriteOptions(), ToString(table * 100 + i), "val");
    }
    db_->Flush(FlushOptions());
  }

  TablePropertiesCollection props;
  ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
  ASSERT_EQ(4U, props.size());
  uint32_t sum = 0;
  for (const auto& item : props) {
    auto& user_collected = item.second->user_collected_properties;
    ASSERT_TRUE(user_collected.find("CoutingUserTblPropCollector") !=
                user_collected.end());
    ASSERT_EQ(user_collected.at("CoutingUserTblPropCollector"), "Rocksdb");
    ASSERT_TRUE(user_collected.find("Count") != user_collected.end());
    Slice key(user_collected.at("Count"));
    uint32_t count;
    ASSERT_TRUE(GetVarint32(&key, &count));
    sum += count;
  }
  ASSERT_EQ(10u + 11u + 12u + 13u, sum);
843

844
  ASSERT_GT(collector_factory->num_created_, 0U);
845 846
  collector_factory->num_created_ = 0;
  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
847
  ASSERT_GT(collector_factory->num_created_, 0U);
848
}
I
Islam AbdelRahman 已提交
849
#endif  // ROCKSDB_LITE
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866

TEST_F(DBTest, UserDefinedTablePropertiesContext) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_flushes = 0;
  options.table_properties_collector_factories.resize(1);
  std::shared_ptr<CoutingUserTblPropCollectorFactory> collector_factory =
      std::make_shared<CoutingUserTblPropCollectorFactory>(1);
  options.table_properties_collector_factories[0] = collector_factory,
  CreateAndReopenWithCF({"pikachu"}, options);
  // Create 2 files
  for (int table = 0; table < 2; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      Put(1, ToString(table * 100 + i), "val");
    }
    Flush(1);
  }
867
  ASSERT_GT(collector_factory->num_created_, 0U);
868 869 870 871 872 873 874 875 876 877

  collector_factory->num_created_ = 0;
  // Trigger automatic compactions.
  for (int table = 0; table < 3; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      Put(1, ToString(table * 100 + i), "val");
    }
    Flush(1);
    dbfull()->TEST_WaitForCompact();
  }
878
  ASSERT_GT(collector_factory->num_created_, 0U);
879 880 881

  collector_factory->num_created_ = 0;
  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
882
  ASSERT_GT(collector_factory->num_created_, 0U);
883 884 885 886 887 888 889 890 891 892 893

  // Come back to write to default column family
  collector_factory->num_created_ = 0;
  collector_factory->set_expected_column_family_id(0);  // default CF
  // Create 4 tables in default column family
  for (int table = 0; table < 2; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      Put(ToString(table * 100 + i), "val");
    }
    Flush();
  }
894
  ASSERT_GT(collector_factory->num_created_, 0U);
895 896 897 898 899 900 901 902 903 904

  collector_factory->num_created_ = 0;
  // Trigger automatic compactions.
  for (int table = 0; table < 3; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      Put(ToString(table * 100 + i), "val");
    }
    Flush();
    dbfull()->TEST_WaitForCompact();
  }
905
  ASSERT_GT(collector_factory->num_created_, 0U);
906 907 908

  collector_factory->num_created_ = 0;
  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
909
  ASSERT_GT(collector_factory->num_created_, 0U);
910 911
}

I
Islam AbdelRahman 已提交
912
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
913
TEST_F(DBTest, LevelLimitReopen) {
914
  Options options = CurrentOptions();
L
Lei Jin 已提交
915
  CreateAndReopenWithCF({"pikachu"}, options);
916 917 918

  const std::string value(1024 * 1024, ' ');
  int i = 0;
919 920
  while (NumTableFilesAtLevel(2, 1) == 0) {
    ASSERT_OK(Put(1, Key(i++), value));
921 922 923
  }

  options.num_levels = 1;
924
  options.max_bytes_for_level_multiplier_additional.resize(1, 1);
L
Lei Jin 已提交
925
  Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
926
  ASSERT_EQ(s.IsInvalidArgument(), true);
927
  ASSERT_EQ(s.ToString(),
928
            "Invalid argument: db has more levels than options.num_levels");
929 930

  options.num_levels = 10;
931
  options.max_bytes_for_level_multiplier_additional.resize(10, 1);
L
Lei Jin 已提交
932
  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
933
}
I
Islam AbdelRahman 已提交
934
#endif  // ROCKSDB_LITE
935

I
Igor Sugak 已提交
936
TEST_F(DBTest, PutDeleteGet) {
S
Sanjay Ghemawat 已提交
937
  do {
L
Lei Jin 已提交
938
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
939 940 941 942 943 944
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_OK(Put(1, "foo", "v2"));
    ASSERT_EQ("v2", Get(1, "foo"));
    ASSERT_OK(Delete(1, "foo"));
    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
S
Sanjay Ghemawat 已提交
945
  } while (ChangeOptions());
J
jorlow@chromium.org 已提交
946 947
}

A
Andres Noetzli 已提交
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
TEST_F(DBTest, PutSingleDeleteGet) {
  do {
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_OK(Put(1, "foo2", "v2"));
    ASSERT_EQ("v2", Get(1, "foo2"));
    ASSERT_OK(SingleDelete(1, "foo"));
    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
    // Skip HashCuckooRep as it does not support single delete. FIFO and
    // universal compaction do not apply to the test case. Skip MergePut
    // because single delete does not get removed when it encounters a merge.
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
                         kSkipUniversalCompaction | kSkipMergePut));
}

TEST_F(DBTest, SingleDeleteFlush) {
  // Test to check whether flushing preserves a single delete hidden
  // behind a put.
  do {
    Random rnd(301);

    Options options = CurrentOptions();
    options.disable_auto_compactions = true;
    CreateAndReopenWithCF({"pikachu"}, options);

    // Put values on second level (so that they will not be in the same
    // compaction as the other operations.
    Put(1, "foo", "first");
    Put(1, "bar", "one");
    ASSERT_OK(Flush(1));
    MoveFilesToLevel(2, 1);

    // (Single) delete hidden by a put
    SingleDelete(1, "foo");
    Put(1, "foo", "second");
    Delete(1, "bar");
    Put(1, "bar", "two");
    ASSERT_OK(Flush(1));

    SingleDelete(1, "foo");
    Delete(1, "bar");
    ASSERT_OK(Flush(1));

    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);

    ASSERT_EQ("NOT_FOUND", Get(1, "bar"));
    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
    // Skip HashCuckooRep as it does not support single delete. FIFO and
    // universal compaction do not apply to the test case. Skip MergePut
    // because merges cannot be combined with single deletions.
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
                         kSkipUniversalCompaction | kSkipMergePut));
}

TEST_F(DBTest, SingleDeletePutFlush) {
  // Single deletes that encounter the matching put in a flush should get
  // removed.
  do {
    Random rnd(301);

    Options options = CurrentOptions();
    options.disable_auto_compactions = true;
    CreateAndReopenWithCF({"pikachu"}, options);

    Put(1, "foo", Slice());
    Put(1, "a", Slice());
    SingleDelete(1, "a");
    ASSERT_OK(Flush(1));

    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
    // Skip HashCuckooRep as it does not support single delete. FIFO and
    // universal compaction do not apply to the test case. Skip MergePut
    // because merges cannot be combined with single deletions.
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
                         kSkipUniversalCompaction | kSkipMergePut));
}

TEST_F(DBTest, EmptyFlush) {
  // It is possible to produce empty flushes when using single deletes. Tests
  // whether empty flushes cause issues.
  do {
    Random rnd(301);

    Options options = CurrentOptions();
    options.disable_auto_compactions = true;
    CreateAndReopenWithCF({"pikachu"}, options);

    Put(1, "a", Slice());
    SingleDelete(1, "a");
    ASSERT_OK(Flush(1));

    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
    // Skip HashCuckooRep as it does not support single delete. FIFO and
    // universal compaction do not apply to the test case. Skip MergePut
    // because merges cannot be combined with single deletions.
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
                         kSkipUniversalCompaction | kSkipMergePut));
}

I
Igor Sugak 已提交
1049
TEST_F(DBTest, GetFromImmutableLayer) {
S
Sanjay Ghemawat 已提交
1050
  do {
1051
    Options options;
S
Sanjay Ghemawat 已提交
1052 1053
    options.env = env_;
    options.write_buffer_size = 100000;  // Small write buffer
1054
    options = CurrentOptions(options);
L
Lei Jin 已提交
1055
    CreateAndReopenWithCF({"pikachu"}, options);
1056

1057 1058
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_EQ("v1", Get(1, "foo"));
1059

I
Igor Canadi 已提交
1060 1061
    // Block sync calls
    env_->delay_sstable_sync_.store(true, std::memory_order_release);
1062 1063 1064 1065
    Put(1, "k1", std::string(100000, 'x'));          // Fill memtable
    Put(1, "k2", std::string(100000, 'y'));          // Trigger flush
    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
I
Igor Canadi 已提交
1066 1067
    // Release sync calls
    env_->delay_sstable_sync_.store(false, std::memory_order_release);
S
Sanjay Ghemawat 已提交
1068
  } while (ChangeOptions());
1069 1070
}

I
Igor Sugak 已提交
1071
TEST_F(DBTest, GetFromVersions) {
S
Sanjay Ghemawat 已提交
1072
  do {
L
Lei Jin 已提交
1073
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1074 1075 1076 1077
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_OK(Flush(1));
    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
S
Sanjay Ghemawat 已提交
1078
  } while (ChangeOptions());
1079 1080
}

I
Islam AbdelRahman 已提交
1081
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
1082
TEST_F(DBTest, GetSnapshot) {
1083 1084
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
S
Sanjay Ghemawat 已提交
1085
  do {
1086
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
S
Sanjay Ghemawat 已提交
1087 1088 1089
    // Try with both a short key and a long key
    for (int i = 0; i < 2; i++) {
      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
1090
      ASSERT_OK(Put(1, key, "v1"));
S
Sanjay Ghemawat 已提交
1091
      const Snapshot* s1 = db_->GetSnapshot();
S
sdong 已提交
1092
      if (option_config_ == kHashCuckoo) {
1093
        // Unsupported case.
S
sdong 已提交
1094 1095 1096
        ASSERT_TRUE(s1 == nullptr);
        break;
      }
1097 1098 1099 1100 1101 1102
      ASSERT_OK(Put(1, key, "v2"));
      ASSERT_EQ("v2", Get(1, key));
      ASSERT_EQ("v1", Get(1, key, s1));
      ASSERT_OK(Flush(1));
      ASSERT_EQ("v2", Get(1, key));
      ASSERT_EQ("v1", Get(1, key, s1));
S
Sanjay Ghemawat 已提交
1103 1104
      db_->ReleaseSnapshot(s1);
    }
S
sdong 已提交
1105
  } while (ChangeOptions());
1106
}
I
Islam AbdelRahman 已提交
1107
#endif  // ROCKSDB_LITE
1108

I
Igor Sugak 已提交
1109
TEST_F(DBTest, GetLevel0Ordering) {
S
Sanjay Ghemawat 已提交
1110
  do {
L
Lei Jin 已提交
1111
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
S
Sanjay Ghemawat 已提交
1112 1113 1114 1115
    // Check that we process level-0 files in correct order.  The code
    // below generates two level-0 files where the earlier one comes
    // before the later one in the level-0 file list since the earlier
    // one has a smaller "smallest" key.
1116 1117 1118 1119 1120 1121
    ASSERT_OK(Put(1, "bar", "b"));
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_OK(Flush(1));
    ASSERT_OK(Put(1, "foo", "v2"));
    ASSERT_OK(Flush(1));
    ASSERT_EQ("v2", Get(1, "foo"));
S
Sanjay Ghemawat 已提交
1122
  } while (ChangeOptions());
1123 1124
}

I
Igor Sugak 已提交
1125
TEST_F(DBTest, WrongLevel0Config) {
1126 1127 1128 1129 1130 1131 1132 1133 1134
  Options options = CurrentOptions();
  Close();
  ASSERT_OK(DestroyDB(dbname_, options));
  options.level0_stop_writes_trigger = 1;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_file_num_compaction_trigger = 3;
  ASSERT_OK(DB::Open(options, dbname_, &db_));
}

I
Islam AbdelRahman 已提交
1135
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
1136
TEST_F(DBTest, GetOrderedByLevels) {
S
Sanjay Ghemawat 已提交
1137
  do {
L
Lei Jin 已提交
1138
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1139 1140 1141 1142 1143 1144 1145
    ASSERT_OK(Put(1, "foo", "v1"));
    Compact(1, "a", "z");
    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_OK(Put(1, "foo", "v2"));
    ASSERT_EQ("v2", Get(1, "foo"));
    ASSERT_OK(Flush(1));
    ASSERT_EQ("v2", Get(1, "foo"));
S
Sanjay Ghemawat 已提交
1146
  } while (ChangeOptions());
1147 1148
}

I
Igor Sugak 已提交
1149
TEST_F(DBTest, GetPicksCorrectFile) {
S
Sanjay Ghemawat 已提交
1150
  do {
L
Lei Jin 已提交
1151
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
S
Sanjay Ghemawat 已提交
1152
    // Arrange to have multiple files in a non-level-0 level.
1153 1154 1155 1156 1157 1158 1159 1160 1161
    ASSERT_OK(Put(1, "a", "va"));
    Compact(1, "a", "b");
    ASSERT_OK(Put(1, "x", "vx"));
    Compact(1, "x", "y");
    ASSERT_OK(Put(1, "f", "vf"));
    Compact(1, "f", "g");
    ASSERT_EQ("va", Get(1, "a"));
    ASSERT_EQ("vf", Get(1, "f"));
    ASSERT_EQ("vx", Get(1, "x"));
S
Sanjay Ghemawat 已提交
1162
  } while (ChangeOptions());
1163 1164
}

I
Igor Sugak 已提交
1165
TEST_F(DBTest, GetEncountersEmptyLevel) {
S
Sanjay Ghemawat 已提交
1166
  do {
I
Igor Canadi 已提交
1167 1168
    Options options = CurrentOptions();
    options.disableDataSync = true;
L
Lei Jin 已提交
1169
    CreateAndReopenWithCF({"pikachu"}, options);
S
Sanjay Ghemawat 已提交
1170 1171 1172 1173 1174 1175
    // Arrange for the following to happen:
    //   * sstable A in level 0
    //   * nothing in level 1
    //   * sstable B in level 2
    // Then do enough Get() calls to arrange for an automatic compaction
    // of sstable A.  A bug would cause the compaction to be marked as
C
clark.kang 已提交
1176
    // occurring at level 1 (instead of the correct level 0).
S
Sanjay Ghemawat 已提交
1177 1178

    // Step 1: First place sstables in levels 0 and 2
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
    Put(1, "a", "begin");
    Put(1, "z", "end");
    ASSERT_OK(Flush(1));
    dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
    dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
    Put(1, "a", "begin");
    Put(1, "z", "end");
    ASSERT_OK(Flush(1));
    ASSERT_GT(NumTableFilesAtLevel(0, 1), 0);
    ASSERT_GT(NumTableFilesAtLevel(2, 1), 0);
1189

S
Sanjay Ghemawat 已提交
1190
    // Step 2: clear level 1 if necessary.
1191 1192 1193 1194
    dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
    ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
    ASSERT_EQ(NumTableFilesAtLevel(2, 1), 1);
S
Sanjay Ghemawat 已提交
1195

H
heyongqiang 已提交
1196 1197
    // Step 3: read a bunch of times
    for (int i = 0; i < 1000; i++) {
1198
      ASSERT_EQ("NOT_FOUND", Get(1, "missing"));
S
Sanjay Ghemawat 已提交
1199
    }
H
heyongqiang 已提交
1200 1201

    // Step 4: Wait for compaction to finish
1202
    dbfull()->TEST_WaitForCompact();
H
heyongqiang 已提交
1203

1204
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);  // XXX
I
Igor Canadi 已提交
1205
  } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction));
1206
}
I
Islam AbdelRahman 已提交
1207
#endif  // ROCKSDB_LITE
1208

1209 1210 1211
// KeyMayExist can lead to a few false positives, but not false negatives.
// To make test deterministic, use a much larger number of bits per key-20 than
// bits in the key, so that false positives are eliminated
I
Igor Sugak 已提交
1212
TEST_F(DBTest, KeyMayExist) {
1213
  do {
1214 1215
    ReadOptions ropts;
    std::string value;
1216 1217 1218
    anon::OptionsOverride options_override;
    options_override.filter_policy.reset(NewBloomFilterPolicy(20));
    Options options = CurrentOptions(options_override);
1219
    options.statistics = rocksdb::CreateDBStatistics();
L
Lei Jin 已提交
1220
    CreateAndReopenWithCF({"pikachu"}, options);
1221

1222
    ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
1223

1224
    ASSERT_OK(Put(1, "a", "b"));
1225
    bool value_found = false;
1226 1227
    ASSERT_TRUE(
        db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found));
1228 1229
    ASSERT_TRUE(value_found);
    ASSERT_EQ("b", value);
1230

1231
    ASSERT_OK(Flush(1));
1232
    value.clear();
1233

I
Igor Canadi 已提交
1234 1235
    long numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    long cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1236 1237
    ASSERT_TRUE(
        db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found));
M
Mayank Agarwal 已提交
1238
    ASSERT_TRUE(!value_found);
1239 1240
    // assert that no new files were opened and no new blocks were
    // read into block cache.
I
Igor Canadi 已提交
1241 1242
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1243

1244
    ASSERT_OK(Delete(1, "a"));
1245

I
Igor Canadi 已提交
1246 1247
    numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1248
    ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
I
Igor Canadi 已提交
1249 1250
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1251

1252
    ASSERT_OK(Flush(1));
1253 1254
    dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1],
                                true /* disallow trivial move */);
1255

I
Igor Canadi 已提交
1256 1257
    numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1258
    ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
I
Igor Canadi 已提交
1259 1260
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1261

1262
    ASSERT_OK(Delete(1, "c"));
1263

I
Igor Canadi 已提交
1264 1265
    numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1266
    ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "c", &value));
I
Igor Canadi 已提交
1267 1268
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1269

1270 1271
    // KeyMayExist function only checks data in block caches, which is not used
    // by plain table format.
I
Igor Canadi 已提交
1272 1273
  } while (
      ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
1274 1275
}

I
Igor Sugak 已提交
1276
TEST_F(DBTest, NonBlockingIteration) {
1277 1278 1279
  do {
    ReadOptions non_blocking_opts, regular_opts;
    Options options = CurrentOptions();
1280
    options.statistics = rocksdb::CreateDBStatistics();
1281
    non_blocking_opts.read_tier = kBlockCacheTier;
L
Lei Jin 已提交
1282
    CreateAndReopenWithCF({"pikachu"}, options);
1283
    // write one kv to the database.
1284
    ASSERT_OK(Put(1, "a", "b"));
1285 1286 1287

    // scan using non-blocking iterator. We should find it because
    // it is in memtable.
1288
    Iterator* iter = db_->NewIterator(non_blocking_opts, handles_[1]);
1289 1290
    int count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
N
Naman Gupta 已提交
1291
      ASSERT_OK(iter->status());
1292 1293 1294 1295 1296 1297 1298
      count++;
    }
    ASSERT_EQ(count, 1);
    delete iter;

    // flush memtable to storage. Now, the key should not be in the
    // memtable neither in the block cache.
1299
    ASSERT_OK(Flush(1));
1300 1301 1302

    // verify that a non-blocking iterator does not find any
    // kvs. Neither does it do any IOs to storage.
I
Igor Canadi 已提交
1303 1304
    long numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    long cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1305
    iter = db_->NewIterator(non_blocking_opts, handles_[1]);
1306 1307 1308 1309 1310 1311
    count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      count++;
    }
    ASSERT_EQ(count, 0);
    ASSERT_TRUE(iter->status().IsIncomplete());
I
Igor Canadi 已提交
1312 1313
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1314 1315 1316
    delete iter;

    // read in the specified block via a regular get
1317
    ASSERT_EQ(Get(1, "a"), "b");
1318 1319

    // verify that we can find it via a non-blocking scan
I
Igor Canadi 已提交
1320 1321
    numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
1322
    iter = db_->NewIterator(non_blocking_opts, handles_[1]);
1323 1324
    count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
N
Naman Gupta 已提交
1325
      ASSERT_OK(iter->status());
1326 1327 1328
      count++;
    }
    ASSERT_EQ(count, 1);
I
Igor Canadi 已提交
1329 1330
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1331 1332
    delete iter;

1333 1334
    // This test verifies block cache behaviors, which is not used by plain
    // table format.
1335
    // Exclude kHashCuckoo as it does not support iteration currently
I
Igor Canadi 已提交
1336 1337
  } while (ChangeOptions(kSkipPlainTable | kSkipNoSeekToLast | kSkipHashCuckoo |
                         kSkipMmapReads));
1338 1339
}

I
Islam AbdelRahman 已提交
1340
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
1341
TEST_F(DBTest, ManagedNonBlockingIteration) {
V
Venkatesh Radhakrishnan 已提交
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
  do {
    ReadOptions non_blocking_opts, regular_opts;
    Options options = CurrentOptions();
    options.statistics = rocksdb::CreateDBStatistics();
    non_blocking_opts.read_tier = kBlockCacheTier;
    non_blocking_opts.managed = true;
    CreateAndReopenWithCF({"pikachu"}, options);
    // write one kv to the database.
    ASSERT_OK(Put(1, "a", "b"));

    // scan using non-blocking iterator. We should find it because
    // it is in memtable.
    Iterator* iter = db_->NewIterator(non_blocking_opts, handles_[1]);
    int count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      ASSERT_OK(iter->status());
      count++;
    }
    ASSERT_EQ(count, 1);
    delete iter;

    // flush memtable to storage. Now, the key should not be in the
    // memtable neither in the block cache.
    ASSERT_OK(Flush(1));

    // verify that a non-blocking iterator does not find any
    // kvs. Neither does it do any IOs to storage.
    int64_t numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    int64_t cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
    iter = db_->NewIterator(non_blocking_opts, handles_[1]);
    count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      count++;
    }
    ASSERT_EQ(count, 0);
    ASSERT_TRUE(iter->status().IsIncomplete());
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
    delete iter;

    // read in the specified block via a regular get
    ASSERT_EQ(Get(1, "a"), "b");

    // verify that we can find it via a non-blocking scan
    numopen = TestGetTickerCount(options, NO_FILE_OPENS);
    cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
    iter = db_->NewIterator(non_blocking_opts, handles_[1]);
    count = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      ASSERT_OK(iter->status());
      count++;
    }
    ASSERT_EQ(count, 1);
    ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
    ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
    delete iter;

    // This test verifies block cache behaviors, which is not used by plain
    // table format.
    // Exclude kHashCuckoo as it does not support iteration currently
  } while (ChangeOptions(kSkipPlainTable | kSkipNoSeekToLast | kSkipHashCuckoo |
                         kSkipMmapReads));
}
I
Islam AbdelRahman 已提交
1405
#endif  // ROCKSDB_LITE
V
Venkatesh Radhakrishnan 已提交
1406

1407 1408
// A delete is skipped for key if KeyMayExist(key) returns False
// Tests Writebatch consistency and proper delete behaviour
I
Igor Sugak 已提交
1409
TEST_F(DBTest, FilterDeletes) {
1410
  do {
1411 1412 1413
    anon::OptionsOverride options_override;
    options_override.filter_policy.reset(NewBloomFilterPolicy(20));
    Options options = CurrentOptions(options_override);
1414
    options.filter_deletes = true;
L
Lei Jin 已提交
1415
    CreateAndReopenWithCF({"pikachu"}, options);
1416 1417
    WriteBatch batch;

I
Igor Canadi 已提交
1418
    batch.Delete(handles_[1], "a");
1419
    dbfull()->Write(WriteOptions(), &batch);
1420
    ASSERT_EQ(AllEntriesFor("a", 1), "[ ]");  // Delete skipped
1421 1422
    batch.Clear();

I
Igor Canadi 已提交
1423 1424
    batch.Put(handles_[1], "a", "b");
    batch.Delete(handles_[1], "a");
1425
    dbfull()->Write(WriteOptions(), &batch);
1426 1427
    ASSERT_EQ(Get(1, "a"), "NOT_FOUND");
    ASSERT_EQ(AllEntriesFor("a", 1), "[ DEL, b ]");  // Delete issued
1428 1429
    batch.Clear();

I
Igor Canadi 已提交
1430 1431
    batch.Delete(handles_[1], "c");
    batch.Put(handles_[1], "c", "d");
1432
    dbfull()->Write(WriteOptions(), &batch);
1433 1434
    ASSERT_EQ(Get(1, "c"), "d");
    ASSERT_EQ(AllEntriesFor("c", 1), "[ d ]");  // Delete skipped
1435 1436
    batch.Clear();

1437
    ASSERT_OK(Flush(1));  // A stray Flush
1438

I
Igor Canadi 已提交
1439
    batch.Delete(handles_[1], "c");
1440
    dbfull()->Write(WriteOptions(), &batch);
1441
    ASSERT_EQ(AllEntriesFor("c", 1), "[ DEL, d ]");  // Delete issued
1442 1443
    batch.Clear();
  } while (ChangeCompactOptions());
1444 1445
}

I
Igor Sugak 已提交
1446
TEST_F(DBTest, GetFilterByPrefixBloom) {
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
  Options options = last_options_;
  options.prefix_extractor.reset(NewFixedPrefixTransform(8));
  options.statistics = rocksdb::CreateDBStatistics();
  BlockBasedTableOptions bbto;
  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
  bbto.whole_key_filtering = false;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  DestroyAndReopen(options);

  WriteOptions wo;
  ReadOptions ro;
  FlushOptions fo;
  fo.wait = true;
  std::string value;

  ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
  ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
  ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));

  dbfull()->Flush(fo);

  ASSERT_EQ("foo", Get("barbarbar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
  ASSERT_EQ("foo2", Get("barbarbar2"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
  ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);

  ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

  ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
}
I
Igor Canadi 已提交
1481

I
Igor Sugak 已提交
1482
TEST_F(DBTest, WholeKeyFilterProp) {
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
  Options options = last_options_;
  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
  options.statistics = rocksdb::CreateDBStatistics();

  BlockBasedTableOptions bbto;
  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
  bbto.whole_key_filtering = false;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  DestroyAndReopen(options);

  WriteOptions wo;
  ReadOptions ro;
  FlushOptions fo;
  fo.wait = true;
  std::string value;

  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
  // Needs insert some keys to make sure files are not filtered out by key
  // ranges.
  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
  dbfull()->Flush(fo);

  Reopen(options);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

  // Reopen with whole key filtering enabled and prefix extractor
  // NULL. Bloom filter should be off for both of whole key and
  // prefix bloom.
  bbto.whole_key_filtering = true;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  options.prefix_extractor.reset();
  Reopen(options);

  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  // Write DB with only full key filtering.
  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
  // Needs insert some keys to make sure files are not filtered out by key
  // ranges.
  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
1535
  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557

  // Reopen with both of whole key off and prefix extractor enabled.
  // Still no bloom filter should be used.
  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
  bbto.whole_key_filtering = false;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  Reopen(options);

  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);

  // Try to create a DB with mixed files:
  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
  // Needs insert some keys to make sure files are not filtered out by key
  // ranges.
  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
1558
  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630

  options.prefix_extractor.reset();
  bbto.whole_key_filtering = true;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  Reopen(options);

  // Try to create a DB with mixed files.
  ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
  // In this case needs insert some keys to make sure files are
  // not filtered out by key ranges.
  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
  Flush();

  // Now we have two files:
  // File 1: An older file with prefix bloom.
  // File 2: A newer file with whole bloom filter.
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
  ASSERT_EQ("bar", Get("barfoo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);

  // Reopen with the same setting: only whole key is used
  Reopen(options);
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
  ASSERT_EQ("bar", Get("barfoo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);

  // Restart with both filters are allowed
  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
  bbto.whole_key_filtering = true;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  Reopen(options);
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
  // File 1 will has it filtered out.
  // File 2 will not, as prefix `foo` exists in the file.
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
  ASSERT_EQ("bar", Get("barfoo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);

  // Restart with only prefix bloom is allowed.
  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
  bbto.whole_key_filtering = false;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  Reopen(options);
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
  ASSERT_EQ("NOT_FOUND", Get("foo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
  ASSERT_EQ("NOT_FOUND", Get("bar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
  ASSERT_EQ("foo", Get("foobar"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
  ASSERT_EQ("bar", Get("barfoo"));
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
}

I
Igor Sugak 已提交
1631
TEST_F(DBTest, IterSeekBeforePrev) {
I
Igor Canadi 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
  ASSERT_OK(Put("a", "b"));
  ASSERT_OK(Put("c", "d"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put("0", "f"));
  ASSERT_OK(Put("1", "h"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put("2", "j"));
  auto iter = db_->NewIterator(ReadOptions());
  iter->Seek(Slice("c"));
  iter->Prev();
  iter->Seek(Slice("a"));
  iter->Prev();
  delete iter;
}

I
Igor Canadi 已提交
1647
namespace {
1648 1649 1650
std::string MakeLongKey(size_t length, char c) {
  return std::string(length, c);
}
I
Igor Canadi 已提交
1651
}  // namespace
1652

I
Igor Sugak 已提交
1653
TEST_F(DBTest, IterLongKeys) {
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
  ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
  ASSERT_OK(Put(MakeLongKey(32, 2), "2"));
  ASSERT_OK(Put("a", "b"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put(MakeLongKey(50, 1), "1"));
  ASSERT_OK(Put(MakeLongKey(127, 3), "3"));
  ASSERT_OK(Put(MakeLongKey(64, 4), "4"));
  auto iter = db_->NewIterator(ReadOptions());

  // Create a key that needs to be skipped for Seq too new
  iter->Seek(MakeLongKey(20, 0));
  ASSERT_EQ(IterStatus(iter), MakeLongKey(20, 0) + "->0");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(64, 4) + "->4");
  delete iter;

  iter = db_->NewIterator(ReadOptions());
  iter->Seek(MakeLongKey(50, 1));
  ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3");
  delete iter;
}

I
Igor Sugak 已提交
1686
TEST_F(DBTest, IterNextWithNewerSeq) {
S
sdong 已提交
1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
  ASSERT_OK(Put("0", "0"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put("a", "b"));
  ASSERT_OK(Put("c", "d"));
  ASSERT_OK(Put("d", "e"));
  auto iter = db_->NewIterator(ReadOptions());

  // Create a key that needs to be skipped for Seq too new
  for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
       i++) {
    ASSERT_OK(Put("b", "f"));
  }

  iter->Seek(Slice("a"));
  ASSERT_EQ(IterStatus(iter), "a->b");
  iter->Next();
  ASSERT_EQ(IterStatus(iter), "c->d");
  delete iter;
}

I
Igor Sugak 已提交
1707
TEST_F(DBTest, IterPrevWithNewerSeq) {
S
sdong 已提交
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
  ASSERT_OK(Put("0", "0"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put("a", "b"));
  ASSERT_OK(Put("c", "d"));
  ASSERT_OK(Put("d", "e"));
  auto iter = db_->NewIterator(ReadOptions());

  // Create a key that needs to be skipped for Seq too new
  for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
       i++) {
    ASSERT_OK(Put("b", "f"));
  }

  iter->Seek(Slice("d"));
  ASSERT_EQ(IterStatus(iter), "d->e");
  iter->Prev();
  ASSERT_EQ(IterStatus(iter), "c->d");
  iter->Prev();
  ASSERT_EQ(IterStatus(iter), "a->b");

  iter->Prev();
  delete iter;
}

I
Igor Sugak 已提交
1732
TEST_F(DBTest, IterPrevWithNewerSeq2) {
S
sdong 已提交
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
  ASSERT_OK(Put("0", "0"));
  dbfull()->Flush(FlushOptions());
  ASSERT_OK(Put("a", "b"));
  ASSERT_OK(Put("c", "d"));
  ASSERT_OK(Put("d", "e"));
  auto iter = db_->NewIterator(ReadOptions());
  iter->Seek(Slice("c"));
  ASSERT_EQ(IterStatus(iter), "c->d");

  // Create a key that needs to be skipped for Seq too new
  for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
      i++) {
    ASSERT_OK(Put("b", "f"));
  }

  iter->Prev();
  ASSERT_EQ(IterStatus(iter), "a->b");

  iter->Prev();
  delete iter;
}

I
Igor Sugak 已提交
1755
TEST_F(DBTest, IterEmpty) {
1756
  do {
L
Lei Jin 已提交
1757
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1758
    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
J
jorlow@chromium.org 已提交
1759

1760 1761
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
1762

1763 1764
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
1765

1766 1767
    iter->Seek("foo");
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
1768

1769 1770
    delete iter;
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
1771 1772
}

I
Igor Sugak 已提交
1773
TEST_F(DBTest, IterSingle) {
1774
  do {
L
Lei Jin 已提交
1775
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1776 1777
    ASSERT_OK(Put(1, "a", "va"));
    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
J
jorlow@chromium.org 已提交
1778

1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    iter->Seek("");
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
1801

1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
    iter->Seek("a");
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    iter->Seek("b");
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    delete iter;
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
1812 1813
}

I
Igor Sugak 已提交
1814
TEST_F(DBTest, IterMulti) {
1815
  do {
L
Lei Jin 已提交
1816
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1817 1818 1819 1820
    ASSERT_OK(Put(1, "a", "va"));
    ASSERT_OK(Put(1, "b", "vb"));
    ASSERT_OK(Put(1, "c", "vc"));
    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
J
jorlow@chromium.org 已提交
1821

1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "b->vb");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
1834

1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "b->vb");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    iter->Seek("");
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Seek("a");
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Seek("ax");
    ASSERT_EQ(IterStatus(iter), "b->vb");
1854

1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
    iter->Seek("b");
    ASSERT_EQ(IterStatus(iter), "b->vb");
    iter->Seek("z");
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    // Switch from reverse to forward
    iter->SeekToLast();
    iter->Prev();
    iter->Prev();
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "b->vb");

    // Switch from forward to reverse
    iter->SeekToFirst();
    iter->Next();
    iter->Next();
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "b->vb");

    // Make sure iter stays at snapshot
1875 1876 1877 1878 1879
    ASSERT_OK(Put(1, "a", "va2"));
    ASSERT_OK(Put(1, "a2", "va3"));
    ASSERT_OK(Put(1, "b", "vb2"));
    ASSERT_OK(Put(1, "c", "vc2"));
    ASSERT_OK(Delete(1, "b"));
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "b->vb");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "b->vb");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    delete iter;
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
1899 1900
}

1901 1902
// Check that we can skip over a run of user keys
// by using reseek rather than sequential scan
I
Igor Sugak 已提交
1903
TEST_F(DBTest, IterReseek) {
1904 1905 1906
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
  Options options = CurrentOptions(options_override);
1907 1908
  options.max_sequential_skip_in_iterations = 3;
  options.create_if_missing = true;
1909
  options.statistics = rocksdb::CreateDBStatistics();
L
Lei Jin 已提交
1910
  DestroyAndReopen(options);
L
Lei Jin 已提交
1911
  CreateAndReopenWithCF({"pikachu"}, options);
1912

1913
  // insert three keys with same userkey and verify that
1914 1915
  // reseek is not invoked. For each of these test cases,
  // verify that we can find the next key "b".
1916
  ASSERT_OK(Put(1, "a", "zero"));
1917 1918 1919 1920
  ASSERT_OK(Put(1, "a", "one"));
  ASSERT_OK(Put(1, "a", "two"));
  ASSERT_OK(Put(1, "b", "bone"));
  Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
1921
  iter->SeekToFirst();
I
Igor Canadi 已提交
1922
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
1923 1924
  ASSERT_EQ(IterStatus(iter), "a->two");
  iter->Next();
I
Igor Canadi 已提交
1925
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
1926 1927 1928 1929 1930
  ASSERT_EQ(IterStatus(iter), "b->bone");
  delete iter;

  // insert a total of three keys with same userkey and verify
  // that reseek is still not invoked.
1931 1932
  ASSERT_OK(Put(1, "a", "three"));
  iter = db_->NewIterator(ReadOptions(), handles_[1]);
1933 1934 1935
  iter->SeekToFirst();
  ASSERT_EQ(IterStatus(iter), "a->three");
  iter->Next();
I
Igor Canadi 已提交
1936
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
1937 1938 1939 1940 1941
  ASSERT_EQ(IterStatus(iter), "b->bone");
  delete iter;

  // insert a total of four keys with same userkey and verify
  // that reseek is invoked.
1942 1943
  ASSERT_OK(Put(1, "a", "four"));
  iter = db_->NewIterator(ReadOptions(), handles_[1]);
1944 1945
  iter->SeekToFirst();
  ASSERT_EQ(IterStatus(iter), "a->four");
I
Igor Canadi 已提交
1946
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
1947
  iter->Next();
I
Igor Canadi 已提交
1948
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1);
1949 1950 1951 1952 1953 1954
  ASSERT_EQ(IterStatus(iter), "b->bone");
  delete iter;

  // Testing reverse iterator
  // At this point, we have three versions of "a" and one version of "b".
  // The reseek statistics is already at 1.
I
Igor Canadi 已提交
1955 1956
  int num_reseeks =
      (int)TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION);
1957 1958

  // Insert another version of b and assert that reseek is not invoked
1959 1960
  ASSERT_OK(Put(1, "b", "btwo"));
  iter = db_->NewIterator(ReadOptions(), handles_[1]);
1961 1962
  iter->SeekToLast();
  ASSERT_EQ(IterStatus(iter), "b->btwo");
I
Igor Canadi 已提交
1963 1964
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
            num_reseeks);
1965
  iter->Prev();
I
Igor Canadi 已提交
1966 1967
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
            num_reseeks + 1);
1968 1969 1970 1971 1972
  ASSERT_EQ(IterStatus(iter), "a->four");
  delete iter;

  // insert two more versions of b. This makes a total of 4 versions
  // of b and 4 versions of a.
1973 1974 1975
  ASSERT_OK(Put(1, "b", "bthree"));
  ASSERT_OK(Put(1, "b", "bfour"));
  iter = db_->NewIterator(ReadOptions(), handles_[1]);
1976 1977
  iter->SeekToLast();
  ASSERT_EQ(IterStatus(iter), "b->bfour");
I
Igor Canadi 已提交
1978 1979
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
            num_reseeks + 2);
1980 1981 1982
  iter->Prev();

  // the previous Prev call should have invoked reseek
I
Igor Canadi 已提交
1983 1984
  ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
            num_reseeks + 3);
1985 1986 1987 1988
  ASSERT_EQ(IterStatus(iter), "a->four");
  delete iter;
}

I
Igor Sugak 已提交
1989
TEST_F(DBTest, IterSmallAndLargeMix) {
1990
  do {
L
Lei Jin 已提交
1991
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
1992 1993 1994 1995 1996
    ASSERT_OK(Put(1, "a", "va"));
    ASSERT_OK(Put(1, "b", std::string(100000, 'b')));
    ASSERT_OK(Put(1, "c", "vc"));
    ASSERT_OK(Put(1, "d", std::string(100000, 'd')));
    ASSERT_OK(Put(1, "e", std::string(100000, 'e')));
J
jorlow@chromium.org 已提交
1997

1998
    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
J
jorlow@chromium.org 已提交
1999

2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
    iter->SeekToFirst();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
    iter->Next();
    ASSERT_EQ(IterStatus(iter), "(invalid)");
J
jorlow@chromium.org 已提交
2012

2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
    iter->SeekToLast();
    ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "c->vc");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "a->va");
    iter->Prev();
    ASSERT_EQ(IterStatus(iter), "(invalid)");

    delete iter;
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
2028 2029
}

I
Igor Sugak 已提交
2030
TEST_F(DBTest, IterMultiWithDelete) {
S
Sanjay Ghemawat 已提交
2031
  do {
L
Lei Jin 已提交
2032
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
2033 2034 2035 2036 2037
    ASSERT_OK(Put(1, "ka", "va"));
    ASSERT_OK(Put(1, "kb", "vb"));
    ASSERT_OK(Put(1, "kc", "vc"));
    ASSERT_OK(Delete(1, "kb"));
    ASSERT_EQ("NOT_FOUND", Get(1, "kb"));
2038 2039

    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
2040 2041
    iter->Seek("kc");
    ASSERT_EQ(IterStatus(iter), "kc->vc");
2042 2043
    if (!CurrentOptions().merge_operator) {
      // TODO: merge operator does not support backward iteration yet
2044 2045 2046 2047 2048 2049
      if (kPlainTableAllBytesPrefix != option_config_&&
          kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
          kHashLinkList != option_config_) {
        iter->Prev();
        ASSERT_EQ(IterStatus(iter), "ka->va");
      }
2050
    }
S
Sanjay Ghemawat 已提交
2051 2052
    delete iter;
  } while (ChangeOptions());
2053 2054
}

I
Igor Sugak 已提交
2055
TEST_F(DBTest, IterPrevMaxSkip) {
2056
  do {
L
Lei Jin 已提交
2057
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
2058
    for (int i = 0; i < 2; i++) {
2059 2060 2061 2062 2063
      ASSERT_OK(Put(1, "key1", "v1"));
      ASSERT_OK(Put(1, "key2", "v2"));
      ASSERT_OK(Put(1, "key3", "v3"));
      ASSERT_OK(Put(1, "key4", "v4"));
      ASSERT_OK(Put(1, "key5", "v5"));
2064 2065
    }

2066
    VerifyIterLast("key5->v5", 1);
2067

2068 2069
    ASSERT_OK(Delete(1, "key5"));
    VerifyIterLast("key4->v4", 1);
2070

2071 2072
    ASSERT_OK(Delete(1, "key4"));
    VerifyIterLast("key3->v3", 1);
2073

2074 2075
    ASSERT_OK(Delete(1, "key3"));
    VerifyIterLast("key2->v2", 1);
2076

2077 2078
    ASSERT_OK(Delete(1, "key2"));
    VerifyIterLast("key1->v1", 1);
2079

2080 2081
    ASSERT_OK(Delete(1, "key1"));
    VerifyIterLast("(invalid)", 1);
2082
  } while (ChangeOptions(kSkipMergePut | kSkipNoSeekToLast));
2083 2084
}

I
Igor Sugak 已提交
2085
TEST_F(DBTest, IterWithSnapshot) {
2086 2087
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
2088
  do {
2089
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
2090 2091 2092 2093 2094
    ASSERT_OK(Put(1, "key1", "val1"));
    ASSERT_OK(Put(1, "key2", "val2"));
    ASSERT_OK(Put(1, "key3", "val3"));
    ASSERT_OK(Put(1, "key4", "val4"));
    ASSERT_OK(Put(1, "key5", "val5"));
2095 2096 2097 2098

    const Snapshot *snapshot = db_->GetSnapshot();
    ReadOptions options;
    options.snapshot = snapshot;
2099
    Iterator* iter = db_->NewIterator(options, handles_[1]);
2100 2101

    // Put more values after the snapshot
2102 2103
    ASSERT_OK(Put(1, "key100", "val100"));
    ASSERT_OK(Put(1, "key101", "val101"));
2104 2105 2106 2107 2108

    iter->Seek("key5");
    ASSERT_EQ(IterStatus(iter), "key5->val5");
    if (!CurrentOptions().merge_operator) {
      // TODO: merge operator does not support backward iteration yet
2109 2110 2111 2112 2113 2114 2115
      if (kPlainTableAllBytesPrefix != option_config_&&
        kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
        kHashLinkList != option_config_) {
        iter->Prev();
        ASSERT_EQ(IterStatus(iter), "key4->val4");
        iter->Prev();
        ASSERT_EQ(IterStatus(iter), "key3->val3");
2116

2117 2118 2119 2120 2121
        iter->Next();
        ASSERT_EQ(IterStatus(iter), "key4->val4");
        iter->Next();
        ASSERT_EQ(IterStatus(iter), "key5->val5");
      }
2122 2123 2124 2125 2126
      iter->Next();
      ASSERT_TRUE(!iter->Valid());
    }
    db_->ReleaseSnapshot(snapshot);
    delete iter;
2127 2128
    // skip as HashCuckooRep does not support snapshot
  } while (ChangeOptions(kSkipHashCuckoo));
2129 2130
}

I
Igor Sugak 已提交
2131
TEST_F(DBTest, Recover) {
S
Sanjay Ghemawat 已提交
2132
  do {
L
Lei Jin 已提交
2133
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
2134 2135 2136
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_OK(Put(1, "baz", "v5"));

L
Lei Jin 已提交
2137
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2138 2139 2140 2141 2142 2143 2144
    ASSERT_EQ("v1", Get(1, "foo"));

    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_EQ("v5", Get(1, "baz"));
    ASSERT_OK(Put(1, "bar", "v2"));
    ASSERT_OK(Put(1, "foo", "v3"));

L
Lei Jin 已提交
2145
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2146 2147 2148 2149 2150
    ASSERT_EQ("v3", Get(1, "foo"));
    ASSERT_OK(Put(1, "foo", "v4"));
    ASSERT_EQ("v4", Get(1, "foo"));
    ASSERT_EQ("v2", Get(1, "bar"));
    ASSERT_EQ("v5", Get(1, "baz"));
S
Sanjay Ghemawat 已提交
2151
  } while (ChangeOptions());
J
jorlow@chromium.org 已提交
2152 2153
}

I
Igor Sugak 已提交
2154
TEST_F(DBTest, RecoverWithTableHandle) {
2155
  do {
2156
    Options options;
2157 2158 2159
    options.create_if_missing = true;
    options.write_buffer_size = 100;
    options.disable_auto_compactions = true;
2160
    options = CurrentOptions(options);
L
Lei Jin 已提交
2161
    DestroyAndReopen(options);
L
Lei Jin 已提交
2162
    CreateAndReopenWithCF({"pikachu"}, options);
2163

2164 2165 2166 2167 2168 2169 2170
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_OK(Put(1, "bar", "v2"));
    ASSERT_OK(Flush(1));
    ASSERT_OK(Put(1, "foo", "v3"));
    ASSERT_OK(Put(1, "bar", "v4"));
    ASSERT_OK(Flush(1));
    ASSERT_OK(Put(1, "big", std::string(100, 'a')));
L
Lei Jin 已提交
2171
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2172 2173

    std::vector<std::vector<FileMetaData>> files;
2174
    dbfull()->TEST_GetFilesMetaData(handles_[1], &files);
V
Vasili Svirski 已提交
2175
    size_t total_files = 0;
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
    for (const auto& level : files) {
      total_files += level.size();
    }
    ASSERT_EQ(total_files, 3);
    for (const auto& level : files) {
      for (const auto& file : level) {
        if (kInfiniteMaxOpenFiles == option_config_) {
          ASSERT_TRUE(file.table_reader_handle != nullptr);
        } else {
          ASSERT_TRUE(file.table_reader_handle == nullptr);
        }
      }
    }
  } while (ChangeOptions());
}

I
Igor Sugak 已提交
2192
TEST_F(DBTest, IgnoreRecoveredLog) {
I
Igor Canadi 已提交
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
  std::string backup_logs = dbname_ + "/backup_logs";

  // delete old files in backup_logs directory
  env_->CreateDirIfMissing(backup_logs);
  std::vector<std::string> old_files;
  env_->GetChildren(backup_logs, &old_files);
  for (auto& file : old_files) {
    if (file != "." && file != "..") {
      env_->DeleteFile(backup_logs + "/" + file);
    }
  }

  do {
    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.merge_operator = MergeOperators::CreateUInt64AddOperator();
2209 2210
    options.wal_dir = dbname_ + "/logs";
    DestroyAndReopen(options);
I
Igor Canadi 已提交
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229

    // fill up the DB
    std::string one, two;
    PutFixed64(&one, 1);
    PutFixed64(&two, 2);
    ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one)));
    ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one)));
    ASSERT_OK(db_->Merge(WriteOptions(), Slice("bar"), Slice(one)));

    // copy the logs to backup
    std::vector<std::string> logs;
    env_->GetChildren(options.wal_dir, &logs);
    for (auto& log : logs) {
      if (log != ".." && log != ".") {
        CopyFile(options.wal_dir + "/" + log, backup_logs + "/" + log);
      }
    }

    // recover the DB
L
Lei Jin 已提交
2230
    Reopen(options);
I
Igor Canadi 已提交
2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243
    ASSERT_EQ(two, Get("foo"));
    ASSERT_EQ(one, Get("bar"));
    Close();

    // copy the logs from backup back to wal dir
    for (auto& log : logs) {
      if (log != ".." && log != ".") {
        CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
      }
    }
    // this should ignore the log files, recovery should not happen again
    // if the recovery happens, the same merge operator would be called twice,
    // leading to incorrect results
L
Lei Jin 已提交
2244
    Reopen(options);
I
Igor Canadi 已提交
2245 2246 2247
    ASSERT_EQ(two, Get("foo"));
    ASSERT_EQ(one, Get("bar"));
    Close();
L
Lei Jin 已提交
2248
    Destroy(options);
L
Lei Jin 已提交
2249
    Reopen(options);
2250
    Close();
I
Igor Canadi 已提交
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260

    // copy the logs from backup back to wal dir
    env_->CreateDirIfMissing(options.wal_dir);
    for (auto& log : logs) {
      if (log != ".." && log != ".") {
        CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
      }
    }
    // assert that we successfully recovered only from logs, even though we
    // destroyed the DB
L
Lei Jin 已提交
2261
    Reopen(options);
I
Igor Canadi 已提交
2262 2263
    ASSERT_EQ(two, Get("foo"));
    ASSERT_EQ(one, Get("bar"));
2264 2265

    // Recovery will fail if DB directory doesn't exist.
L
Lei Jin 已提交
2266
    Destroy(options);
2267 2268 2269 2270 2271 2272 2273 2274 2275
    // copy the logs from backup back to wal dir
    env_->CreateDirIfMissing(options.wal_dir);
    for (auto& log : logs) {
      if (log != ".." && log != ".") {
        CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
        // we won't be needing this file no more
        env_->DeleteFile(backup_logs + "/" + log);
      }
    }
L
Lei Jin 已提交
2276
    Status s = TryReopen(options);
2277
    ASSERT_TRUE(!s.ok());
2278
  } while (ChangeOptions(kSkipHashCuckoo));
I
Igor Canadi 已提交
2279 2280
}

I
Igor Sugak 已提交
2281
TEST_F(DBTest, CheckLock) {
2282 2283 2284
  do {
    DB* localdb;
    Options options = CurrentOptions();
L
Lei Jin 已提交
2285
    ASSERT_OK(TryReopen(options));
2286 2287

    // second open should fail
2288
    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
2289
  } while (ChangeCompactOptions());
2290 2291
}

I
Igor Sugak 已提交
2292
TEST_F(DBTest, FlushMultipleMemtable) {
2293 2294 2295 2296 2297 2298
  do {
    Options options = CurrentOptions();
    WriteOptions writeOpt = WriteOptions();
    writeOpt.disableWAL = true;
    options.max_write_buffer_number = 4;
    options.min_write_buffer_number_to_merge = 3;
2299
    options.max_write_buffer_number_to_maintain = -1;
L
Lei Jin 已提交
2300
    CreateAndReopenWithCF({"pikachu"}, options);
2301 2302 2303 2304 2305 2306 2307
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
    ASSERT_OK(Flush(1));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));

    ASSERT_EQ("v1", Get(1, "foo"));
    ASSERT_EQ("v1", Get(1, "bar"));
    ASSERT_OK(Flush(1));
2308 2309 2310
  } while (ChangeCompactOptions());
}

I
Islam AbdelRahman 已提交
2311
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
2312
TEST_F(DBTest, NumImmutableMemTable) {
2313 2314 2315 2316 2317 2318
  do {
    Options options = CurrentOptions();
    WriteOptions writeOpt = WriteOptions();
    writeOpt.disableWAL = true;
    options.max_write_buffer_number = 4;
    options.min_write_buffer_number_to_merge = 3;
2319
    options.max_write_buffer_number_to_maintain = 0;
2320
    options.write_buffer_size = 1000000;
L
Lei Jin 已提交
2321
    CreateAndReopenWithCF({"pikachu"}, options);
2322

2323
    std::string big_value(1000000 * 2, 'x');
2324
    std::string num;
2325
    SetPerfLevel(kEnableTime);;
2326
    ASSERT_TRUE(GetPerfLevel() == kEnableTime);
2327

2328 2329 2330
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k1", big_value));
    ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
                                      "rocksdb.num-immutable-mem-table", &num));
2331
    ASSERT_EQ(num, "0");
2332 2333 2334
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ(num, "1");
2335
    perf_context.Reset();
2336
    Get(1, "k1");
2337
    ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
2338

2339 2340 2341
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
    ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
                                      "rocksdb.num-immutable-mem-table", &num));
2342
    ASSERT_EQ(num, "1");
2343 2344 2345 2346 2347 2348 2349
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ(num, "1");
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
    ASSERT_EQ(num, "1");

2350
    perf_context.Reset();
2351
    Get(1, "k1");
2352 2353
    ASSERT_EQ(2, (int) perf_context.get_from_memtable_count);
    perf_context.Reset();
2354
    Get(1, "k2");
2355
    ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
2356

2357
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
2358 2359
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.cur-size-active-mem-table", &num));
2360 2361
    ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
                                      "rocksdb.num-immutable-mem-table", &num));
2362
    ASSERT_EQ(num, "2");
2363 2364 2365 2366 2367 2368
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &num));
    ASSERT_EQ(num, "1");
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
    ASSERT_EQ(num, "2");
2369
    perf_context.Reset();
2370
    Get(1, "k2");
2371 2372
    ASSERT_EQ(2, (int) perf_context.get_from_memtable_count);
    perf_context.Reset();
2373
    Get(1, "k3");
2374 2375
    ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
    perf_context.Reset();
2376
    Get(1, "k1");
2377
    ASSERT_EQ(3, (int) perf_context.get_from_memtable_count);
2378

2379 2380 2381
    ASSERT_OK(Flush(1));
    ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
                                      "rocksdb.num-immutable-mem-table", &num));
2382
    ASSERT_EQ(num, "0");
2383 2384
    ASSERT_TRUE(dbfull()->GetProperty(
        handles_[1], "rocksdb.cur-size-active-mem-table", &num));
2385
    // "192" is the size of the metadata of an empty skiplist, this would
2386
    // break if we change the default skiplist implementation
2387
    ASSERT_EQ(num, "192");
2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416

    uint64_t int_num;
    uint64_t base_total_size;
    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.estimate-num-keys", &base_total_size));

    ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k2"));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", ""));
    ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k3"));
    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.num-deletes-active-mem-table", &int_num));
    ASSERT_EQ(int_num, 2U);
    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.num-entries-active-mem-table", &int_num));
    ASSERT_EQ(int_num, 3U);

    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.num-entries-imm-mem-tables", &int_num));
    ASSERT_EQ(int_num, 4U);
    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.num-deletes-imm-mem-tables", &int_num));
    ASSERT_EQ(int_num, 2U);

    ASSERT_TRUE(dbfull()->GetIntProperty(
        handles_[1], "rocksdb.estimate-num-keys", &int_num));
    ASSERT_EQ(int_num, base_total_size + 1);

2417
    SetPerfLevel(kDisable);
2418
    ASSERT_TRUE(GetPerfLevel() == kDisable);
2419 2420
  } while (ChangeCompactOptions());
}
I
Islam AbdelRahman 已提交
2421
#endif  // ROCKSDB_LITE
2422

I
Igor Sugak 已提交
2423
TEST_F(DBTest, FlushEmptyColumnFamily) {
2424 2425 2426
  // Block flush thread and disable compaction thread
  env_->SetBackgroundThreads(1, Env::HIGH);
  env_->SetBackgroundThreads(1, Env::LOW);
2427 2428
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
2429
                 Env::Priority::LOW);
2430 2431 2432
  test::SleepingBackgroundTask sleeping_task_high;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
                 &sleeping_task_high, Env::Priority::HIGH);
2433 2434 2435 2436 2437 2438 2439 2440

  Options options = CurrentOptions();
  // disable compaction
  options.disable_auto_compactions = true;
  WriteOptions writeOpt = WriteOptions();
  writeOpt.disableWAL = true;
  options.max_write_buffer_number = 2;
  options.min_write_buffer_number_to_merge = 1;
2441
  options.max_write_buffer_number_to_maintain = 1;
L
Lei Jin 已提交
2442
  CreateAndReopenWithCF({"pikachu"}, options);
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466

  // Compaction can still go through even if no thread can flush the
  // mem table.
  ASSERT_OK(Flush(0));
  ASSERT_OK(Flush(1));

  // Insert can go through
  ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1"));
  ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));

  ASSERT_EQ("v1", Get(0, "foo"));
  ASSERT_EQ("v1", Get(1, "bar"));

  sleeping_task_high.WakeUp();
  sleeping_task_high.WaitUntilDone();

  // Flush can still go through.
  ASSERT_OK(Flush(0));
  ASSERT_OK(Flush(1));

  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilDone();
}

I
Islam AbdelRahman 已提交
2467
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
2468
TEST_F(DBTest, GetProperty) {
2469 2470 2471
  // Set sizes to both background thread pool to be 1 and block them.
  env_->SetBackgroundThreads(1, Env::HIGH);
  env_->SetBackgroundThreads(1, Env::LOW);
2472 2473
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
2474
                 Env::Priority::LOW);
2475 2476 2477
  test::SleepingBackgroundTask sleeping_task_high;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
                 &sleeping_task_high, Env::Priority::HIGH);
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488

  Options options = CurrentOptions();
  WriteOptions writeOpt = WriteOptions();
  writeOpt.disableWAL = true;
  options.compaction_style = kCompactionStyleUniversal;
  options.level0_file_num_compaction_trigger = 1;
  options.compaction_options_universal.size_ratio = 50;
  options.max_background_compactions = 1;
  options.max_background_flushes = 1;
  options.max_write_buffer_number = 10;
  options.min_write_buffer_number_to_merge = 1;
2489
  options.max_write_buffer_number_to_maintain = 0;
2490
  options.write_buffer_size = 1000000;
L
Lei Jin 已提交
2491
  Reopen(options);
2492 2493 2494

  std::string big_value(1000000 * 2, 'x');
  std::string num;
2495
  uint64_t int_num;
2496 2497
  SetPerfLevel(kEnableTime);

2498 2499 2500
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
  ASSERT_EQ(int_num, 0U);
A
Andres Notzli 已提交
2501 2502 2503
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-live-data-size", &int_num));
  ASSERT_EQ(int_num, 0U);
2504

2505 2506 2507 2508 2509 2510 2511
  ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
  ASSERT_EQ(num, "0");
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
  ASSERT_EQ(num, "0");
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
  ASSERT_EQ(num, "0");
S
sdong 已提交
2512 2513
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
  ASSERT_EQ(num, "1");
2514 2515 2516 2517 2518
  perf_context.Reset();

  ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
  ASSERT_EQ(num, "1");
S
sdong 已提交
2519
  ASSERT_OK(dbfull()->Delete(writeOpt, "k-non-existing"));
2520 2521 2522 2523 2524 2525 2526
  ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
  ASSERT_EQ(num, "2");
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
  ASSERT_EQ(num, "1");
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
  ASSERT_EQ(num, "0");
S
sdong 已提交
2527
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
2528
  ASSERT_EQ(num, "2");
2529 2530 2531 2532 2533 2534 2535 2536
  // Verify the same set of properties through GetIntProperty
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.num-immutable-mem-table", &int_num));
  ASSERT_EQ(int_num, 2U);
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.mem-table-flush-pending", &int_num));
  ASSERT_EQ(int_num, 1U);
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.compaction-pending", &int_num));
2537
  ASSERT_EQ(int_num, 0U);
2538
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num));
2539
  ASSERT_EQ(int_num, 2U);
2540

2541 2542 2543 2544
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
  ASSERT_EQ(int_num, 0U);

2545
  sleeping_task_high.WakeUp();
I
Igor Canadi 已提交
2546
  sleeping_task_high.WaitUntilDone();
2547 2548 2549 2550 2551 2552 2553 2554 2555
  dbfull()->TEST_WaitForFlushMemTable();

  ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value));
  ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value));
  dbfull()->TEST_WaitForFlushMemTable();
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
  ASSERT_EQ(num, "0");
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
  ASSERT_EQ(num, "1");
S
sdong 已提交
2556 2557
  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
  ASSERT_EQ(num, "4");
2558 2559 2560 2561 2562

  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
  ASSERT_GT(int_num, 0U);

2563
  sleeping_task_low.WakeUp();
I
Igor Canadi 已提交
2564
  sleeping_task_low.WaitUntilDone();
2565

A
Andres Noetzli 已提交
2566 2567 2568 2569
  // Wait for compaction to be done. This is important because otherwise RocksDB
  // might schedule a compaction when reopening the database, failing assertion
  // (A) as a result.
  dbfull()->TEST_WaitForCompact();
2570
  options.max_open_files = 10;
L
Lei Jin 已提交
2571
  Reopen(options);
2572 2573 2574
  // After reopening, no table reader is loaded, so no memory for table readers
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
A
Andres Noetzli 已提交
2575
  ASSERT_EQ(int_num, 0U);  // (A)
2576 2577 2578 2579 2580 2581 2582 2583
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num));
  ASSERT_GT(int_num, 0U);

  // After reading a key, at least one table reader is loaded.
  Get("k5");
  ASSERT_TRUE(
      dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
  ASSERT_GT(int_num, 0U);
2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620

  // Test rocksdb.num-live-versions
  {
    options.level0_file_num_compaction_trigger = 20;
    Reopen(options);
    ASSERT_TRUE(
        dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
    ASSERT_EQ(int_num, 1U);

    // Use an iterator to hold current version
    std::unique_ptr<Iterator> iter1(dbfull()->NewIterator(ReadOptions()));

    ASSERT_OK(dbfull()->Put(writeOpt, "k6", big_value));
    Flush();
    ASSERT_TRUE(
        dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
    ASSERT_EQ(int_num, 2U);

    // Use an iterator to hold current version
    std::unique_ptr<Iterator> iter2(dbfull()->NewIterator(ReadOptions()));

    ASSERT_OK(dbfull()->Put(writeOpt, "k7", big_value));
    Flush();
    ASSERT_TRUE(
        dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
    ASSERT_EQ(int_num, 3U);

    iter2.reset();
    ASSERT_TRUE(
        dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
    ASSERT_EQ(int_num, 2U);

    iter1.reset();
    ASSERT_TRUE(
        dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
    ASSERT_EQ(int_num, 1U);
  }
2621 2622
}

2623 2624
TEST_F(DBTest, ApproximateMemoryUsage) {
  const int kNumRounds = 10;
2625 2626
  // TODO(noetzli) kFlushesPerRound does not really correlate with how many
  // flushes happen.
2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647
  const int kFlushesPerRound = 10;
  const int kWritesPerFlush = 10;
  const int kKeySize = 100;
  const int kValueSize = 1000;
  Options options;
  options.write_buffer_size = 1000;  // small write buffer
  options.min_write_buffer_number_to_merge = 4;
  options.compression = kNoCompression;
  options.create_if_missing = true;
  options = CurrentOptions(options);
  DestroyAndReopen(options);

  Random rnd(301);

  std::vector<Iterator*> iters;

  uint64_t active_mem;
  uint64_t unflushed_mem;
  uint64_t all_mem;
  uint64_t prev_all_mem;

2648 2649
  // Phase 0. The verify the initial value of all these properties are the same
  // as we have no mem-tables.
2650
  dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
2651
  dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
2652 2653 2654 2655
  dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
  ASSERT_EQ(all_mem, active_mem);
  ASSERT_EQ(all_mem, unflushed_mem);

2656 2657
  // Phase 1. Simply issue Put() and expect "cur-size-all-mem-tables" equals to
  // "size-all-mem-tables"
2658 2659 2660 2661 2662 2663
  for (int r = 0; r < kNumRounds; ++r) {
    for (int f = 0; f < kFlushesPerRound; ++f) {
      for (int w = 0; w < kWritesPerFlush; ++w) {
        Put(RandomString(&rnd, kKeySize), RandomString(&rnd, kValueSize));
      }
    }
2664 2665
    // Make sure that there is no flush between getting the two properties.
    dbfull()->TEST_WaitForFlushMemTable();
2666
    dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
2667 2668 2669 2670 2671 2672
    dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
    // in no iterator case, these two number should be the same.
    ASSERT_EQ(unflushed_mem, all_mem);
  }
  prev_all_mem = all_mem;

2673 2674
  // Phase 2. Keep issuing Put() but also create new iterators. This time we
  // expect "size-all-mem-tables" > "cur-size-all-mem-tables".
2675 2676 2677 2678 2679 2680 2681
  for (int r = 0; r < kNumRounds; ++r) {
    iters.push_back(db_->NewIterator(ReadOptions()));
    for (int f = 0; f < kFlushesPerRound; ++f) {
      for (int w = 0; w < kWritesPerFlush; ++w) {
        Put(RandomString(&rnd, kKeySize), RandomString(&rnd, kValueSize));
      }
    }
2682 2683 2684 2685
    // Force flush to prevent flush from happening between getting the
    // properties or after getting the properties and before the new round.
    Flush();

2686 2687
    // In the second round, add iterators.
    dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
2688
    dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
2689 2690 2691 2692 2693 2694 2695
    dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
    ASSERT_GT(all_mem, active_mem);
    ASSERT_GT(all_mem, unflushed_mem);
    ASSERT_GT(all_mem, prev_all_mem);
    prev_all_mem = all_mem;
  }

2696 2697
  // Phase 3. Delete iterators and expect "size-all-mem-tables" shrinks
  // whenever we release an iterator.
2698 2699
  for (auto* iter : iters) {
    delete iter;
2700 2701 2702
    dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
    // Expect the size shrinking
    ASSERT_LT(all_mem, prev_all_mem);
2703 2704
    prev_all_mem = all_mem;
  }
2705

2706
  // Expect all these three counters to be the same.
2707
  dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
2708
  dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
2709 2710 2711 2712
  dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
  ASSERT_EQ(active_mem, unflushed_mem);
  ASSERT_EQ(unflushed_mem, all_mem);

2713
  // Phase 5. Reopen, and expect all these three counters to be the same again.
2714 2715
  Reopen(options);
  dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
2716
  dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
2717 2718 2719 2720 2721
  dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
  ASSERT_EQ(active_mem, unflushed_mem);
  ASSERT_EQ(unflushed_mem, all_mem);
}

2722 2723 2724 2725
TEST_F(DBTest, EstimatePendingCompBytes) {
  // Set sizes to both background thread pool to be 1 and block them.
  env_->SetBackgroundThreads(1, Env::HIGH);
  env_->SetBackgroundThreads(1, Env::LOW);
2726 2727
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772
                 Env::Priority::LOW);

  Options options = CurrentOptions();
  WriteOptions writeOpt = WriteOptions();
  writeOpt.disableWAL = true;
  options.compaction_style = kCompactionStyleLevel;
  options.level0_file_num_compaction_trigger = 2;
  options.max_background_compactions = 1;
  options.max_background_flushes = 1;
  options.max_write_buffer_number = 10;
  options.min_write_buffer_number_to_merge = 1;
  options.max_write_buffer_number_to_maintain = 0;
  options.write_buffer_size = 1000000;
  Reopen(options);

  std::string big_value(1000000 * 2, 'x');
  std::string num;
  uint64_t int_num;

  ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
  Flush();
  ASSERT_TRUE(dbfull()->GetIntProperty(
      "rocksdb.estimate-pending-compaction-bytes", &int_num));
  ASSERT_EQ(int_num, 0U);

  ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
  Flush();
  ASSERT_TRUE(dbfull()->GetIntProperty(
      "rocksdb.estimate-pending-compaction-bytes", &int_num));
  ASSERT_EQ(int_num, 0U);

  ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
  Flush();
  ASSERT_TRUE(dbfull()->GetIntProperty(
      "rocksdb.estimate-pending-compaction-bytes", &int_num));
  ASSERT_GT(int_num, 0U);

  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilDone();

  dbfull()->TEST_WaitForCompact();
  ASSERT_TRUE(dbfull()->GetIntProperty(
      "rocksdb.estimate-pending-compaction-bytes", &int_num));
  ASSERT_EQ(int_num, 0U);
}
I
Islam AbdelRahman 已提交
2773
#endif  // ROCKSDB_LITE
2774

I
Igor Sugak 已提交
2775
TEST_F(DBTest, FLUSH) {
2776
  do {
L
Lei Jin 已提交
2777
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
2778 2779
    WriteOptions writeOpt = WriteOptions();
    writeOpt.disableWAL = true;
2780
    SetPerfLevel(kEnableTime);;
2781
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
2782
    // this will now also flush the last 2 writes
2783 2784
    ASSERT_OK(Flush(1));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
H
heyongqiang 已提交
2785

2786
    perf_context.Reset();
2787
    Get(1, "foo");
2788 2789
    ASSERT_TRUE((int) perf_context.get_from_output_files_time > 0);

L
Lei Jin 已提交
2790
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2791
    ASSERT_EQ("v1", Get(1, "foo"));
2792
    ASSERT_EQ("v1", Get(1, "bar"));
H
heyongqiang 已提交
2793

2794
    writeOpt.disableWAL = true;
2795 2796 2797
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
    ASSERT_OK(Flush(1));
H
heyongqiang 已提交
2798

L
Lei Jin 已提交
2799
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2800
    ASSERT_EQ("v2", Get(1, "bar"));
2801
    perf_context.Reset();
2802
    ASSERT_EQ("v2", Get(1, "foo"));
2803
    ASSERT_TRUE((int) perf_context.get_from_output_files_time > 0);
H
heyongqiang 已提交
2804

2805
    writeOpt.disableWAL = false;
2806 2807 2808
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
    ASSERT_OK(Flush(1));
H
heyongqiang 已提交
2809

L
Lei Jin 已提交
2810
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2811 2812
    // 'foo' should be there because its put
    // has WAL enabled.
2813 2814
    ASSERT_EQ("v3", Get(1, "foo"));
    ASSERT_EQ("v3", Get(1, "bar"));
2815 2816

    SetPerfLevel(kDisable);
2817
  } while (ChangeCompactOptions());
H
heyongqiang 已提交
2818 2819
}

I
Igor Sugak 已提交
2820
TEST_F(DBTest, RecoveryWithEmptyLog) {
S
Sanjay Ghemawat 已提交
2821
  do {
L
Lei Jin 已提交
2822
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
2823 2824
    ASSERT_OK(Put(1, "foo", "v1"));
    ASSERT_OK(Put(1, "foo", "v2"));
L
Lei Jin 已提交
2825 2826
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2827
    ASSERT_OK(Put(1, "foo", "v3"));
L
Lei Jin 已提交
2828
    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
2829
    ASSERT_EQ("v3", Get(1, "foo"));
S
Sanjay Ghemawat 已提交
2830
  } while (ChangeOptions());
J
jorlow@chromium.org 已提交
2831 2832
}

I
Islam AbdelRahman 已提交
2833
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
2834
TEST_F(DBTest, FlushSchedule) {
I
Igor Canadi 已提交
2835 2836 2837 2838 2839
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.level0_stop_writes_trigger = 1 << 10;
  options.level0_slowdown_writes_trigger = 1 << 10;
  options.min_write_buffer_number_to_merge = 1;
2840
  options.max_write_buffer_number_to_maintain = 1;
I
Igor Canadi 已提交
2841
  options.max_write_buffer_number = 2;
2842
  options.write_buffer_size = 120 * 1024;
L
Lei Jin 已提交
2843
  CreateAndReopenWithCF({"pikachu"}, options);
I
Igor Canadi 已提交
2844 2845
  std::vector<std::thread> threads;

I
Igor Canadi 已提交
2846
  std::atomic<int> thread_num(0);
I
Igor Canadi 已提交
2847 2848
  // each column family will have 5 thread, each thread generating 2 memtables.
  // each column family should end up with 10 table files
2849 2850 2851 2852 2853
  std::function<void()> fill_memtable_func = [&]() {
    int a = thread_num.fetch_add(1);
    Random rnd(a);
    WriteOptions wo;
    // this should fill up 2 memtables
A
agiardullo 已提交
2854
    for (int k = 0; k < 5000; ++k) {
2855 2856 2857 2858
      ASSERT_OK(db_->Put(wo, handles_[a & 1], RandomString(&rnd, 13), ""));
    }
  };

I
Igor Canadi 已提交
2859
  for (int i = 0; i < 10; ++i) {
2860
    threads.emplace_back(fill_memtable_func);
I
Igor Canadi 已提交
2861 2862 2863 2864 2865 2866
  }

  for (auto& t : threads) {
    t.join();
  }

I
Igor Canadi 已提交
2867 2868 2869 2870 2871 2872
  auto default_tables = GetNumberOfSstFilesForColumnFamily(db_, "default");
  auto pikachu_tables = GetNumberOfSstFilesForColumnFamily(db_, "pikachu");
  ASSERT_LE(default_tables, static_cast<uint64_t>(10));
  ASSERT_GT(default_tables, static_cast<uint64_t>(0));
  ASSERT_LE(pikachu_tables, static_cast<uint64_t>(10));
  ASSERT_GT(pikachu_tables, static_cast<uint64_t>(0));
I
Igor Canadi 已提交
2873
}
I
Islam AbdelRahman 已提交
2874
#endif  // ROCKSDB_LITE
J
jorlow@chromium.org 已提交
2875

I
Igor Sugak 已提交
2876
TEST_F(DBTest, ManifestRollOver) {
2877
  do {
2878
    Options options;
2879
    options.max_manifest_file_size = 10 ;  // 10 bytes
2880
    options = CurrentOptions(options);
L
Lei Jin 已提交
2881
    CreateAndReopenWithCF({"pikachu"}, options);
2882
    {
2883 2884 2885 2886 2887 2888
      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
M
Mayank Agarwal 已提交
2889
      ASSERT_GT(manifest_after_flush, manifest_before_flush);
L
Lei Jin 已提交
2890
      ReopenWithColumnFamilies({"default", "pikachu"}, options);
2891
      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
2892
      // check if a new manifest file got inserted or not.
2893 2894 2895
      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
2896 2897
    }
  } while (ChangeCompactOptions());
A
Abhishek Kona 已提交
2898 2899
}

I
Igor Sugak 已提交
2900
TEST_F(DBTest, IdentityAcrossRestarts) {
M
Mayank Agarwal 已提交
2901
  do {
2902 2903
    std::string id1;
    ASSERT_OK(db_->GetDbIdentity(id1));
M
Mayank Agarwal 已提交
2904 2905

    Options options = CurrentOptions();
L
Lei Jin 已提交
2906
    Reopen(options);
2907 2908
    std::string id2;
    ASSERT_OK(db_->GetDbIdentity(id2));
M
Mayank Agarwal 已提交
2909
    // id1 should match id2 because identity was not regenerated
2910
    ASSERT_EQ(id1.compare(id2), 0);
M
Mayank Agarwal 已提交
2911

2912
    std::string idfilename = IdentityFileName(dbname_);
M
Mayank Agarwal 已提交
2913
    ASSERT_OK(env_->DeleteFile(idfilename));
L
Lei Jin 已提交
2914
    Reopen(options);
2915 2916 2917 2918
    std::string id3;
    ASSERT_OK(db_->GetDbIdentity(id3));
    // id1 should NOT match id3 because identity was regenerated
    ASSERT_NE(id1.compare(id3), 0);
M
Mayank Agarwal 已提交
2919 2920 2921
  } while (ChangeCompactOptions());
}

I
Islam AbdelRahman 已提交
2922
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
2923
TEST_F(DBTest, RecoverWithLargeLog) {
2924 2925 2926
  do {
    {
      Options options = CurrentOptions();
L
Lei Jin 已提交
2927
      CreateAndReopenWithCF({"pikachu"}, options);
2928 2929 2930 2931 2932
      ASSERT_OK(Put(1, "big1", std::string(200000, '1')));
      ASSERT_OK(Put(1, "big2", std::string(200000, '2')));
      ASSERT_OK(Put(1, "small3", std::string(10, '3')));
      ASSERT_OK(Put(1, "small4", std::string(10, '4')));
      ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
2933 2934 2935 2936
    }

    // Make sure that if we re-open with a small write buffer size that
    // we flush table files in the middle of a large log file.
2937
    Options options;
2938
    options.write_buffer_size = 100000;
2939
    options = CurrentOptions(options);
L
Lei Jin 已提交
2940
    ReopenWithColumnFamilies({"default", "pikachu"}, options);
2941 2942 2943 2944 2945 2946
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 3);
    ASSERT_EQ(std::string(200000, '1'), Get(1, "big1"));
    ASSERT_EQ(std::string(200000, '2'), Get(1, "big2"));
    ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
    ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
    ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
2947
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
2948
}
I
Islam AbdelRahman 已提交
2949
#endif  // ROCKSDB_LITE
J
jorlow@chromium.org 已提交
2950

2951 2952
namespace {
class KeepFilter : public CompactionFilter {
2953 2954 2955 2956 2957 2958 2959
 public:
  virtual bool Filter(int level, const Slice& key, const Slice& value,
                      std::string* new_value, bool* value_changed) const
      override {
    return false;
  }

2960
  virtual const char* Name() const override { return "KeepFilter"; }
2961 2962 2963 2964 2965 2966 2967 2968
};

class KeepFilterFactory : public CompactionFilterFactory {
 public:
  explicit KeepFilterFactory(bool check_context = false)
      : check_context_(check_context) {}

  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
2969
      const CompactionFilter::Context& context) override {
2970
    if (check_context_) {
2971 2972
      EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction);
      EXPECT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction);
2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
    }
    return std::unique_ptr<CompactionFilter>(new KeepFilter());
  }

  virtual const char* Name() const override { return "KeepFilterFactory"; }
  bool check_context_;
  std::atomic_bool expect_full_compaction_;
  std::atomic_bool expect_manual_compaction_;
};

2983
class DelayFilter : public CompactionFilter {
2984
 public:
2985
  explicit DelayFilter(DBTestBase* d) : db_test(d) {}
2986 2987 2988 2989 2990
  virtual bool Filter(int level, const Slice& key, const Slice& value,
                      std::string* new_value,
                      bool* value_changed) const override {
    db_test->env_->addon_time_.fetch_add(1000);
    return true;
2991 2992
  }

2993 2994 2995
  virtual const char* Name() const override { return "DelayFilter"; }

 private:
2996
  DBTestBase* db_test;
2997 2998
};

2999 3000
class DelayFilterFactory : public CompactionFilterFactory {
 public:
3001
  explicit DelayFilterFactory(DBTestBase* d) : db_test(d) {}
3002 3003 3004 3005 3006 3007 3008 3009
  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
      const CompactionFilter::Context& context) override {
    return std::unique_ptr<CompactionFilter>(new DelayFilter(db_test));
  }

  virtual const char* Name() const override { return "DelayFilterFactory"; }

 private:
3010
  DBTestBase* db_test;
3011
};
3012
}  // namespace
S
sdong 已提交
3013

I
Islam AbdelRahman 已提交
3014
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
3015
TEST_F(DBTest, CompressedCache) {
I
Igor Canadi 已提交
3016
  if (!Snappy_Supported()) {
3017 3018
    return;
  }
3019 3020 3021 3022 3023 3024
  int num_iter = 80;

  // Run this test three iterations.
  // Iteration 1: only a uncompressed block cache
  // Iteration 2: only a compressed block cache
  // Iteration 3: both block cache and compressed cache
I
Igor Canadi 已提交
3025 3026 3027
  // Iteration 4: both block cache and compressed cache, but DB is not
  // compressed
  for (int iter = 0; iter < 4; iter++) {
3028
    Options options;
3029 3030
    options.write_buffer_size = 64*1024;        // small write buffer
    options.statistics = rocksdb::CreateDBStatistics();
3031
    options = CurrentOptions(options);
3032

3033
    BlockBasedTableOptions table_options;
3034 3035 3036
    switch (iter) {
      case 0:
        // only uncompressed block cache
3037 3038 3039
        table_options.block_cache = NewLRUCache(8*1024);
        table_options.block_cache_compressed = nullptr;
        options.table_factory.reset(NewBlockBasedTableFactory(table_options));
3040 3041 3042
        break;
      case 1:
        // no block cache, only compressed cache
3043 3044 3045 3046
        table_options.no_block_cache = true;
        table_options.block_cache = nullptr;
        table_options.block_cache_compressed = NewLRUCache(8*1024);
        options.table_factory.reset(NewBlockBasedTableFactory(table_options));
3047 3048 3049
        break;
      case 2:
        // both compressed and uncompressed block cache
3050 3051 3052
        table_options.block_cache = NewLRUCache(1024);
        table_options.block_cache_compressed = NewLRUCache(8*1024);
        options.table_factory.reset(NewBlockBasedTableFactory(table_options));
3053
        break;
I
Igor Canadi 已提交
3054 3055 3056
      case 3:
        // both block cache and compressed cache, but DB is not compressed
        // also, make block cache sizes bigger, to trigger block cache hits
3057 3058 3059
        table_options.block_cache = NewLRUCache(1024 * 1024);
        table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
        options.table_factory.reset(NewBlockBasedTableFactory(table_options));
I
Igor Canadi 已提交
3060 3061
        options.compression = kNoCompression;
        break;
3062 3063 3064
      default:
        ASSERT_TRUE(false);
    }
L
Lei Jin 已提交
3065
    CreateAndReopenWithCF({"pikachu"}, options);
3066 3067 3068
    // default column family doesn't have block cache
    Options no_block_cache_opts;
    no_block_cache_opts.statistics = options.statistics;
3069
    no_block_cache_opts = CurrentOptions(no_block_cache_opts);
3070 3071 3072 3073
    BlockBasedTableOptions table_options_no_bc;
    table_options_no_bc.no_block_cache = true;
    no_block_cache_opts.table_factory.reset(
        NewBlockBasedTableFactory(table_options_no_bc));
3074
    ReopenWithColumnFamilies({"default", "pikachu"},
L
Lei Jin 已提交
3075
        std::vector<Options>({no_block_cache_opts, options}));
3076 3077 3078 3079

    Random rnd(301);

    // Write 8MB (80 values, each 100K)
3080
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
3081 3082 3083 3084 3085 3086 3087
    std::vector<std::string> values;
    std::string str;
    for (int i = 0; i < num_iter; i++) {
      if (i % 4 == 0) {        // high compression ratio
        str = RandomString(&rnd, 1000);
      }
      values.push_back(str);
3088
      ASSERT_OK(Put(1, Key(i), values[i]));
3089 3090 3091
    }

    // flush all data from memtable so that reads are from block cache
3092
    ASSERT_OK(Flush(1));
3093 3094

    for (int i = 0; i < num_iter; i++) {
3095
      ASSERT_EQ(Get(1, Key(i)), values[i]);
3096 3097 3098 3099 3100 3101
    }

    // check that we triggered the appropriate code paths in the cache
    switch (iter) {
      case 0:
        // only uncompressed block cache
I
Igor Canadi 已提交
3102 3103
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
3104 3105 3106
        break;
      case 1:
        // no block cache, only compressed cache
I
Igor Canadi 已提交
3107 3108
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
3109 3110 3111
        break;
      case 2:
        // both compressed and uncompressed block cache
I
Igor Canadi 已提交
3112 3113
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
3114
        break;
I
Igor Canadi 已提交
3115 3116 3117 3118 3119 3120 3121 3122 3123
      case 3:
        // both compressed and uncompressed block cache
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_HIT), 0);
        ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
        // compressed doesn't have any hits since blocks are not compressed on
        // storage
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
        break;
3124 3125 3126
      default:
        ASSERT_TRUE(false);
    }
3127 3128

    options.create_if_missing = true;
L
Lei Jin 已提交
3129
    DestroyAndReopen(options);
3130 3131 3132
  }
}

K
kailiu 已提交
3133 3134 3135 3136 3137
static std::string CompressibleString(Random* rnd, int len) {
  std::string r;
  test::CompressibleString(rnd, 0.8, len, &r);
  return r;
}
I
Islam AbdelRahman 已提交
3138
#endif  // ROCKSDB_LITE
K
kailiu 已提交
3139

I
Igor Sugak 已提交
3140
TEST_F(DBTest, FailMoreDbPaths) {
3141
  Options options = CurrentOptions();
3142 3143 3144 3145 3146
  options.db_paths.emplace_back(dbname_, 10000000);
  options.db_paths.emplace_back(dbname_ + "_2", 1000000);
  options.db_paths.emplace_back(dbname_ + "_3", 1000000);
  options.db_paths.emplace_back(dbname_ + "_4", 1000000);
  options.db_paths.emplace_back(dbname_ + "_5", 1000000);
L
Lei Jin 已提交
3147
  ASSERT_TRUE(TryReopen(options).IsNotSupported());
3148
}
3149

3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168
void CheckColumnFamilyMeta(const ColumnFamilyMetaData& cf_meta) {
  uint64_t cf_size = 0;
  uint64_t cf_csize = 0;
  size_t file_count = 0;
  for (auto level_meta : cf_meta.levels) {
    uint64_t level_size = 0;
    uint64_t level_csize = 0;
    file_count += level_meta.files.size();
    for (auto file_meta : level_meta.files) {
      level_size += file_meta.size;
    }
    ASSERT_EQ(level_meta.size, level_size);
    cf_size += level_size;
    cf_csize += level_csize;
  }
  ASSERT_EQ(cf_meta.file_count, file_count);
  ASSERT_EQ(cf_meta.size, cf_size);
}

I
Islam AbdelRahman 已提交
3169
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
3170
TEST_F(DBTest, ColumnFamilyMetaDataTest) {
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
  Options options = CurrentOptions();
  options.create_if_missing = true;
  DestroyAndReopen(options);

  Random rnd(301);
  int key_index = 0;
  ColumnFamilyMetaData cf_meta;
  for (int i = 0; i < 100; ++i) {
    GenerateNewFile(&rnd, &key_index);
    db_->GetColumnFamilyMetaData(&cf_meta);
    CheckColumnFamilyMeta(cf_meta);
  }
}

I
Igor Canadi 已提交
3185
namespace {
3186 3187 3188 3189 3190 3191 3192 3193 3194 3195
void MinLevelHelper(DBTest* self, Options& options) {
  Random rnd(301);

  for (int num = 0;
    num < options.level0_file_num_compaction_trigger - 1;
    num++)
  {
    std::vector<std::string> values;
    // Write 120KB (12 values, each 10K)
    for (int i = 0; i < 12; i++) {
3196 3197
      values.push_back(DBTestBase::RandomString(&rnd, 10000));
      ASSERT_OK(self->Put(DBTestBase::Key(i), values[i]));
3198
    }
3199
    self->dbfull()->TEST_WaitForFlushMemTable();
3200 3201 3202 3203 3204 3205
    ASSERT_EQ(self->NumTableFilesAtLevel(0), num + 1);
  }

  //generate one more file in level-0, and should trigger level-0 compaction
  std::vector<std::string> values;
  for (int i = 0; i < 12; i++) {
3206 3207
    values.push_back(DBTestBase::RandomString(&rnd, 10000));
    ASSERT_OK(self->Put(DBTestBase::Key(i), values[i]));
3208 3209 3210 3211 3212 3213 3214
  }
  self->dbfull()->TEST_WaitForCompact();

  ASSERT_EQ(self->NumTableFilesAtLevel(0), 0);
  ASSERT_EQ(self->NumTableFilesAtLevel(1), 1);
}

A
amayank 已提交
3215 3216
// returns false if the calling-Test should be skipped
bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
3217
                        int lev, int strategy) {
3218
  fprintf(stderr, "Test with compression options : window_bits = %d, level =  %d, strategy = %d}\n", wbits, lev, strategy);
3219
  options.write_buffer_size = 100<<10; //100KB
3220
  options.arena_block_size = 4096;
3221 3222
  options.num_levels = 3;
  options.level0_file_num_compaction_trigger = 3;
3223
  options.create_if_missing = true;
3224

I
Igor Canadi 已提交
3225
  if (Snappy_Supported()) {
3226 3227
    type = kSnappyCompression;
    fprintf(stderr, "using snappy\n");
I
Igor Canadi 已提交
3228
  } else if (Zlib_Supported()) {
3229 3230
    type = kZlibCompression;
    fprintf(stderr, "using zlib\n");
I
Igor Canadi 已提交
3231
  } else if (BZip2_Supported()) {
3232 3233
    type = kBZip2Compression;
    fprintf(stderr, "using bzip2\n");
I
Igor Canadi 已提交
3234
  } else if (LZ4_Supported()) {
A
Albert Strasheim 已提交
3235 3236
    type = kLZ4Compression;
    fprintf(stderr, "using lz4\n");
3237 3238
  } else {
    fprintf(stderr, "skipping test, compression disabled\n");
A
amayank 已提交
3239
    return false;
3240
  }
3241
  options.compression_per_level.resize(options.num_levels);
3242 3243 3244 3245 3246 3247 3248 3249

  // do not compress L0
  for (int i = 0; i < 1; i++) {
    options.compression_per_level[i] = kNoCompression;
  }
  for (int i = 1; i < options.num_levels; i++) {
    options.compression_per_level[i] = type;
  }
A
amayank 已提交
3250
  return true;
3251
}
I
Igor Canadi 已提交
3252
}  // namespace
3253

I
Igor Sugak 已提交
3254
TEST_F(DBTest, MinLevelToCompress1) {
3255
  Options options = CurrentOptions();
3256
  CompressionType type = kSnappyCompression;
A
amayank 已提交
3257 3258 3259
  if (!MinLevelToCompress(type, options, -14, -1, 0)) {
    return;
  }
L
Lei Jin 已提交
3260
  Reopen(options);
3261 3262 3263 3264 3265 3266 3267 3268 3269
  MinLevelHelper(this, options);

  // do not compress L0 and L1
  for (int i = 0; i < 2; i++) {
    options.compression_per_level[i] = kNoCompression;
  }
  for (int i = 2; i < options.num_levels; i++) {
    options.compression_per_level[i] = type;
  }
L
Lei Jin 已提交
3270
  DestroyAndReopen(options);
3271 3272 3273
  MinLevelHelper(this, options);
}

I
Igor Sugak 已提交
3274
TEST_F(DBTest, MinLevelToCompress2) {
3275
  Options options = CurrentOptions();
3276
  CompressionType type = kSnappyCompression;
A
amayank 已提交
3277 3278 3279
  if (!MinLevelToCompress(type, options, 15, -1, 0)) {
    return;
  }
L
Lei Jin 已提交
3280
  Reopen(options);
3281 3282
  MinLevelHelper(this, options);

3283 3284 3285 3286 3287 3288 3289
  // do not compress L0 and L1
  for (int i = 0; i < 2; i++) {
    options.compression_per_level[i] = kNoCompression;
  }
  for (int i = 2; i < options.num_levels; i++) {
    options.compression_per_level[i] = type;
  }
L
Lei Jin 已提交
3290
  DestroyAndReopen(options);
3291 3292
  MinLevelHelper(this, options);
}
3293

I
Igor Sugak 已提交
3294
TEST_F(DBTest, RepeatedWritesToSameKey) {
3295
  do {
3296
    Options options;
3297 3298
    options.env = env_;
    options.write_buffer_size = 100000;  // Small write buffer
3299
    options = CurrentOptions(options);
L
Lei Jin 已提交
3300
    CreateAndReopenWithCF({"pikachu"}, options);
3301

3302 3303
    // We must have at most one file per level except for level-0,
    // which may have up to kL0_StopWritesTrigger files.
3304 3305
    const int kMaxFiles =
        options.num_levels + options.level0_stop_writes_trigger;
3306

3307
    Random rnd(301);
3308 3309
    std::string value =
        RandomString(&rnd, static_cast<int>(2 * options.write_buffer_size));
3310
    for (int i = 0; i < 5 * kMaxFiles; i++) {
3311 3312
      ASSERT_OK(Put(1, "key", value));
      ASSERT_LE(TotalTableFiles(1), kMaxFiles);
3313 3314
    }
  } while (ChangeCompactOptions());
3315
}
I
Islam AbdelRahman 已提交
3316
#endif  // ROCKSDB_LITE
3317

I
Igor Sugak 已提交
3318
TEST_F(DBTest, SparseMerge) {
3319 3320 3321
  do {
    Options options = CurrentOptions();
    options.compression = kNoCompression;
L
Lei Jin 已提交
3322
    CreateAndReopenWithCF({"pikachu"}, options);
3323

3324
    FillLevels("A", "Z", 1);
3325 3326 3327 3328 3329 3330 3331 3332

    // Suppose there is:
    //    small amount of data with prefix A
    //    large amount of data with prefix B
    //    small amount of data with prefix C
    // and that recent updates have made small changes to all three prefixes.
    // Check that we do not do a compaction that merges all of B in one shot.
    const std::string value(1000, 'x');
3333
    Put(1, "A", "va");
3334 3335 3336 3337
    // Write approximately 100MB of "B" values
    for (int i = 0; i < 100000; i++) {
      char key[100];
      snprintf(key, sizeof(key), "B%010d", i);
3338
      Put(1, key, value);
3339
    }
3340 3341 3342
    Put(1, "C", "vc");
    ASSERT_OK(Flush(1));
    dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
3343

3344
    // Make sparse update
3345 3346 3347 3348
    Put(1, "A", "va2");
    Put(1, "B100", "bvalue2");
    Put(1, "C", "vc2");
    ASSERT_OK(Flush(1));
3349 3350 3351

    // Compactions should not cause us to create a situation where
    // a file overlaps too much data at the next level.
3352 3353
    ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
              20 * 1048576);
3354
    dbfull()->TEST_CompactRange(0, nullptr, nullptr);
3355 3356
    ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
              20 * 1048576);
3357
    dbfull()->TEST_CompactRange(1, nullptr, nullptr);
3358 3359
    ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
              20 * 1048576);
3360
  } while (ChangeCompactOptions());
3361 3362
}

I
Islam AbdelRahman 已提交
3363
#ifndef ROCKSDB_LITE
J
jorlow@chromium.org 已提交
3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  bool result = (val >= low) && (val <= high);
  if (!result) {
    fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
            (unsigned long long)(val),
            (unsigned long long)(low),
            (unsigned long long)(high));
  }
  return result;
}

3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480
TEST_F(DBTest, ApproximateSizesMemTable) {
  Options options;
  options.write_buffer_size = 100000000;  // Large write buffer
  options.compression = kNoCompression;
  options.create_if_missing = true;
  options = CurrentOptions(options);
  DestroyAndReopen(options);

  const int N = 128;
  Random rnd(301);
  for (int i = 0; i < N; i++) {
    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
  }

  uint64_t size;
  std::string start = Key(50);
  std::string end = Key(60);
  Range r(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_GT(size, 6000);
  ASSERT_LT(size, 204800);
  // Zero if not including mem table
  db_->GetApproximateSizes(&r, 1, &size, false);
  ASSERT_EQ(size, 0);

  start = Key(500);
  end = Key(600);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_EQ(size, 0);

  for (int i = 0; i < N; i++) {
    ASSERT_OK(Put(Key(1000 + i), RandomString(&rnd, 1024)));
  }

  start = Key(500);
  end = Key(600);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_EQ(size, 0);

  start = Key(100);
  end = Key(1020);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_GT(size, 6000);

  options.max_write_buffer_number = 8;
  options.min_write_buffer_number_to_merge = 5;
  options.write_buffer_size = 1024 * N;  // Not very large
  DestroyAndReopen(options);

  int keys[N * 3];
  for (int i = 0; i < N; i++) {
    keys[i * 3] = i * 5;
    keys[i * 3 + 1] = i * 5 + 1;
    keys[i * 3 + 2] = i * 5 + 2;
  }
  std::random_shuffle(std::begin(keys), std::end(keys));

  for (int i = 0; i < N * 3; i++) {
    ASSERT_OK(Put(Key(keys[i] + 1000), RandomString(&rnd, 1024)));
  }

  start = Key(100);
  end = Key(300);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_EQ(size, 0);

  start = Key(1050);
  end = Key(1080);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_GT(size, 6000);

  start = Key(2100);
  end = Key(2300);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size, true);
  ASSERT_EQ(size, 0);

  start = Key(1050);
  end = Key(1080);
  r = Range(start, end);
  uint64_t size_with_mt, size_without_mt;
  db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
  ASSERT_GT(size_with_mt, 6000);
  db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
  ASSERT_EQ(size_without_mt, 0);

  Flush();

  for (int i = 0; i < N; i++) {
    ASSERT_OK(Put(Key(i + 1000), RandomString(&rnd, 1024)));
  }

  start = Key(1050);
  end = Key(1080);
  r = Range(start, end);
  db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
  db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
  ASSERT_GT(size_with_mt, size_without_mt);
  ASSERT_GT(size_without_mt, 6000);
}

I
Igor Sugak 已提交
3481
TEST_F(DBTest, ApproximateSizes) {
S
Sanjay Ghemawat 已提交
3482
  do {
3483
    Options options;
S
Sanjay Ghemawat 已提交
3484 3485
    options.write_buffer_size = 100000000;        // Large write buffer
    options.compression = kNoCompression;
L
Lei Jin 已提交
3486
    options.create_if_missing = true;
3487
    options = CurrentOptions(options);
L
Lei Jin 已提交
3488
    DestroyAndReopen(options);
L
Lei Jin 已提交
3489
    CreateAndReopenWithCF({"pikachu"}, options);
J
jorlow@chromium.org 已提交
3490

3491
    ASSERT_TRUE(Between(Size("", "xyz", 1), 0, 0));
L
Lei Jin 已提交
3492
    ReopenWithColumnFamilies({"default", "pikachu"}, options);
3493
    ASSERT_TRUE(Between(Size("", "xyz", 1), 0, 0));
J
jorlow@chromium.org 已提交
3494

S
Sanjay Ghemawat 已提交
3495
    // Write 8MB (80 values, each 100K)
3496
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
S
Sanjay Ghemawat 已提交
3497 3498 3499 3500 3501
    const int N = 80;
    static const int S1 = 100000;
    static const int S2 = 105000;  // Allow some expansion from metadata
    Random rnd(301);
    for (int i = 0; i < N; i++) {
3502
      ASSERT_OK(Put(1, Key(i), RandomString(&rnd, S1)));
S
Sanjay Ghemawat 已提交
3503
    }
J
jorlow@chromium.org 已提交
3504

S
Sanjay Ghemawat 已提交
3505
    // 0 because GetApproximateSizes() does not account for memtable space
3506
    ASSERT_TRUE(Between(Size("", Key(50), 1), 0, 0));
D
 
dgrogan@chromium.org 已提交
3507

S
Sanjay Ghemawat 已提交
3508 3509
    // Check sizes across recovery by reopening a few times
    for (int run = 0; run < 3; run++) {
L
Lei Jin 已提交
3510
      ReopenWithColumnFamilies({"default", "pikachu"}, options);
D
dgrogan@chromium.org 已提交
3511

S
Sanjay Ghemawat 已提交
3512 3513
      for (int compact_start = 0; compact_start < N; compact_start += 10) {
        for (int i = 0; i < N; i += 10) {
3514 3515 3516 3517
          ASSERT_TRUE(Between(Size("", Key(i), 1), S1 * i, S2 * i));
          ASSERT_TRUE(Between(Size("", Key(i) + ".suffix", 1), S1 * (i + 1),
                              S2 * (i + 1)));
          ASSERT_TRUE(Between(Size(Key(i), Key(i + 10), 1), S1 * 10, S2 * 10));
S
Sanjay Ghemawat 已提交
3518
        }
3519 3520 3521
        ASSERT_TRUE(Between(Size("", Key(50), 1), S1 * 50, S2 * 50));
        ASSERT_TRUE(
            Between(Size("", Key(50) + ".suffix", 1), S1 * 50, S2 * 50));
S
Sanjay Ghemawat 已提交
3522 3523 3524 3525 3526

        std::string cstart_str = Key(compact_start);
        std::string cend_str = Key(compact_start + 9);
        Slice cstart = cstart_str;
        Slice cend = cend_str;
3527
        dbfull()->TEST_CompactRange(0, &cstart, &cend, handles_[1]);
J
jorlow@chromium.org 已提交
3528
      }
D
dgrogan@chromium.org 已提交
3529

3530 3531
      ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
      ASSERT_GT(NumTableFilesAtLevel(1, 1), 0);
S
Sanjay Ghemawat 已提交
3532
    }
3533
    // ApproximateOffsetOf() is not yet implemented in plain table format.
I
Igor Canadi 已提交
3534
  } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction |
S
Stanislau Hlebik 已提交
3535
                         kSkipPlainTable | kSkipHashIndex));
J
jorlow@chromium.org 已提交
3536 3537
}

I
Igor Sugak 已提交
3538
TEST_F(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
S
Sanjay Ghemawat 已提交
3539 3540 3541
  do {
    Options options = CurrentOptions();
    options.compression = kNoCompression;
L
Lei Jin 已提交
3542
    CreateAndReopenWithCF({"pikachu"}, options);
J
jorlow@chromium.org 已提交
3543

S
Sanjay Ghemawat 已提交
3544 3545
    Random rnd(301);
    std::string big1 = RandomString(&rnd, 100000);
3546 3547 3548 3549 3550 3551 3552 3553
    ASSERT_OK(Put(1, Key(0), RandomString(&rnd, 10000)));
    ASSERT_OK(Put(1, Key(1), RandomString(&rnd, 10000)));
    ASSERT_OK(Put(1, Key(2), big1));
    ASSERT_OK(Put(1, Key(3), RandomString(&rnd, 10000)));
    ASSERT_OK(Put(1, Key(4), big1));
    ASSERT_OK(Put(1, Key(5), RandomString(&rnd, 10000)));
    ASSERT_OK(Put(1, Key(6), RandomString(&rnd, 300000)));
    ASSERT_OK(Put(1, Key(7), RandomString(&rnd, 10000)));
S
Sanjay Ghemawat 已提交
3554 3555 3556

    // Check sizes across recovery by reopening a few times
    for (int run = 0; run < 3; run++) {
L
Lei Jin 已提交
3557
      ReopenWithColumnFamilies({"default", "pikachu"}, options);
S
Sanjay Ghemawat 已提交
3558

3559 3560 3561 3562 3563 3564 3565 3566 3567
      ASSERT_TRUE(Between(Size("", Key(0), 1), 0, 0));
      ASSERT_TRUE(Between(Size("", Key(1), 1), 10000, 11000));
      ASSERT_TRUE(Between(Size("", Key(2), 1), 20000, 21000));
      ASSERT_TRUE(Between(Size("", Key(3), 1), 120000, 121000));
      ASSERT_TRUE(Between(Size("", Key(4), 1), 130000, 131000));
      ASSERT_TRUE(Between(Size("", Key(5), 1), 230000, 231000));
      ASSERT_TRUE(Between(Size("", Key(6), 1), 240000, 241000));
      ASSERT_TRUE(Between(Size("", Key(7), 1), 540000, 541000));
      ASSERT_TRUE(Between(Size("", Key(8), 1), 550000, 560000));
S
Sanjay Ghemawat 已提交
3568

3569
      ASSERT_TRUE(Between(Size(Key(3), Key(5), 1), 110000, 111000));
S
Sanjay Ghemawat 已提交
3570

3571
      dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
S
Sanjay Ghemawat 已提交
3572
    }
3573 3574
    // ApproximateOffsetOf() is not yet implemented in plain table format.
  } while (ChangeOptions(kSkipPlainTable));
J
jorlow@chromium.org 已提交
3575
}
I
Islam AbdelRahman 已提交
3576
#endif  // ROCKSDB_LITE
J
jorlow@chromium.org 已提交
3577

I
Igor Sugak 已提交
3578
TEST_F(DBTest, IteratorPinsRef) {
3579
  do {
L
Lei Jin 已提交
3580
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
3581
    Put(1, "foo", "hello");
J
jorlow@chromium.org 已提交
3582

3583
    // Get iterator that will yield the current contents of the DB.
3584
    Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
J
jorlow@chromium.org 已提交
3585

3586
    // Write to force compactions
3587
    Put(1, "foo", "newvalue1");
3588
    for (int i = 0; i < 100; i++) {
3589 3590
      // 100K values
      ASSERT_OK(Put(1, Key(i), Key(i) + std::string(100000, 'v')));
3591
    }
3592
    Put(1, "foo", "newvalue2");
J
jorlow@chromium.org 已提交
3593

3594 3595 3596 3597 3598 3599 3600 3601
    iter->SeekToFirst();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ("foo", iter->key().ToString());
    ASSERT_EQ("hello", iter->value().ToString());
    iter->Next();
    ASSERT_TRUE(!iter->Valid());
    delete iter;
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
3602 3603
}

I
Islam AbdelRahman 已提交
3604
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
3605
TEST_F(DBTest, Snapshot) {
3606 3607
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
S
Sanjay Ghemawat 已提交
3608
  do {
3609
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
3610 3611
    Put(0, "foo", "0v1");
    Put(1, "foo", "1v1");
3612

S
Sanjay Ghemawat 已提交
3613
    const Snapshot* s1 = db_->GetSnapshot();
3614 3615 3616
    ASSERT_EQ(1U, GetNumSnapshots());
    uint64_t time_snap1 = GetTimeOldestSnapshots();
    ASSERT_GT(time_snap1, 0U);
3617 3618
    Put(0, "foo", "0v2");
    Put(1, "foo", "1v2");
3619

S
sdong 已提交
3620
    env_->addon_time_.fetch_add(1);
3621

S
Sanjay Ghemawat 已提交
3622
    const Snapshot* s2 = db_->GetSnapshot();
3623 3624
    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
3625 3626
    Put(0, "foo", "0v3");
    Put(1, "foo", "1v3");
3627

A
agiardullo 已提交
3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643
    {
      ManagedSnapshot s3(db_);
      ASSERT_EQ(3U, GetNumSnapshots());
      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());

      Put(0, "foo", "0v4");
      Put(1, "foo", "1v4");
      ASSERT_EQ("0v1", Get(0, "foo", s1));
      ASSERT_EQ("1v1", Get(1, "foo", s1));
      ASSERT_EQ("0v2", Get(0, "foo", s2));
      ASSERT_EQ("1v2", Get(1, "foo", s2));
      ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot()));
      ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot()));
      ASSERT_EQ("0v4", Get(0, "foo"));
      ASSERT_EQ("1v4", Get(1, "foo"));
    }
S
Sanjay Ghemawat 已提交
3644

3645 3646
    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
3647 3648 3649 3650 3651 3652
    ASSERT_EQ("0v1", Get(0, "foo", s1));
    ASSERT_EQ("1v1", Get(1, "foo", s1));
    ASSERT_EQ("0v2", Get(0, "foo", s2));
    ASSERT_EQ("1v2", Get(1, "foo", s2));
    ASSERT_EQ("0v4", Get(0, "foo"));
    ASSERT_EQ("1v4", Get(1, "foo"));
J
jorlow@chromium.org 已提交
3653

S
Sanjay Ghemawat 已提交
3654
    db_->ReleaseSnapshot(s1);
3655 3656 3657 3658
    ASSERT_EQ("0v2", Get(0, "foo", s2));
    ASSERT_EQ("1v2", Get(1, "foo", s2));
    ASSERT_EQ("0v4", Get(0, "foo"));
    ASSERT_EQ("1v4", Get(1, "foo"));
3659 3660
    ASSERT_EQ(1U, GetNumSnapshots());
    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
3661

S
Sanjay Ghemawat 已提交
3662
    db_->ReleaseSnapshot(s2);
3663
    ASSERT_EQ(0U, GetNumSnapshots());
3664 3665
    ASSERT_EQ("0v4", Get(0, "foo"));
    ASSERT_EQ("1v4", Get(1, "foo"));
3666
  } while (ChangeOptions(kSkipHashCuckoo));
S
Sanjay Ghemawat 已提交
3667
}
J
jorlow@chromium.org 已提交
3668

I
Igor Sugak 已提交
3669
TEST_F(DBTest, HiddenValuesAreRemoved) {
3670 3671
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
S
Sanjay Ghemawat 已提交
3672
  do {
3673
    Options options = CurrentOptions(options_override);
L
Lei Jin 已提交
3674
    CreateAndReopenWithCF({"pikachu"}, options);
S
Sanjay Ghemawat 已提交
3675
    Random rnd(301);
3676
    FillLevels("a", "z", 1);
S
Sanjay Ghemawat 已提交
3677 3678

    std::string big = RandomString(&rnd, 50000);
3679 3680
    Put(1, "foo", big);
    Put(1, "pastfoo", "v");
S
Sanjay Ghemawat 已提交
3681
    const Snapshot* snapshot = db_->GetSnapshot();
3682 3683
    Put(1, "foo", "tiny");
    Put(1, "pastfoo2", "v2");  // Advance sequence number one more
S
Sanjay Ghemawat 已提交
3684

3685 3686
    ASSERT_OK(Flush(1));
    ASSERT_GT(NumTableFilesAtLevel(0, 1), 0);
S
Sanjay Ghemawat 已提交
3687

3688 3689
    ASSERT_EQ(big, Get(1, "foo", snapshot));
    ASSERT_TRUE(Between(Size("", "pastfoo", 1), 50000, 60000));
S
Sanjay Ghemawat 已提交
3690
    db_->ReleaseSnapshot(snapshot);
3691
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny, " + big + " ]");
S
Sanjay Ghemawat 已提交
3692
    Slice x("x");
3693 3694 3695 3696 3697 3698 3699 3700
    dbfull()->TEST_CompactRange(0, nullptr, &x, handles_[1]);
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]");
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
    ASSERT_GE(NumTableFilesAtLevel(1, 1), 1);
    dbfull()->TEST_CompactRange(1, nullptr, &x, handles_[1]);
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]");

    ASSERT_TRUE(Between(Size("", "pastfoo", 1), 0, 1000));
3701 3702
    // ApproximateOffsetOf() is not yet implemented in plain table format,
    // which is used by Size().
3703
    // skip HashCuckooRep as it does not support snapshot
I
Igor Canadi 已提交
3704 3705
  } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction |
                         kSkipPlainTable | kSkipHashCuckoo));
J
jorlow@chromium.org 已提交
3706
}
I
Islam AbdelRahman 已提交
3707
#endif  // ROCKSDB_LITE
J
jorlow@chromium.org 已提交
3708

I
Igor Sugak 已提交
3709
TEST_F(DBTest, CompactBetweenSnapshots) {
3710 3711
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
3712
  do {
3713
    Options options = CurrentOptions(options_override);
3714
    options.disable_auto_compactions = true;
L
Lei Jin 已提交
3715
    CreateAndReopenWithCF({"pikachu"}, options);
3716
    Random rnd(301);
3717
    FillLevels("a", "z", 1);
3718

3719
    Put(1, "foo", "first");
3720
    const Snapshot* snapshot1 = db_->GetSnapshot();
3721 3722 3723
    Put(1, "foo", "second");
    Put(1, "foo", "third");
    Put(1, "foo", "fourth");
3724
    const Snapshot* snapshot2 = db_->GetSnapshot();
3725 3726
    Put(1, "foo", "fifth");
    Put(1, "foo", "sixth");
3727 3728

    // All entries (including duplicates) exist
I
Igor Canadi 已提交
3729
    // before any compaction or flush is triggered.
3730
    ASSERT_EQ(AllEntriesFor("foo", 1),
3731
              "[ sixth, fifth, fourth, third, second, first ]");
3732 3733 3734
    ASSERT_EQ("sixth", Get(1, "foo"));
    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
    ASSERT_EQ("first", Get(1, "foo", snapshot1));
I
Igor Canadi 已提交
3735 3736 3737 3738

    // After a flush, "second", "third" and "fifth" should
    // be removed
    ASSERT_OK(Flush(1));
3739
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
3740 3741 3742

    // after we release the snapshot1, only two values left
    db_->ReleaseSnapshot(snapshot1);
3743
    FillLevels("a", "z", 1);
3744 3745
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
3746 3747 3748

    // We have only one valid snapshot snapshot2. Since snapshot1 is
    // not valid anymore, "first" should be removed by a compaction.
3749 3750 3751
    ASSERT_EQ("sixth", Get(1, "foo"));
    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
3752 3753 3754

    // after we release the snapshot2, only one value should be left
    db_->ReleaseSnapshot(snapshot2);
3755
    FillLevels("a", "z", 1);
3756 3757
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
3758 3759
    ASSERT_EQ("sixth", Get(1, "foo"));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
3760
    // skip HashCuckooRep as it does not support snapshot
I
Igor Canadi 已提交
3761
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
3762 3763
}

A
Andres Noetzli 已提交
3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804
TEST_F(DBTest, UnremovableSingleDelete) {
  // If we compact:
  //
  // Put(A, v1) Snapshot SingleDelete(A) Put(A, v2)
  //
  // We do not want to end up with:
  //
  // Put(A, v1) Snapshot Put(A, v2)
  //
  // Because a subsequent SingleDelete(A) would delete the Put(A, v2)
  // but not Put(A, v1), so Get(A) would return v1.
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
  do {
    Options options = CurrentOptions(options_override);
    options.disable_auto_compactions = true;
    CreateAndReopenWithCF({"pikachu"}, options);

    Put(1, "foo", "first");
    const Snapshot* snapshot = db_->GetSnapshot();
    SingleDelete(1, "foo");
    Put(1, "foo", "second");
    ASSERT_OK(Flush(1));

    ASSERT_EQ("first", Get(1, "foo", snapshot));
    ASSERT_EQ("second", Get(1, "foo"));

    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
    ASSERT_EQ("[ second, SDEL, first ]", AllEntriesFor("foo", 1));

    SingleDelete(1, "foo");

    ASSERT_EQ("first", Get(1, "foo", snapshot));
    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));

    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);

    ASSERT_EQ("first", Get(1, "foo", snapshot));
    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
3805
    db_->ReleaseSnapshot(snapshot);
A
Andres Noetzli 已提交
3806 3807 3808 3809 3810 3811 3812
    // Skip HashCuckooRep as it does not support single delete.  FIFO and
    // universal compaction do not apply to the test case.  Skip MergePut
    // because single delete does not get removed when it encounters a merge.
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
                         kSkipUniversalCompaction | kSkipMergePut));
}

I
Islam AbdelRahman 已提交
3813
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
3814
TEST_F(DBTest, DeletionMarkers1) {
I
Igor Canadi 已提交
3815 3816
  Options options = CurrentOptions();
  options.max_background_flushes = 0;
L
Lei Jin 已提交
3817
  CreateAndReopenWithCF({"pikachu"}, options);
3818 3819
  Put(1, "foo", "v1");
  ASSERT_OK(Flush(1));
3820 3821
  const int last = 2;
  MoveFilesToLevel(last, 1);
3822 3823
  // foo => v1 is now in last level
  ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
3824 3825

  // Place a table at level last-1 to prevent merging with preceding mutation
3826 3827 3828
  Put(1, "a", "begin");
  Put(1, "z", "end");
  Flush(1);
3829
  MoveFilesToLevel(last - 1, 1);
3830 3831 3832 3833 3834 3835 3836
  ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
  ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1);

  Delete(1, "foo");
  Put(1, "foo", "v2");
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
  ASSERT_OK(Flush(1));  // Moves to level last-2
3837
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
G
Gabor Cselle 已提交
3838
  Slice z("z");
3839
  dbfull()->TEST_CompactRange(last - 2, nullptr, &z, handles_[1]);
J
jorlow@chromium.org 已提交
3840 3841
  // DEL eliminated, but v1 remains because we aren't compacting that level
  // (DEL can be eliminated because v2 hides v1).
3842 3843
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
  dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1]);
3844 3845
  // Merging last-1 w/ last, so we are the base level for "foo", so
  // DEL is removed.  (as is v1).
3846
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
J
jorlow@chromium.org 已提交
3847 3848
}

I
Igor Sugak 已提交
3849
TEST_F(DBTest, DeletionMarkers2) {
I
Igor Canadi 已提交
3850
  Options options = CurrentOptions();
L
Lei Jin 已提交
3851
  CreateAndReopenWithCF({"pikachu"}, options);
3852 3853
  Put(1, "foo", "v1");
  ASSERT_OK(Flush(1));
3854 3855
  const int last = 2;
  MoveFilesToLevel(last, 1);
3856 3857
  // foo => v1 is now in last level
  ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
3858 3859

  // Place a table at level last-1 to prevent merging with preceding mutation
3860 3861 3862
  Put(1, "a", "begin");
  Put(1, "z", "end");
  Flush(1);
3863
  MoveFilesToLevel(last - 1, 1);
3864 3865 3866 3867 3868 3869 3870 3871
  ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
  ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1);

  Delete(1, "foo");
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
  ASSERT_OK(Flush(1));  // Moves to level last-2
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
  dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr, handles_[1]);
3872
  // DEL kept: "last" file overlaps
3873 3874
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
  dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1]);
3875 3876
  // Merging last-1 w/ last, so we are the base level for "foo", so
  // DEL is removed.  (as is v1).
3877
  ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
J
jorlow@chromium.org 已提交
3878 3879
}

I
Igor Sugak 已提交
3880
TEST_F(DBTest, OverlapInLevel0) {
S
Sanjay Ghemawat 已提交
3881
  do {
I
Igor Canadi 已提交
3882
    Options options = CurrentOptions();
L
Lei Jin 已提交
3883
    CreateAndReopenWithCF({"pikachu"}, options);
G
Gabor Cselle 已提交
3884

N
Naman Gupta 已提交
3885
    //Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
3886 3887 3888
    ASSERT_OK(Put(1, "100", "v100"));
    ASSERT_OK(Put(1, "999", "v999"));
    Flush(1);
3889
    MoveFilesToLevel(2, 1);
3890 3891 3892
    ASSERT_OK(Delete(1, "100"));
    ASSERT_OK(Delete(1, "999"));
    Flush(1);
3893
    MoveFilesToLevel(1, 1);
3894
    ASSERT_EQ("0,1,1", FilesPerLevel(1));
S
Sanjay Ghemawat 已提交
3895 3896 3897 3898 3899

    // Make files spanning the following ranges in level-0:
    //  files[0]  200 .. 900
    //  files[1]  300 .. 500
    // Note that files are sorted by smallest key.
3900 3901 3902 3903 3904 3905 3906 3907
    ASSERT_OK(Put(1, "300", "v300"));
    ASSERT_OK(Put(1, "500", "v500"));
    Flush(1);
    ASSERT_OK(Put(1, "200", "v200"));
    ASSERT_OK(Put(1, "600", "v600"));
    ASSERT_OK(Put(1, "900", "v900"));
    Flush(1);
    ASSERT_EQ("2,1,1", FilesPerLevel(1));
G
Gabor Cselle 已提交
3908

S
Sanjay Ghemawat 已提交
3909
    // Compact away the placeholder files we created initially
3910 3911 3912
    dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
    dbfull()->TEST_CompactRange(2, nullptr, nullptr, handles_[1]);
    ASSERT_EQ("2", FilesPerLevel(1));
G
Gabor Cselle 已提交
3913

S
Sanjay Ghemawat 已提交
3914 3915 3916
    // Do a memtable compaction.  Before bug-fix, the compaction would
    // not detect the overlap with level-0 files and would incorrectly place
    // the deletion in a deeper level.
3917 3918 3919 3920
    ASSERT_OK(Delete(1, "600"));
    Flush(1);
    ASSERT_EQ("3", FilesPerLevel(1));
    ASSERT_EQ("NOT_FOUND", Get(1, "600"));
I
Igor Canadi 已提交
3921
  } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction));
G
Gabor Cselle 已提交
3922
}
I
Islam AbdelRahman 已提交
3923
#endif  // ROCKSDB_LITE
G
Gabor Cselle 已提交
3924

I
Igor Sugak 已提交
3925
TEST_F(DBTest, ComparatorCheck) {
J
jorlow@chromium.org 已提交
3926 3927
  class NewComparator : public Comparator {
   public:
I
Igor Sugak 已提交
3928 3929 3930 3931
    virtual const char* Name() const override {
      return "rocksdb.NewComparator";
    }
    virtual int Compare(const Slice& a, const Slice& b) const override {
J
jorlow@chromium.org 已提交
3932 3933
      return BytewiseComparator()->Compare(a, b);
    }
I
Igor Sugak 已提交
3934 3935
    virtual void FindShortestSeparator(std::string* s,
                                       const Slice& l) const override {
J
jorlow@chromium.org 已提交
3936 3937
      BytewiseComparator()->FindShortestSeparator(s, l);
    }
I
Igor Sugak 已提交
3938
    virtual void FindShortSuccessor(std::string* key) const override {
J
jorlow@chromium.org 已提交
3939 3940 3941
      BytewiseComparator()->FindShortSuccessor(key);
    }
  };
3942
  Options new_options, options;
3943
  NewComparator cmp;
3944
  do {
3945
    options = CurrentOptions();
L
Lei Jin 已提交
3946
    CreateAndReopenWithCF({"pikachu"}, options);
3947
    new_options = CurrentOptions();
3948
    new_options.comparator = &cmp;
3949 3950
    // only the non-default column family has non-matching comparator
    Status s = TryReopenWithColumnFamilies({"default", "pikachu"},
L
Lei Jin 已提交
3951
        std::vector<Options>({options, new_options}));
3952 3953 3954
    ASSERT_TRUE(!s.ok());
    ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
        << s.ToString();
3955
  } while (ChangeCompactOptions());
J
jorlow@chromium.org 已提交
3956 3957
}

I
Igor Sugak 已提交
3958
TEST_F(DBTest, CustomComparator) {
H
Hans Wennborg 已提交
3959 3960
  class NumberComparator : public Comparator {
   public:
I
Igor Sugak 已提交
3961 3962 3963 3964
    virtual const char* Name() const override {
      return "test.NumberComparator";
    }
    virtual int Compare(const Slice& a, const Slice& b) const override {
3965 3966
      return ToNumber(a) - ToNumber(b);
    }
I
Igor Sugak 已提交
3967 3968
    virtual void FindShortestSeparator(std::string* s,
                                       const Slice& l) const override {
3969 3970 3971
      ToNumber(*s);     // Check format
      ToNumber(l);      // Check format
    }
I
Igor Sugak 已提交
3972
    virtual void FindShortSuccessor(std::string* key) const override {
3973 3974 3975 3976 3977
      ToNumber(*key);   // Check format
    }
   private:
    static int ToNumber(const Slice& x) {
      // Check that there are no extra characters.
3978
      EXPECT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']')
3979 3980 3981
          << EscapeString(x);
      int val;
      char ignored;
3982
      EXPECT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
3983 3984
          << EscapeString(x);
      return val;
H
Hans Wennborg 已提交
3985 3986
    }
  };
3987 3988
  Options new_options;
  NumberComparator cmp;
3989
  do {
3990
    new_options = CurrentOptions();
3991 3992
    new_options.create_if_missing = true;
    new_options.comparator = &cmp;
3993 3994
    new_options.write_buffer_size = 4096;  // Compact more often
    new_options.arena_block_size = 4096;
3995
    new_options = CurrentOptions(new_options);
L
Lei Jin 已提交
3996
    DestroyAndReopen(new_options);
L
Lei Jin 已提交
3997
    CreateAndReopenWithCF({"pikachu"}, new_options);
3998 3999
    ASSERT_OK(Put(1, "[10]", "ten"));
    ASSERT_OK(Put(1, "[0x14]", "twenty"));
4000
    for (int i = 0; i < 2; i++) {
4001 4002 4003 4004 4005 4006 4007
      ASSERT_EQ("ten", Get(1, "[10]"));
      ASSERT_EQ("ten", Get(1, "[0xa]"));
      ASSERT_EQ("twenty", Get(1, "[20]"));
      ASSERT_EQ("twenty", Get(1, "[0x14]"));
      ASSERT_EQ("NOT_FOUND", Get(1, "[15]"));
      ASSERT_EQ("NOT_FOUND", Get(1, "[0xf]"));
      Compact(1, "[0]", "[9999]");
4008 4009 4010 4011 4012
    }

    for (int run = 0; run < 2; run++) {
      for (int i = 0; i < 1000; i++) {
        char buf[100];
4013 4014 4015 4016
        snprintf(buf, sizeof(buf), "[%d]", i*10);
        ASSERT_OK(Put(1, buf, buf));
      }
      Compact(1, "[0]", "[1000000]");
4017
    }
4018
  } while (ChangeCompactOptions());
4019 4020
}

I
Igor Sugak 已提交
4021
TEST_F(DBTest, DBOpen_Options) {
4022 4023 4024
  Options options = CurrentOptions();
  std::string dbname = test::TmpDir(env_) + "/db_options_test";
  ASSERT_OK(DestroyDB(dbname, options));
J
jorlow@chromium.org 已提交
4025 4026

  // Does not exist, and create_if_missing == false: error
A
Abhishek Kona 已提交
4027
  DB* db = nullptr;
4028 4029
  options.create_if_missing = false;
  Status s = DB::Open(options, dbname, &db);
A
Abhishek Kona 已提交
4030 4031
  ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
  ASSERT_TRUE(db == nullptr);
J
jorlow@chromium.org 已提交
4032 4033

  // Does not exist, and create_if_missing == true: OK
4034 4035
  options.create_if_missing = true;
  s = DB::Open(options, dbname, &db);
J
jorlow@chromium.org 已提交
4036
  ASSERT_OK(s);
A
Abhishek Kona 已提交
4037
  ASSERT_TRUE(db != nullptr);
J
jorlow@chromium.org 已提交
4038 4039

  delete db;
A
Abhishek Kona 已提交
4040
  db = nullptr;
J
jorlow@chromium.org 已提交
4041 4042

  // Does exist, and error_if_exists == true: error
4043 4044 4045
  options.create_if_missing = false;
  options.error_if_exists = true;
  s = DB::Open(options, dbname, &db);
A
Abhishek Kona 已提交
4046 4047
  ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
  ASSERT_TRUE(db == nullptr);
J
jorlow@chromium.org 已提交
4048 4049

  // Does exist, and error_if_exists == false: OK
4050 4051 4052
  options.create_if_missing = true;
  options.error_if_exists = false;
  s = DB::Open(options, dbname, &db);
J
jorlow@chromium.org 已提交
4053
  ASSERT_OK(s);
A
Abhishek Kona 已提交
4054
  ASSERT_TRUE(db != nullptr);
J
jorlow@chromium.org 已提交
4055 4056

  delete db;
A
Abhishek Kona 已提交
4057
  db = nullptr;
J
jorlow@chromium.org 已提交
4058 4059
}

I
Igor Sugak 已提交
4060
TEST_F(DBTest, DBOpen_Change_NumLevels) {
4061 4062 4063
  Options options = CurrentOptions();
  options.create_if_missing = true;
  DestroyAndReopen(options);
4064
  ASSERT_TRUE(db_ != nullptr);
4065
  CreateAndReopenWithCF({"pikachu"}, options);
4066 4067 4068

  ASSERT_OK(Put(1, "a", "123"));
  ASSERT_OK(Put(1, "b", "234"));
4069 4070
  Flush(1);
  MoveFilesToLevel(3, 1);
4071
  Close();
4072

4073 4074 4075
  options.create_if_missing = false;
  options.num_levels = 2;
  Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
4076
  ASSERT_TRUE(strstr(s.ToString().c_str(), "Invalid argument") != nullptr);
4077
  ASSERT_TRUE(db_ == nullptr);
4078 4079
}

I
Igor Sugak 已提交
4080
TEST_F(DBTest, DestroyDBMetaDatabase) {
4081
  std::string dbname = test::TmpDir(env_) + "/db_meta";
4082
  ASSERT_OK(env_->CreateDirIfMissing(dbname));
K
Kosie van der Merwe 已提交
4083
  std::string metadbname = MetaDatabaseName(dbname, 0);
4084
  ASSERT_OK(env_->CreateDirIfMissing(metadbname));
K
Kosie van der Merwe 已提交
4085
  std::string metametadbname = MetaDatabaseName(metadbname, 0);
4086
  ASSERT_OK(env_->CreateDirIfMissing(metametadbname));
K
Kosie van der Merwe 已提交
4087 4088

  // Destroy previous versions if they exist. Using the long way.
4089 4090 4091 4092
  Options options = CurrentOptions();
  ASSERT_OK(DestroyDB(metametadbname, options));
  ASSERT_OK(DestroyDB(metadbname, options));
  ASSERT_OK(DestroyDB(dbname, options));
K
Kosie van der Merwe 已提交
4093 4094

  // Setup databases
A
Abhishek Kona 已提交
4095
  DB* db = nullptr;
4096
  ASSERT_OK(DB::Open(options, dbname, &db));
K
Kosie van der Merwe 已提交
4097
  delete db;
A
Abhishek Kona 已提交
4098
  db = nullptr;
4099
  ASSERT_OK(DB::Open(options, metadbname, &db));
K
Kosie van der Merwe 已提交
4100
  delete db;
A
Abhishek Kona 已提交
4101
  db = nullptr;
4102
  ASSERT_OK(DB::Open(options, metametadbname, &db));
K
Kosie van der Merwe 已提交
4103
  delete db;
A
Abhishek Kona 已提交
4104
  db = nullptr;
K
Kosie van der Merwe 已提交
4105 4106

  // Delete databases
4107
  ASSERT_OK(DestroyDB(dbname, options));
K
Kosie van der Merwe 已提交
4108 4109

  // Check if deletion worked.
4110 4111 4112 4113
  options.create_if_missing = false;
  ASSERT_TRUE(!(DB::Open(options, dbname, &db)).ok());
  ASSERT_TRUE(!(DB::Open(options, metadbname, &db)).ok());
  ASSERT_TRUE(!(DB::Open(options, metametadbname, &db)).ok());
K
Kosie van der Merwe 已提交
4114 4115
}

I
Islam AbdelRahman 已提交
4116
#ifndef ROCKSDB_LITE
4117
// Check that number of files does not grow when writes are dropped
I
Igor Sugak 已提交
4118
TEST_F(DBTest, DropWrites) {
4119 4120 4121
  do {
    Options options = CurrentOptions();
    options.env = env_;
4122
    options.paranoid_checks = false;
L
Lei Jin 已提交
4123
    Reopen(options);
4124

4125 4126 4127
    ASSERT_OK(Put("foo", "v1"));
    ASSERT_EQ("v1", Get("foo"));
    Compact("a", "z");
4128
    const size_t num_files = CountFiles();
I
Igor Canadi 已提交
4129 4130
    // Force out-of-space errors
    env_->drop_writes_.store(true, std::memory_order_release);
4131
    env_->sleep_counter_.Reset();
K
krad 已提交
4132
    env_->no_sleep_ = true;
4133
    for (int i = 0; i < 5; i++) {
4134 4135
      if (option_config_ != kUniversalCompactionMultiLevel &&
          option_config_ != kUniversalSubcompactions) {
4136 4137 4138 4139
        for (int level = 0; level < dbfull()->NumberLevels(); level++) {
          if (level > 0 && level == dbfull()->NumberLevels() - 1) {
            break;
          }
4140 4141
          dbfull()->TEST_CompactRange(level, nullptr, nullptr, nullptr,
                                      true /* disallow trivial move */);
S
sdong 已提交
4142
        }
4143
      } else {
4144
        dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
4145
      }
4146
    }
4147 4148 4149 4150 4151

    std::string property_value;
    ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
    ASSERT_EQ("5", property_value);

I
Igor Canadi 已提交
4152
    env_->drop_writes_.store(false, std::memory_order_release);
4153
    ASSERT_LT(CountFiles(), num_files + 3);
4154

4155
    // Check that compaction attempts slept after errors
4156 4157 4158
    // TODO @krad: Figure out why ASSERT_EQ 5 keeps failing in certain compiler
    // versions
    ASSERT_GE(env_->sleep_counter_.Read(), 4);
4159
  } while (ChangeCompactOptions());
4160 4161
}

4162
// Check background error counter bumped on flush failures.
I
Igor Sugak 已提交
4163
TEST_F(DBTest, DropWritesFlush) {
4164 4165 4166 4167
  do {
    Options options = CurrentOptions();
    options.env = env_;
    options.max_background_flushes = 1;
L
Lei Jin 已提交
4168
    Reopen(options);
4169 4170

    ASSERT_OK(Put("foo", "v1"));
I
Igor Canadi 已提交
4171 4172
    // Force out-of-space errors
    env_->drop_writes_.store(true, std::memory_order_release);
4173 4174 4175 4176 4177 4178

    std::string property_value;
    // Background error count is 0 now.
    ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
    ASSERT_EQ("0", property_value);

L
Lei Jin 已提交
4179 4180 4181
    dbfull()->TEST_FlushMemTable(true);

    ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
4182 4183
    ASSERT_EQ("1", property_value);

I
Igor Canadi 已提交
4184
    env_->drop_writes_.store(false, std::memory_order_release);
4185 4186
  } while (ChangeCompactOptions());
}
I
Islam AbdelRahman 已提交
4187
#endif  // ROCKSDB_LITE
4188 4189 4190

// Check that CompactRange() returns failure if there is not enough space left
// on device
I
Igor Sugak 已提交
4191
TEST_F(DBTest, NoSpaceCompactRange) {
4192 4193 4194 4195
  do {
    Options options = CurrentOptions();
    options.env = env_;
    options.disable_auto_compactions = true;
L
Lei Jin 已提交
4196
    Reopen(options);
4197 4198 4199 4200 4201 4202 4203

    // generate 5 tables
    for (int i = 0; i < 5; ++i) {
      ASSERT_OK(Put(Key(i), Key(i) + "v"));
      ASSERT_OK(Flush());
    }

I
Igor Canadi 已提交
4204 4205
    // Force out-of-space errors
    env_->no_space_.store(true, std::memory_order_release);
4206

4207 4208
    Status s = dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
                                           true /* disallow trivial move */);
4209 4210
    ASSERT_TRUE(s.IsIOError());

I
Igor Canadi 已提交
4211
    env_->no_space_.store(false, std::memory_order_release);
4212 4213 4214
  } while (ChangeCompactOptions());
}

I
Igor Sugak 已提交
4215
TEST_F(DBTest, NonWritableFileSystem) {
4216 4217
  do {
    Options options = CurrentOptions();
4218 4219
    options.write_buffer_size = 4096;
    options.arena_block_size = 4096;
4220
    options.env = env_;
L
Lei Jin 已提交
4221
    Reopen(options);
4222
    ASSERT_OK(Put("foo", "v1"));
4223
    env_->non_writeable_rate_.store(100);
4224 4225 4226 4227 4228 4229 4230
    std::string big(100000, 'x');
    int errors = 0;
    for (int i = 0; i < 20; i++) {
      if (!Put("foo", big).ok()) {
        errors++;
        env_->SleepForMicroseconds(100000);
      }
4231
    }
4232
    ASSERT_GT(errors, 0);
4233
    env_->non_writeable_rate_.store(0);
4234
  } while (ChangeCompactOptions());
4235 4236
}

I
Islam AbdelRahman 已提交
4237
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
4238
TEST_F(DBTest, ManifestWriteError) {
4239 4240 4241 4242 4243 4244 4245 4246 4247
  // Test for the following problem:
  // (a) Compaction produces file F
  // (b) Log record containing F is written to MANIFEST file, but Sync() fails
  // (c) GC deletes F
  // (d) After reopening DB, reads fail since deleted F is named in log record

  // We iterate twice.  In the second iteration, everything is the
  // same except the log record never makes it to the MANIFEST file.
  for (int iter = 0; iter < 2; iter++) {
I
Igor Canadi 已提交
4248
    std::atomic<bool>* error_type = (iter == 0)
4249 4250 4251 4252 4253 4254 4255 4256
        ? &env_->manifest_sync_error_
        : &env_->manifest_write_error_;

    // Insert foo=>bar mapping
    Options options = CurrentOptions();
    options.env = env_;
    options.create_if_missing = true;
    options.error_if_exists = false;
L
Lei Jin 已提交
4257
    DestroyAndReopen(options);
4258 4259 4260 4261
    ASSERT_OK(Put("foo", "bar"));
    ASSERT_EQ("bar", Get("foo"));

    // Memtable compaction (will succeed)
4262
    Flush();
4263
    ASSERT_EQ("bar", Get("foo"));
4264 4265
    const int last = 2;
    MoveFilesToLevel(2);
4266 4267 4268
    ASSERT_EQ(NumTableFilesAtLevel(last), 1);   // foo=>bar is now in last level

    // Merging compaction (will fail)
I
Igor Canadi 已提交
4269
    error_type->store(true, std::memory_order_release);
A
Abhishek Kona 已提交
4270
    dbfull()->TEST_CompactRange(last, nullptr, nullptr);  // Should fail
4271 4272 4273
    ASSERT_EQ("bar", Get("foo"));

    // Recovery: should not lose data
I
Igor Canadi 已提交
4274
    error_type->store(false, std::memory_order_release);
L
Lei Jin 已提交
4275
    Reopen(options);
4276 4277 4278
    ASSERT_EQ("bar", Get("foo"));
  }
}
I
Islam AbdelRahman 已提交
4279
#endif  // ROCKSDB_LITE
4280

I
Igor Sugak 已提交
4281
TEST_F(DBTest, PutFailsParanoid) {
I
Igor Canadi 已提交
4282 4283 4284 4285 4286 4287 4288 4289 4290 4291
  // Test the following:
  // (a) A random put fails in paranoid mode (simulate by sync fail)
  // (b) All other puts have to fail, even if writes would succeed
  // (c) All of that should happen ONLY if paranoid_checks = true

  Options options = CurrentOptions();
  options.env = env_;
  options.create_if_missing = true;
  options.error_if_exists = false;
  options.paranoid_checks = true;
L
Lei Jin 已提交
4292
  DestroyAndReopen(options);
L
Lei Jin 已提交
4293
  CreateAndReopenWithCF({"pikachu"}, options);
I
Igor Canadi 已提交
4294 4295
  Status s;

4296 4297
  ASSERT_OK(Put(1, "foo", "bar"));
  ASSERT_OK(Put(1, "foo1", "bar1"));
I
Igor Canadi 已提交
4298
  // simulate error
I
Igor Canadi 已提交
4299
  env_->log_write_error_.store(true, std::memory_order_release);
4300
  s = Put(1, "foo2", "bar2");
I
Igor Canadi 已提交
4301
  ASSERT_TRUE(!s.ok());
I
Igor Canadi 已提交
4302
  env_->log_write_error_.store(false, std::memory_order_release);
4303
  s = Put(1, "foo3", "bar3");
I
Igor Canadi 已提交
4304 4305 4306
  // the next put should fail, too
  ASSERT_TRUE(!s.ok());
  // but we're still able to read
4307
  ASSERT_EQ("bar", Get(1, "foo"));
I
Igor Canadi 已提交
4308 4309 4310

  // do the same thing with paranoid checks off
  options.paranoid_checks = false;
L
Lei Jin 已提交
4311
  DestroyAndReopen(options);
L
Lei Jin 已提交
4312
  CreateAndReopenWithCF({"pikachu"}, options);
I
Igor Canadi 已提交
4313

4314 4315
  ASSERT_OK(Put(1, "foo", "bar"));
  ASSERT_OK(Put(1, "foo1", "bar1"));
I
Igor Canadi 已提交
4316
  // simulate error
I
Igor Canadi 已提交
4317
  env_->log_write_error_.store(true, std::memory_order_release);
4318
  s = Put(1, "foo2", "bar2");
I
Igor Canadi 已提交
4319
  ASSERT_TRUE(!s.ok());
I
Igor Canadi 已提交
4320
  env_->log_write_error_.store(false, std::memory_order_release);
4321
  s = Put(1, "foo3", "bar3");
I
Igor Canadi 已提交
4322 4323 4324 4325
  // the next put should NOT fail
  ASSERT_TRUE(s.ok());
}

I
Igor Sugak 已提交
4326
TEST_F(DBTest, BloomFilter) {
4327 4328
  do {
    Options options = CurrentOptions();
4329
    env_->count_random_reads_ = true;
4330
    options.env = env_;
4331 4332 4333 4334 4335 4336 4337
    // ChangeCompactOptions() only changes compaction style, which does not
    // trigger reset of table_factory
    BlockBasedTableOptions table_options;
    table_options.no_block_cache = true;
    table_options.filter_policy.reset(NewBloomFilterPolicy(10));
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));

L
Lei Jin 已提交
4338
    CreateAndReopenWithCF({"pikachu"}, options);
S
Sanjay Ghemawat 已提交
4339

4340 4341 4342
    // Populate multiple layers
    const int N = 10000;
    for (int i = 0; i < N; i++) {
4343
      ASSERT_OK(Put(1, Key(i), Key(i)));
4344
    }
4345
    Compact(1, "a", "z");
4346
    for (int i = 0; i < N; i += 100) {
4347
      ASSERT_OK(Put(1, Key(i), Key(i)));
4348
    }
4349
    Flush(1);
S
Sanjay Ghemawat 已提交
4350

4351
    // Prevent auto compactions triggered by seeks
I
Igor Canadi 已提交
4352
    env_->delay_sstable_sync_.store(true, std::memory_order_release);
S
Sanjay Ghemawat 已提交
4353

4354 4355 4356
    // Lookup present keys.  Should rarely read from small sstable.
    env_->random_read_counter_.Reset();
    for (int i = 0; i < N; i++) {
4357
      ASSERT_EQ(Key(i), Get(1, Key(i)));
4358 4359 4360 4361 4362
    }
    int reads = env_->random_read_counter_.Read();
    fprintf(stderr, "%d present => %d reads\n", N, reads);
    ASSERT_GE(reads, N);
    ASSERT_LE(reads, N + 2*N/100);
S
Sanjay Ghemawat 已提交
4363

4364 4365 4366
    // Lookup present keys.  Should rarely read from either sstable.
    env_->random_read_counter_.Reset();
    for (int i = 0; i < N; i++) {
4367
      ASSERT_EQ("NOT_FOUND", Get(1, Key(i) + ".missing"));
4368 4369 4370 4371
    }
    reads = env_->random_read_counter_.Read();
    fprintf(stderr, "%d missing => %d reads\n", N, reads);
    ASSERT_LE(reads, 3*N/100);
S
Sanjay Ghemawat 已提交
4372

I
Igor Canadi 已提交
4373
    env_->delay_sstable_sync_.store(false, std::memory_order_release);
4374 4375
    Close();
  } while (ChangeCompactOptions());
S
Sanjay Ghemawat 已提交
4376 4377
}

I
Igor Sugak 已提交
4378
TEST_F(DBTest, BloomFilterRate) {
4379 4380 4381
  while (ChangeFilterOptions()) {
    Options options = CurrentOptions();
    options.statistics = rocksdb::CreateDBStatistics();
L
Lei Jin 已提交
4382
    CreateAndReopenWithCF({"pikachu"}, options);
4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405

    const int maxKey = 10000;
    for (int i = 0; i < maxKey; i++) {
      ASSERT_OK(Put(1, Key(i), Key(i)));
    }
    // Add a large key to make the file contain wide range
    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
    Flush(1);

    // Check if they can be found
    for (int i = 0; i < maxKey; i++) {
      ASSERT_EQ(Key(i), Get(1, Key(i)));
    }
    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);

    // Check if filter is useful
    for (int i = 0; i < maxKey; i++) {
      ASSERT_EQ("NOT_FOUND", Get(1, Key(i+33333)));
    }
    ASSERT_GE(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), maxKey*0.98);
  }
}

I
Igor Sugak 已提交
4406
TEST_F(DBTest, BloomFilterCompatibility) {
4407
  Options options = CurrentOptions();
4408 4409 4410 4411 4412 4413
  options.statistics = rocksdb::CreateDBStatistics();
  BlockBasedTableOptions table_options;
  table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));

  // Create with block based filter
L
Lei Jin 已提交
4414
  CreateAndReopenWithCF({"pikachu"}, options);
4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425

  const int maxKey = 10000;
  for (int i = 0; i < maxKey; i++) {
    ASSERT_OK(Put(1, Key(i), Key(i)));
  }
  ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
  Flush(1);

  // Check db with full filter
  table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
4426
  ReopenWithColumnFamilies({"default", "pikachu"}, options);
4427 4428 4429 4430 4431 4432 4433 4434

  // Check if they can be found
  for (int i = 0; i < maxKey; i++) {
    ASSERT_EQ(Key(i), Get(1, Key(i)));
  }
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
}

I
Igor Sugak 已提交
4435
TEST_F(DBTest, BloomFilterReverseCompatibility) {
4436
  Options options = CurrentOptions();
4437 4438 4439 4440 4441 4442
  options.statistics = rocksdb::CreateDBStatistics();
  BlockBasedTableOptions table_options;
  table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));

  // Create with full filter
L
Lei Jin 已提交
4443
  CreateAndReopenWithCF({"pikachu"}, options);
4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454

  const int maxKey = 10000;
  for (int i = 0; i < maxKey; i++) {
    ASSERT_OK(Put(1, Key(i), Key(i)));
  }
  ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
  Flush(1);

  // Check db with block_based filter
  table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
4455
  ReopenWithColumnFamilies({"default", "pikachu"}, options);
4456 4457 4458 4459 4460 4461 4462 4463

  // Check if they can be found
  for (int i = 0; i < maxKey; i++) {
    ASSERT_EQ(Key(i), Get(1, Key(i)));
  }
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
}

F
Feng Zhu 已提交
4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496
namespace {
// A wrapped bloom over default FilterPolicy
class WrappedBloom : public FilterPolicy {
 public:
  explicit WrappedBloom(int bits_per_key) :
        filter_(NewBloomFilterPolicy(bits_per_key)),
        counter_(0) {}

  ~WrappedBloom() { delete filter_; }

  const char* Name() const override { return "WrappedRocksDbFilterPolicy"; }

  void CreateFilter(const rocksdb::Slice* keys, int n, std::string* dst)
      const override {
    std::unique_ptr<rocksdb::Slice[]> user_keys(new rocksdb::Slice[n]);
    for (int i = 0; i < n; ++i) {
      user_keys[i] = convertKey(keys[i]);
    }
    return filter_->CreateFilter(user_keys.get(), n, dst);
  }

  bool KeyMayMatch(const rocksdb::Slice& key, const rocksdb::Slice& filter)
      const override {
    counter_++;
    return filter_->KeyMayMatch(convertKey(key), filter);
  }

  uint32_t GetCounter() { return counter_; }

 private:
  const FilterPolicy* filter_;
  mutable uint32_t counter_;

4497
  rocksdb::Slice convertKey(const rocksdb::Slice& key) const {
F
Feng Zhu 已提交
4498 4499 4500 4501 4502
    return key;
  }
};
}  // namespace

I
Igor Sugak 已提交
4503
TEST_F(DBTest, BloomFilterWrapper) {
4504
  Options options = CurrentOptions();
F
Feng Zhu 已提交
4505 4506 4507 4508 4509 4510 4511
  options.statistics = rocksdb::CreateDBStatistics();

  BlockBasedTableOptions table_options;
  WrappedBloom* policy = new WrappedBloom(10);
  table_options.filter_policy.reset(policy);
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));

L
Lei Jin 已提交
4512
  CreateAndReopenWithCF({"pikachu"}, options);
F
Feng Zhu 已提交
4513 4514 4515 4516 4517 4518 4519

  const int maxKey = 10000;
  for (int i = 0; i < maxKey; i++) {
    ASSERT_OK(Put(1, Key(i), Key(i)));
  }
  // Add a large key to make the file contain wide range
  ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
C
Chilledheart 已提交
4520
  ASSERT_EQ(0U, policy->GetCounter());
F
Feng Zhu 已提交
4521 4522 4523 4524 4525 4526 4527
  Flush(1);

  // Check if they can be found
  for (int i = 0; i < maxKey; i++) {
    ASSERT_EQ(Key(i), Get(1, Key(i)));
  }
  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
C
Chilledheart 已提交
4528
  ASSERT_EQ(1U * maxKey, policy->GetCounter());
F
Feng Zhu 已提交
4529 4530 4531 4532 4533 4534

  // Check if filter is useful
  for (int i = 0; i < maxKey; i++) {
    ASSERT_EQ("NOT_FOUND", Get(1, Key(i+33333)));
  }
  ASSERT_GE(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), maxKey*0.98);
C
Chilledheart 已提交
4535
  ASSERT_EQ(2U * maxKey, policy->GetCounter());
F
Feng Zhu 已提交
4536 4537
}

I
Islam AbdelRahman 已提交
4538
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
4539
TEST_F(DBTest, SnapshotFiles) {
4540 4541 4542
  do {
    Options options = CurrentOptions();
    options.write_buffer_size = 100000000;        // Large write buffer
L
Lei Jin 已提交
4543
    CreateAndReopenWithCF({"pikachu"}, options);
4544

4545
    Random rnd(301);
4546

4547
    // Write 8MB (80 values, each 100K)
4548
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
4549 4550 4551
    std::vector<std::string> values;
    for (int i = 0; i < 80; i++) {
      values.push_back(RandomString(&rnd, 100000));
4552
      ASSERT_OK(Put((i < 40), Key(i), values[i]));
4553
    }
4554

4555
    // assert that nothing makes it to disk yet.
4556
    ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
4557

4558 4559 4560 4561 4562 4563 4564
    // get a file snapshot
    uint64_t manifest_number = 0;
    uint64_t manifest_size = 0;
    std::vector<std::string> files;
    dbfull()->DisableFileDeletions();
    dbfull()->GetLiveFiles(files, &manifest_size);

4565 4566
    // CURRENT, MANIFEST, *.sst files (one for each CF)
    ASSERT_EQ(files.size(), 4U);
4567 4568 4569 4570 4571 4572

    uint64_t number = 0;
    FileType type;

    // copy these files to a new snapshot directory
    std::string snapdir = dbname_ + ".snapdir/";
4573
    ASSERT_OK(env_->CreateDirIfMissing(snapdir));
4574 4575

    for (unsigned int i = 0; i < files.size(); i++) {
I
Igor Canadi 已提交
4576 4577 4578 4579 4580
      // our clients require that GetLiveFiles returns
      // files with "/" as first character!
      ASSERT_EQ(files[i][0], '/');
      std::string src = dbname_ + files[i];
      std::string dest = snapdir + files[i];
4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593

      uint64_t size;
      ASSERT_OK(env_->GetFileSize(src, &size));

      // record the number and the size of the
      // latest manifest file
      if (ParseFileName(files[i].substr(1), &number, &type)) {
        if (type == kDescriptorFile) {
          if (number > manifest_number) {
            manifest_number = number;
            ASSERT_GE(size, manifest_size);
            size = manifest_size; // copy only valid MANIFEST data
          }
4594 4595
        }
      }
I
Igor Canadi 已提交
4596
      CopyFile(src, dest, size);
4597 4598
    }

4599 4600 4601 4602 4603 4604
    // release file snapshot
    dbfull()->DisableFileDeletions();
    // overwrite one key, this key should not appear in the snapshot
    std::vector<std::string> extras;
    for (unsigned int i = 0; i < 1; i++) {
      extras.push_back(RandomString(&rnd, 100000));
4605
      ASSERT_OK(Put(0, Key(i), extras[i]));
4606
    }
4607

4608
    // verify that data in the snapshot are correct
4609 4610 4611 4612
    std::vector<ColumnFamilyDescriptor> column_families;
    column_families.emplace_back("default", ColumnFamilyOptions());
    column_families.emplace_back("pikachu", ColumnFamilyOptions());
    std::vector<ColumnFamilyHandle*> cf_handles;
4613
    DB* snapdb;
4614
    DBOptions opts;
4615
    opts.env = env_;
4616
    opts.create_if_missing = false;
4617 4618
    Status stat =
        DB::Open(opts, snapdir, column_families, &cf_handles, &snapdb);
N
Naman Gupta 已提交
4619
    ASSERT_OK(stat);
4620 4621 4622 4623

    ReadOptions roptions;
    std::string val;
    for (unsigned int i = 0; i < 80; i++) {
4624
      stat = snapdb->Get(roptions, cf_handles[i < 40], Key(i), &val);
4625 4626
      ASSERT_EQ(values[i].compare(val), 0);
    }
4627 4628 4629
    for (auto cfh : cf_handles) {
      delete cfh;
    }
4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655
    delete snapdb;

    // look at the new live files after we added an 'extra' key
    // and after we took the first snapshot.
    uint64_t new_manifest_number = 0;
    uint64_t new_manifest_size = 0;
    std::vector<std::string> newfiles;
    dbfull()->DisableFileDeletions();
    dbfull()->GetLiveFiles(newfiles, &new_manifest_size);

    // find the new manifest file. assert that this manifest file is
    // the same one as in the previous snapshot. But its size should be
    // larger because we added an extra key after taking the
    // previous shapshot.
    for (unsigned int i = 0; i < newfiles.size(); i++) {
      std::string src = dbname_ + "/" + newfiles[i];
      // record the lognumber and the size of the
      // latest manifest file
      if (ParseFileName(newfiles[i].substr(1), &number, &type)) {
        if (type == kDescriptorFile) {
          if (number > new_manifest_number) {
            uint64_t size;
            new_manifest_number = number;
            ASSERT_OK(env_->GetFileSize(src, &size));
            ASSERT_GE(size, new_manifest_size);
          }
4656 4657 4658
        }
      }
    }
4659 4660
    ASSERT_EQ(manifest_number, new_manifest_number);
    ASSERT_GT(new_manifest_size, manifest_size);
4661

4662 4663 4664
    // release file snapshot
    dbfull()->DisableFileDeletions();
  } while (ChangeCompactOptions());
4665
}
I
Islam AbdelRahman 已提交
4666
#endif
4667

I
Igor Sugak 已提交
4668
TEST_F(DBTest, CompactOnFlush) {
4669 4670
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
4671
  do {
4672
    Options options = CurrentOptions(options_override);
4673
    options.disable_auto_compactions = true;
L
Lei Jin 已提交
4674
    CreateAndReopenWithCF({"pikachu"}, options);
4675

4676 4677 4678
    Put(1, "foo", "v1");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
4679

4680
    // Write two new keys
4681 4682 4683
    Put(1, "a", "begin");
    Put(1, "z", "end");
    Flush(1);
4684

4685
    // Case1: Delete followed by a put
4686 4687 4688
    Delete(1, "foo");
    Put(1, "foo", "v2");
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
4689

4690 4691
    // After the current memtable is flushed, the DEL should
    // have been removed
4692 4693
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
4694

4695 4696
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4697
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
4698

4699
    // Case 2: Delete followed by another delete
4700 4701 4702 4703 4704
    Delete(1, "foo");
    Delete(1, "foo");
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
4705 4706
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4707
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
4708

4709
    // Case 3: Put followed by a delete
4710 4711 4712 4713 4714
    Put(1, "foo", "v3");
    Delete(1, "foo");
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
4715 4716
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4717
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
4718

4719
    // Case 4: Put followed by another Put
4720 4721 4722 4723 4724
    Put(1, "foo", "v4");
    Put(1, "foo", "v5");
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
4725 4726
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4727
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
4728

4729
    // clear database
4730
    Delete(1, "foo");
4731 4732
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4733
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
4734

4735 4736
    // Case 5: Put followed by snapshot followed by another Put
    // Both puts should remain.
4737
    Put(1, "foo", "v6");
4738
    const Snapshot* snapshot = db_->GetSnapshot();
4739 4740 4741
    Put(1, "foo", "v7");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
4742
    db_->ReleaseSnapshot(snapshot);
4743

4744
    // clear database
4745
    Delete(1, "foo");
4746 4747
    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
                           nullptr);
4748
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
4749

4750 4751 4752
    // Case 5: snapshot followed by a put followed by another Put
    // Only the last put should remain.
    const Snapshot* snapshot1 = db_->GetSnapshot();
4753 4754 4755 4756
    Put(1, "foo", "v8");
    Put(1, "foo", "v9");
    ASSERT_OK(Flush(1));
    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
4757 4758
    db_->ReleaseSnapshot(snapshot1);
  } while (ChangeCompactOptions());
4759 4760
}

I
Igor Canadi 已提交
4761
namespace {
S
Stanislau Hlebik 已提交
4762 4763
std::vector<std::uint64_t> ListSpecificFiles(
    Env* env, const std::string& path, const FileType expected_file_type) {
4764
  std::vector<std::string> files;
S
Stanislau Hlebik 已提交
4765
  std::vector<uint64_t> file_numbers;
4766 4767 4768 4769 4770
  env->GetChildren(path, &files);
  uint64_t number;
  FileType type;
  for (size_t i = 0; i < files.size(); ++i) {
    if (ParseFileName(files[i], &number, &type)) {
S
Stanislau Hlebik 已提交
4771
      if (type == expected_file_type) {
S
Stanislau Hlebik 已提交
4772
        file_numbers.push_back(number);
4773 4774 4775
      }
    }
  }
S
Stanislau Hlebik 已提交
4776
  return std::move(file_numbers);
4777
}
S
Stanislau Hlebik 已提交
4778 4779 4780 4781

std::vector<std::uint64_t> ListTableFiles(Env* env, const std::string& path) {
  return ListSpecificFiles(env, path, kTableFile);
}
I
Igor Canadi 已提交
4782
}  // namespace
4783

I
Igor Sugak 已提交
4784
TEST_F(DBTest, FlushOneColumnFamily) {
4785
  Options options = CurrentOptions();
S
Stanislau Hlebik 已提交
4786 4787
  CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
                         "alyosha", "popovich"},
L
Lei Jin 已提交
4788
                        options);
S
Stanislau Hlebik 已提交
4789 4790 4791 4792 4793 4794 4795 4796 4797 4798

  ASSERT_OK(Put(0, "Default", "Default"));
  ASSERT_OK(Put(1, "pikachu", "pikachu"));
  ASSERT_OK(Put(2, "ilya", "ilya"));
  ASSERT_OK(Put(3, "muromec", "muromec"));
  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
  ASSERT_OK(Put(5, "nikitich", "nikitich"));
  ASSERT_OK(Put(6, "alyosha", "alyosha"));
  ASSERT_OK(Put(7, "popovich", "popovich"));

4799
  for (int i = 0; i < 8; ++i) {
S
Stanislau Hlebik 已提交
4800 4801
    Flush(i);
    auto tables = ListTableFiles(env_, dbname_);
4802
    ASSERT_EQ(tables.size(), i + 1U);
S
Stanislau Hlebik 已提交
4803 4804 4805
  }
}

I
Islam AbdelRahman 已提交
4806
#ifndef ROCKSDB_LITE
S
Stanislau Hlebik 已提交
4807 4808 4809 4810 4811
// In https://reviews.facebook.net/D20661 we change
// recovery behavior: previously for each log file each column family
// memtable was flushed, even it was empty. Now it's changed:
// we try to create the smallest number of table files by merging
// updates from multiple logs
I
Igor Sugak 已提交
4812
TEST_F(DBTest, RecoverCheckFileAmountWithSmallWriteBuffer) {
4813
  Options options = CurrentOptions();
S
Stanislau Hlebik 已提交
4814
  options.write_buffer_size = 5000000;
L
Lei Jin 已提交
4815
  CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options);
S
Stanislau Hlebik 已提交
4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830

  // Since we will reopen DB with smaller write_buffer_size,
  // each key will go to new SST file
  ASSERT_OK(Put(1, Key(10), DummyString(1000000)));
  ASSERT_OK(Put(1, Key(10), DummyString(1000000)));
  ASSERT_OK(Put(1, Key(10), DummyString(1000000)));
  ASSERT_OK(Put(1, Key(10), DummyString(1000000)));

  ASSERT_OK(Put(3, Key(10), DummyString(1)));
  // Make 'dobrynia' to be flushed and new WAL file to be created
  ASSERT_OK(Put(2, Key(10), DummyString(7500000)));
  ASSERT_OK(Put(2, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  {
    auto tables = ListTableFiles(env_, dbname_);
I
Igor Canadi 已提交
4831
    ASSERT_EQ(tables.size(), static_cast<size_t>(1));
S
Stanislau Hlebik 已提交
4832
    // Make sure 'dobrynia' was flushed: check sst files amount
I
Igor Canadi 已提交
4833 4834
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(1));
S
Stanislau Hlebik 已提交
4835 4836 4837 4838 4839 4840 4841 4842
  }
  // New WAL file
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(3, Key(10), DummyString(1)));
  ASSERT_OK(Put(3, Key(10), DummyString(1)));
  ASSERT_OK(Put(3, Key(10), DummyString(1)));

4843 4844
  options.write_buffer_size = 4096;
  options.arena_block_size = 4096;
S
Stanislau Hlebik 已提交
4845
  ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"},
L
Lei Jin 已提交
4846
                           options);
S
Stanislau Hlebik 已提交
4847 4848
  {
    // No inserts => default is empty
I
Igor Canadi 已提交
4849 4850
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
              static_cast<uint64_t>(0));
S
Stanislau Hlebik 已提交
4851
    // First 4 keys goes to separate SSTs + 1 more SST for 2 smaller keys
I
Igor Canadi 已提交
4852 4853
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
              static_cast<uint64_t>(5));
S
Stanislau Hlebik 已提交
4854
    // 1 SST for big key + 1 SST for small one
I
Igor Canadi 已提交
4855 4856
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(2));
S
Stanislau Hlebik 已提交
4857
    // 1 SST for all keys
I
Igor Canadi 已提交
4858 4859
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(1));
S
Stanislau Hlebik 已提交
4860 4861 4862 4863 4864 4865 4866 4867
  }
}

// In https://reviews.facebook.net/D20661 we change
// recovery behavior: previously for each log file each column family
// memtable was flushed, even it wasn't empty. Now it's changed:
// we try to create the smallest number of table files by merging
// updates from multiple logs
I
Igor Sugak 已提交
4868
TEST_F(DBTest, RecoverCheckFileAmount) {
4869
  Options options = CurrentOptions();
S
Stanislau Hlebik 已提交
4870
  options.write_buffer_size = 100000;
4871
  options.arena_block_size = 4 * 1024;
L
Lei Jin 已提交
4872
  CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options);
S
Stanislau Hlebik 已提交
4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884

  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(2, Key(1), DummyString(1)));

  // Make 'nikitich' memtable to be flushed
  ASSERT_OK(Put(3, Key(10), DummyString(1002400)));
  ASSERT_OK(Put(3, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  // 4 memtable are not flushed, 1 sst file
  {
    auto tables = ListTableFiles(env_, dbname_);
I
Igor Canadi 已提交
4885 4886 4887
    ASSERT_EQ(tables.size(), static_cast<size_t>(1));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(1));
S
Stanislau Hlebik 已提交
4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908
  }
  // Memtable for 'nikitich' has flushed, new WAL file has opened
  // 4 memtable still not flushed

  // Write to new WAL file
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(2, Key(1), DummyString(1)));

  // Fill up 'nikitich' one more time
  ASSERT_OK(Put(3, Key(10), DummyString(1002400)));
  // make it flush
  ASSERT_OK(Put(3, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  // There are still 4 memtable not flushed, and 2 sst tables
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(2, Key(1), DummyString(1)));

  {
    auto tables = ListTableFiles(env_, dbname_);
I
Igor Canadi 已提交
4909 4910 4911
    ASSERT_EQ(tables.size(), static_cast<size_t>(2));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(2));
S
Stanislau Hlebik 已提交
4912 4913 4914
  }

  ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"},
L
Lei Jin 已提交
4915
                           options);
S
Stanislau Hlebik 已提交
4916 4917 4918 4919 4920 4921
  {
    std::vector<uint64_t> table_files = ListTableFiles(env_, dbname_);
    // Check, that records for 'default', 'dobrynia' and 'pikachu' from
    // first, second and third WALs  went to the same SST.
    // So, there is 6 SSTs: three  for 'nikitich', one for 'default', one for
    // 'dobrynia', one for 'pikachu'
I
Igor Canadi 已提交
4922 4923 4924 4925 4926 4927 4928 4929
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
              static_cast<uint64_t>(1));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(3));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(1));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
              static_cast<uint64_t>(1));
S
Stanislau Hlebik 已提交
4930 4931 4932
  }
}

I
Igor Sugak 已提交
4933
TEST_F(DBTest, SharedWriteBuffer) {
4934
  Options options = CurrentOptions();
4935 4936 4937 4938
  options.db_write_buffer_size = 100000;  // this is the real limit
  options.write_buffer_size    = 500000;  // this is never hit
  CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options);

4939
  // Trigger a flush on CF "nikitich"
4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  ASSERT_OK(Put(1, Key(1), DummyString(1)));
  ASSERT_OK(Put(3, Key(1), DummyString(90000)));
  ASSERT_OK(Put(2, Key(2), DummyString(20000)));
  ASSERT_OK(Put(2, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
4951
              static_cast<uint64_t>(0));
4952
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
4953 4954 4955 4956
              static_cast<uint64_t>(0));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(0));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
4957
              static_cast<uint64_t>(1));
4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972
  }

  // "dobrynia": 20KB
  // Flush 'dobrynia'
  ASSERT_OK(Put(3, Key(2), DummyString(40000)));
  ASSERT_OK(Put(2, Key(2), DummyString(70000)));
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
              static_cast<uint64_t>(0));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
              static_cast<uint64_t>(0));
4973 4974 4975 4976 4977 4978
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(1));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(1));
  }

4979 4980
  // "nikitich" still has has data of 80KB
  // Inserting Data in "dobrynia" triggers "nikitich" flushing.
4981
  ASSERT_OK(Put(3, Key(2), DummyString(40000)));
4982 4983
  ASSERT_OK(Put(2, Key(2), DummyString(40000)));
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
4984 4985 4986 4987 4988
  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
4989
              static_cast<uint64_t>(0));
4990
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
4991
              static_cast<uint64_t>(0));
4992
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
4993
              static_cast<uint64_t>(1));
4994 4995 4996 4997
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
              static_cast<uint64_t>(2));
  }

4998
  // "dobrynia" still has 40KB
4999
  ASSERT_OK(Put(1, Key(2), DummyString(20000)));
5000
  ASSERT_OK(Put(0, Key(1), DummyString(10000)));
5001
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
5002 5003
  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
5004 5005
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
5006
  // This should triggers no flush
5007 5008
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
5009
              static_cast<uint64_t>(0));
5010
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
5011 5012 5013 5014
              static_cast<uint64_t>(0));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
              static_cast<uint64_t>(1));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
5015
              static_cast<uint64_t>(2));
5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030
  }

  // "default": 10KB, "pikachu": 20KB, "dobrynia": 40KB
  ASSERT_OK(Put(1, Key(2), DummyString(40000)));
  ASSERT_OK(Put(0, Key(1), DummyString(1)));
  dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
  dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
  // This should triggers flush of "pikachu"
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
              static_cast<uint64_t>(0));
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
              static_cast<uint64_t>(1));
5031
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
5032
              static_cast<uint64_t>(1));
5033
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
5034
              static_cast<uint64_t>(2));
5035 5036
  }

5037 5038 5039
  // "default": 10KB, "dobrynia": 40KB
  // Some remaining writes so 'default', 'dobrynia' and 'nikitich' flush on
  // closure.
5040 5041 5042 5043 5044
  ASSERT_OK(Put(3, Key(1), DummyString(1)));
  ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"},
                           options);
  {
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
5045
              static_cast<uint64_t>(1));
5046
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
5047
              static_cast<uint64_t>(1));
5048
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
5049
              static_cast<uint64_t>(2));
5050
    ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
5051
              static_cast<uint64_t>(3));
5052 5053
  }
}
I
Islam AbdelRahman 已提交
5054
#endif  // ROCKSDB_LITE
5055

I
Igor Sugak 已提交
5056
TEST_F(DBTest, PurgeInfoLogs) {
5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067
  Options options = CurrentOptions();
  options.keep_log_file_num = 5;
  options.create_if_missing = true;
  for (int mode = 0; mode <= 1; mode++) {
    if (mode == 1) {
      options.db_log_dir = dbname_ + "_logs";
      env_->CreateDirIfMissing(options.db_log_dir);
    } else {
      options.db_log_dir = "";
    }
    for (int i = 0; i < 8; i++) {
L
Lei Jin 已提交
5068
      Reopen(options);
5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080
    }

    std::vector<std::string> files;
    env_->GetChildren(options.db_log_dir.empty() ? dbname_ : options.db_log_dir,
                      &files);
    int info_log_count = 0;
    for (std::string file : files) {
      if (file.find("LOG") != std::string::npos) {
        info_log_count++;
      }
    }
    ASSERT_EQ(5, info_log_count);
5081

L
Lei Jin 已提交
5082
    Destroy(options);
I
Igor Canadi 已提交
5083
    // For mode (1), test DestroyDB() to delete all the logs under DB dir.
5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
    // For mode (2), no info log file should have been put under DB dir.
    std::vector<std::string> db_files;
    env_->GetChildren(dbname_, &db_files);
    for (std::string file : db_files) {
      ASSERT_TRUE(file.find("LOG") == std::string::npos);
    }

    if (mode == 1) {
      // Cleaning up
      env_->GetChildren(options.db_log_dir, &files);
      for (std::string file : files) {
        env_->DeleteFile(options.db_log_dir + "/" + file);
      }
      env_->DeleteDir(options.db_log_dir);
    }
5099 5100 5101
  }
}

A
Andres Noetzli 已提交
5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125
TEST_F(DBTest, SyncMultipleLogs) {
  const uint64_t kNumBatches = 2;
  const int kBatchSize = 1000;

  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.write_buffer_size = 4096;
  Reopen(options);

  WriteBatch batch;
  WriteOptions wo;
  wo.sync = true;

  for (uint64_t b = 0; b < kNumBatches; b++) {
    batch.Clear();
    for (int i = 0; i < kBatchSize; i++) {
      batch.Put(Key(i), DummyString(128));
    }

    dbfull()->Write(wo, &batch);
  }

  ASSERT_OK(dbfull()->SyncWAL());
}
5126

I
Islam AbdelRahman 已提交
5127
#ifndef ROCKSDB_LITE
K
krad 已提交
5128 5129 5130 5131 5132
//
// Test WAL recovery for the various modes available
//
class RecoveryTestHelper {
 public:
5133 5134 5135 5136 5137 5138 5139 5140
  // Number of WAL files to generate
  static const int kWALFilesCount = 10;
  // Starting number for the WAL file name like 00010.log
  static const int kWALFileOffset = 10;
  // Keys to be written per WAL file
  static const int kKeysPerWALFile = 1024;
  // Size of the value
  static const int kValueSize = 10;
K
krad 已提交
5141

5142
  // Create WAL files with values filled in
S
sdong 已提交
5143 5144 5145
  static void FillData(DBTest* test, Options& options, const size_t wal_count,
                       size_t& count) {
    DBOptions& db_options = options;
5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165

    count = 0;

    shared_ptr<Cache> table_cache = NewLRUCache(50000, 16);
    EnvOptions env_options;
    WriteBuffer write_buffer(db_options.db_write_buffer_size);

    unique_ptr<VersionSet> versions;
    unique_ptr<WalManager> wal_manager;
    WriteController write_controller;

    versions.reset(new VersionSet(test->dbname_, &db_options, env_options,
                                  table_cache.get(), &write_buffer,
                                  &write_controller));

    wal_manager.reset(new WalManager(db_options, env_options));

    std::unique_ptr<log::Writer> current_log_writer;

    for (size_t j = kWALFileOffset; j < wal_count + kWALFileOffset; j++) {
S
sdong 已提交
5166
      uint64_t current_log_number = j;
5167 5168 5169
      std::string fname = LogFileName(test->dbname_, current_log_number);
      unique_ptr<WritableFile> file;
      ASSERT_OK(db_options.env->NewWritableFile(fname, &file, env_options));
5170 5171
      unique_ptr<WritableFileWriter> file_writer(
          new WritableFileWriter(std::move(file), env_options));
S
sdong 已提交
5172 5173 5174
      current_log_writer.reset(
          new log::Writer(std::move(file_writer), current_log_number,
                          db_options.recycle_log_file_num > 0));
5175 5176 5177 5178 5179

      for (int i = 0; i < kKeysPerWALFile; i++) {
        std::string key = "key" + ToString(count++);
        std::string value = test->DummyString(kValueSize);
        assert(current_log_writer.get() != nullptr);
S
sdong 已提交
5180
        uint64_t seq = versions->LastSequence() + 1;
5181 5182 5183 5184 5185 5186
        WriteBatch batch;
        batch.Put(key, value);
        WriteBatchInternal::SetSequence(&batch, seq);
        current_log_writer->AddRecord(WriteBatchInternal::Contents(&batch));
        versions->SetLastSequence(seq);
      }
K
krad 已提交
5187
    }
5188 5189 5190 5191 5192 5193 5194 5195 5196 5197
  }

  // Recreate and fill the store with some data
  static size_t FillData(DBTest* test, Options& options) {
    options.create_if_missing = true;
    test->DestroyAndReopen(options);
    test->Close();

    size_t count = 0;
    FillData(test, options, kWALFilesCount, count);
K
krad 已提交
5198 5199 5200 5201 5202 5203
    return count;
  }

  // Read back all the keys we wrote and return the number of keys found
  static size_t GetData(DBTest* test) {
    size_t count = 0;
5204
    for (size_t i = 0; i < kWALFilesCount * kKeysPerWALFile; i++) {
K
krad 已提交
5205 5206 5207 5208 5209 5210 5211
      if (test->Get("key" + ToString(i)) != "NOT_FOUND") {
        ++count;
      }
    }
    return count;
  }

5212
  // Manuall corrupt the specified WAL
S
sdong 已提交
5213 5214 5215
  static void CorruptWAL(DBTest* test, Options& options, const double off,
                         const double len, const int wal_file_id,
                         const bool trunc = false) {
5216 5217 5218 5219 5220
    Env* env = options.env;
    std::string fname = LogFileName(test->dbname_, wal_file_id);
    uint64_t size;
    ASSERT_OK(env->GetFileSize(fname, &size));
    ASSERT_GT(size, 0);
D
Dmitri Smirnov 已提交
5221 5222 5223 5224 5225 5226 5227 5228
#ifdef OS_WIN
    // Windows disk cache behaves differently. When we truncate
    // the original content is still in the cache due to the original
    // handle is still open. Generally, in Windows, one prohibits
    // shared access to files and it is not needed for WAL but we allow
    // it to induce corruption at various tests.
    test->Close();
#endif
5229 5230 5231 5232 5233 5234 5235
    if (trunc) {
      ASSERT_EQ(0, truncate(fname.c_str(), size * off));
    } else {
      InduceCorruption(fname, size * off, size * len);
    }
  }

K
krad 已提交
5236 5237 5238
  // Overwrite data with 'a' from offset for length len
  static void InduceCorruption(const std::string& filename, uint32_t offset,
                               uint32_t len) {
5239
    ASSERT_GT(len, 0U);
K
krad 已提交
5240 5241 5242 5243 5244 5245

    int fd = open(filename.c_str(), O_RDWR);

    ASSERT_GT(fd, 0);
    ASSERT_EQ(offset, lseek(fd, offset, SEEK_SET));

D
Dmitri Smirnov 已提交
5246
    void* buf = alloca(len);
K
krad 已提交
5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258
    memset(buf, 'a', len);
    ASSERT_EQ(len, write(fd, buf, len));

    close(fd);
  }
};

// Test scope:
// - We expect to open the data store when there is incomplete trailing writes
// at the end of any of the logs
// - We do not expect to open the data store for corruption
TEST_F(DBTest, kTolerateCorruptedTailRecords) {
S
sdong 已提交
5259 5260
  const int jstart = RecoveryTestHelper::kWALFileOffset;
  const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
5261

S
sdong 已提交
5262 5263
  for (auto trunc : {true, false}) {        /* Corruption style */
    for (int i = 0; i < 4; i++) {           /* Corruption offset position */
5264 5265 5266 5267 5268
      for (int j = jstart; j < jend; j++) { /* WAL file */
        // Fill data for testing
        Options options = CurrentOptions();
        const size_t row_count = RecoveryTestHelper::FillData(this, options);
        // test checksum failure or parsing
S
sdong 已提交
5269 5270
        RecoveryTestHelper::CorruptWAL(this, options, /*off=*/i * .3,
                                       /*len%=*/.1, /*wal=*/j, trunc);
5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284

        if (trunc) {
          options.wal_recovery_mode =
              WALRecoveryMode::kTolerateCorruptedTailRecords;
          options.create_if_missing = false;
          ASSERT_OK(TryReopen(options));
          const size_t recovered_row_count = RecoveryTestHelper::GetData(this);
          ASSERT_TRUE(i == 0 || recovered_row_count > 0);
          ASSERT_LT(recovered_row_count, row_count);
        } else {
          options.wal_recovery_mode =
              WALRecoveryMode::kTolerateCorruptedTailRecords;
          ASSERT_NOK(TryReopen(options));
        }
K
krad 已提交
5285 5286 5287 5288 5289 5290 5291 5292 5293
      }
    }
  }
}

// Test scope:
// We don't expect the data store to be opened if there is any corruption
// (leading, middle or trailing -- incomplete writes or corruption)
TEST_F(DBTest, kAbsoluteConsistency) {
S
sdong 已提交
5294 5295
  const int jstart = RecoveryTestHelper::kWALFileOffset;
  const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
5296 5297

  // Verify clean slate behavior
K
krad 已提交
5298 5299 5300
  Options options = CurrentOptions();
  const size_t row_count = RecoveryTestHelper::FillData(this, options);
  options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency;
5301
  options.create_if_missing = false;
K
krad 已提交
5302 5303 5304
  ASSERT_OK(TryReopen(options));
  ASSERT_EQ(RecoveryTestHelper::GetData(this), row_count);

5305
  for (auto trunc : {true, false}) { /* Corruption style */
S
sdong 已提交
5306
    for (int i = 0; i < 4; i++) {    /* Corruption offset position */
K
krad 已提交
5307 5308 5309 5310
      if (trunc && i == 0) {
        continue;
      }

5311 5312 5313 5314
      for (int j = jstart; j < jend; j++) { /* wal files */
        // fill with new date
        RecoveryTestHelper::FillData(this, options);
        // corrupt the wal
S
sdong 已提交
5315
        RecoveryTestHelper::CorruptWAL(this, options, /*off=*/i * .3,
5316 5317 5318 5319 5320 5321
                                       /*len%=*/.1, j, trunc);
        // verify
        options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency;
        options.create_if_missing = false;
        ASSERT_NOK(TryReopen(options));
      }
K
krad 已提交
5322 5323 5324 5325 5326 5327 5328 5329
    }
  }
}

// Test scope:
// - We expect to open data store under all circumstances
// - We expect only data upto the point where the first error was encountered
TEST_F(DBTest, kPointInTimeRecovery) {
S
sdong 已提交
5330
  const int jstart = RecoveryTestHelper::kWALFileOffset;
5331
  const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
S
sdong 已提交
5332 5333
  const int maxkeys =
      RecoveryTestHelper::kWALFilesCount * RecoveryTestHelper::kKeysPerWALFile;
5334

S
sdong 已提交
5335 5336
  for (auto trunc : {true, false}) {        /* Corruption style */
    for (int i = 0; i < 4; i++) {           /* Offset of corruption */
5337 5338 5339 5340 5341 5342
      for (int j = jstart; j < jend; j++) { /* WAL file */
        // Fill data for testing
        Options options = CurrentOptions();
        const size_t row_count = RecoveryTestHelper::FillData(this, options);

        // Corrupt the wal
S
sdong 已提交
5343
        RecoveryTestHelper::CorruptWAL(this, options, /*off=*/i * .3,
5344 5345 5346 5347 5348 5349
                                       /*len%=*/.1, j, trunc);

        // Verify
        options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
        options.create_if_missing = false;
        ASSERT_OK(TryReopen(options));
K
krad 已提交
5350

5351 5352 5353
        // Probe data for invariants
        size_t recovered_row_count = RecoveryTestHelper::GetData(this);
        ASSERT_LT(recovered_row_count, row_count);
K
krad 已提交
5354

5355 5356 5357 5358 5359 5360 5361 5362
        bool expect_data = true;
        for (size_t k = 0; k < maxkeys; ++k) {
          bool found = Get("key" + ToString(i)) != "NOT_FOUND";
          if (expect_data && !found) {
            expect_data = false;
          }
          ASSERT_EQ(found, expect_data);
        }
K
krad 已提交
5363

5364
        const size_t min = RecoveryTestHelper::kKeysPerWALFile *
S
sdong 已提交
5365
                           (j - RecoveryTestHelper::kWALFileOffset);
5366 5367 5368
        ASSERT_GE(recovered_row_count, min);
        if (!trunc && i != 0) {
          const size_t max = RecoveryTestHelper::kKeysPerWALFile *
S
sdong 已提交
5369
                             (j - RecoveryTestHelper::kWALFileOffset + 1);
5370
          ASSERT_LE(recovered_row_count, max);
K
krad 已提交
5371 5372 5373 5374 5375 5376 5377 5378 5379 5380
        }
      }
    }
  }
}

// Test scope:
// - We expect to open the data store under all scenarios
// - We expect to have recovered records past the corruption zone
TEST_F(DBTest, kSkipAnyCorruptedRecords) {
S
sdong 已提交
5381 5382
  const int jstart = RecoveryTestHelper::kWALFileOffset;
  const int jend = jstart + RecoveryTestHelper::kWALFilesCount;
5383

S
sdong 已提交
5384 5385
  for (auto trunc : {true, false}) {        /* Corruption style */
    for (int i = 0; i < 4; i++) {           /* Corruption offset */
5386 5387 5388 5389 5390 5391
      for (int j = jstart; j < jend; j++) { /* wal files */
        // Fill data for testing
        Options options = CurrentOptions();
        const size_t row_count = RecoveryTestHelper::FillData(this, options);

        // Corrupt the WAL
S
sdong 已提交
5392
        RecoveryTestHelper::CorruptWAL(this, options, /*off=*/i * .3,
5393 5394 5395 5396 5397 5398
                                       /*len%=*/.1, j, trunc);

        // Verify behavior
        options.wal_recovery_mode = WALRecoveryMode::kSkipAnyCorruptedRecords;
        options.create_if_missing = false;
        ASSERT_OK(TryReopen(options));
K
krad 已提交
5399

5400 5401 5402
        // Probe data for invariants
        size_t recovered_row_count = RecoveryTestHelper::GetData(this);
        ASSERT_LT(recovered_row_count, row_count);
K
krad 已提交
5403

5404 5405 5406
        if (!trunc) {
          ASSERT_TRUE(i != 0 || recovered_row_count > 0);
        }
K
krad 已提交
5407 5408 5409 5410 5411
      }
    }
  }
}

5412 5413 5414
// Multi-threaded test:
namespace {

5415 5416
static const int kColumnFamilies = 10;
static const int kNumThreads = 10;
5417 5418 5419 5420 5421
static const int kTestSeconds = 10;
static const int kNumKeys = 1000;

struct MTState {
  DBTest* test;
I
Igor Canadi 已提交
5422 5423 5424
  std::atomic<bool> stop;
  std::atomic<int> counter[kNumThreads];
  std::atomic<bool> thread_done[kNumThreads];
5425 5426 5427 5428 5429 5430 5431 5432 5433
};

struct MTThread {
  MTState* state;
  int id;
};

static void MTThreadBody(void* arg) {
  MTThread* t = reinterpret_cast<MTThread*>(arg);
5434
  int id = t->id;
5435
  DB* db = t->state->test->db_;
I
Igor Canadi 已提交
5436
  int counter = 0;
5437 5438
  fprintf(stderr, "... starting thread %d\n", id);
  Random rnd(1000 + id);
5439
  char valbuf[1500];
I
Igor Canadi 已提交
5440 5441
  while (t->state->stop.load(std::memory_order_acquire) == false) {
    t->state->counter[id].store(counter, std::memory_order_release);
5442 5443 5444 5445 5446 5447

    int key = rnd.Uniform(kNumKeys);
    char keybuf[20];
    snprintf(keybuf, sizeof(keybuf), "%016d", key);

    if (rnd.OneIn(2)) {
5448 5449
      // Write values of the form <key, my id, counter, cf, unique_id>.
      // into each of the CFs
5450
      // We add some padding for force compactions.
5451
      int unique_id = rnd.Uniform(1000000);
5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470

      // Half of the time directly use WriteBatch. Half of the time use
      // WriteBatchWithIndex.
      if (rnd.OneIn(2)) {
        WriteBatch batch;
        for (int cf = 0; cf < kColumnFamilies; ++cf) {
          snprintf(valbuf, sizeof(valbuf), "%d.%d.%d.%d.%-1000d", key, id,
                   static_cast<int>(counter), cf, unique_id);
          batch.Put(t->state->test->handles_[cf], Slice(keybuf), Slice(valbuf));
        }
        ASSERT_OK(db->Write(WriteOptions(), &batch));
      } else {
        WriteBatchWithIndex batch(db->GetOptions().comparator);
        for (int cf = 0; cf < kColumnFamilies; ++cf) {
          snprintf(valbuf, sizeof(valbuf), "%d.%d.%d.%d.%-1000d", key, id,
                   static_cast<int>(counter), cf, unique_id);
          batch.Put(t->state->test->handles_[cf], Slice(keybuf), Slice(valbuf));
        }
        ASSERT_OK(db->Write(WriteOptions(), batch.GetWriteBatch()));
5471
      }
5472
    } else {
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486
      // Read a value and verify that it matches the pattern written above
      // and that writes to all column families were atomic (unique_id is the
      // same)
      std::vector<Slice> keys(kColumnFamilies, Slice(keybuf));
      std::vector<std::string> values;
      std::vector<Status> statuses =
          db->MultiGet(ReadOptions(), t->state->test->handles_, keys, &values);
      Status s = statuses[0];
      // all statuses have to be the same
      for (size_t i = 1; i < statuses.size(); ++i) {
        // they are either both ok or both not-found
        ASSERT_TRUE((s.ok() && statuses[i].ok()) ||
                    (s.IsNotFound() && statuses[i].IsNotFound()));
      }
5487 5488 5489 5490 5491
      if (s.IsNotFound()) {
        // Key has not yet been written
      } else {
        // Check that the writer thread counter is >= the counter in the value
        ASSERT_OK(s);
5492 5493 5494 5495 5496 5497 5498 5499 5500
        int unique_id = -1;
        for (int i = 0; i < kColumnFamilies; ++i) {
          int k, w, c, cf, u;
          ASSERT_EQ(5, sscanf(values[i].c_str(), "%d.%d.%d.%d.%d", &k, &w,
                              &c, &cf, &u))
              << values[i];
          ASSERT_EQ(k, key);
          ASSERT_GE(w, 0);
          ASSERT_LT(w, kNumThreads);
I
Igor Canadi 已提交
5501
          ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire));
5502 5503 5504 5505 5506 5507 5508 5509 5510
          ASSERT_EQ(cf, i);
          if (i == 0) {
            unique_id = u;
          } else {
            // this checks that updates across column families happened
            // atomically -- all unique ids are the same
            ASSERT_EQ(u, unique_id);
          }
        }
5511 5512 5513 5514
      }
    }
    counter++;
  }
I
Igor Canadi 已提交
5515
  t->state->thread_done[id].store(true, std::memory_order_release);
5516
  fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
5517 5518
}

H
Hans Wennborg 已提交
5519
}  // namespace
5520

5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538
class MultiThreadedDBTest : public DBTest,
                            public ::testing::WithParamInterface<int> {
 public:
  virtual void SetUp() override { option_config_ = GetParam(); }

  static std::vector<int> GenerateOptionConfigs() {
    std::vector<int> optionConfigs;
    for (int optionConfig = kDefault; optionConfig < kEnd; ++optionConfig) {
      // skip as HashCuckooRep does not support snapshot
      if (optionConfig != kHashCuckoo) {
        optionConfigs.push_back(optionConfig);
      }
    }
    return optionConfigs;
  }
};

TEST_P(MultiThreadedDBTest, MultiThreaded) {
5539 5540
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553
  std::vector<std::string> cfs;
  for (int i = 1; i < kColumnFamilies; ++i) {
    cfs.push_back(ToString(i));
  }
  CreateAndReopenWithCF(cfs, CurrentOptions(options_override));
  // Initialize state
  MTState mt;
  mt.test = this;
  mt.stop.store(false, std::memory_order_release);
  for (int id = 0; id < kNumThreads; id++) {
    mt.counter[id].store(0, std::memory_order_release);
    mt.thread_done[id].store(false, std::memory_order_release);
  }
5554

5555 5556 5557 5558 5559 5560 5561
  // Start threads
  MTThread thread[kNumThreads];
  for (int id = 0; id < kNumThreads; id++) {
    thread[id].state = &mt;
    thread[id].id = id;
    env_->StartThread(MTThreadBody, &thread[id]);
  }
5562

5563 5564
  // Let them run for a while
  env_->SleepForMicroseconds(kTestSeconds * 1000000);
5565

5566 5567 5568 5569 5570
  // Stop the threads and wait for them to finish
  mt.stop.store(true, std::memory_order_release);
  for (int id = 0; id < kNumThreads; id++) {
    while (mt.thread_done[id].load(std::memory_order_acquire) == false) {
      env_->SleepForMicroseconds(100000);
5571
    }
5572
  }
5573 5574
}

5575 5576 5577
INSTANTIATE_TEST_CASE_P(
    MultiThreaded, MultiThreadedDBTest,
    ::testing::ValuesIn(MultiThreadedDBTest::GenerateOptionConfigs()));
I
Islam AbdelRahman 已提交
5578
#endif  // ROCKSDB_LITE
5579

5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598
// Group commit test:
namespace {

static const int kGCNumThreads = 4;
static const int kGCNumKeys = 1000;

struct GCThread {
  DB* db;
  int id;
  std::atomic<bool> done;
};

static void GCThreadBody(void* arg) {
  GCThread* t = reinterpret_cast<GCThread*>(arg);
  int id = t->id;
  DB* db = t->db;
  WriteOptions wo;

  for (int i = 0; i < kGCNumKeys; ++i) {
5599
    std::string kv(ToString(i + id * kGCNumKeys));
5600 5601 5602 5603 5604 5605 5606
    ASSERT_OK(db->Put(wo, kv, kv));
  }
  t->done = true;
}

}  // namespace

I
Igor Sugak 已提交
5607
TEST_F(DBTest, GroupCommitTest) {
5608
  do {
5609
    Options options = CurrentOptions();
5610 5611
    options.env = env_;
    env_->log_write_slowdown_.store(100);
5612
    options.statistics = rocksdb::CreateDBStatistics();
L
Lei Jin 已提交
5613
    Reopen(options);
5614

5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628
    // Start threads
    GCThread thread[kGCNumThreads];
    for (int id = 0; id < kGCNumThreads; id++) {
      thread[id].id = id;
      thread[id].db = db_;
      thread[id].done = false;
      env_->StartThread(GCThreadBody, &thread[id]);
    }

    for (int id = 0; id < kGCNumThreads; id++) {
      while (thread[id].done == false) {
        env_->SleepForMicroseconds(100000);
      }
    }
5629 5630
    env_->log_write_slowdown_.store(0);

5631
    ASSERT_GT(TestGetTickerCount(options, WRITE_DONE_BY_OTHER), 0);
5632 5633 5634

    std::vector<std::string> expected_db;
    for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) {
5635
      expected_db.push_back(ToString(i));
5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647
    }
    sort(expected_db.begin(), expected_db.end());

    Iterator* itr = db_->NewIterator(ReadOptions());
    itr->SeekToFirst();
    for (auto x : expected_db) {
      ASSERT_TRUE(itr->Valid());
      ASSERT_EQ(itr->key().ToString(), x);
      ASSERT_EQ(itr->value().ToString(), x);
      itr->Next();
    }
    ASSERT_TRUE(!itr->Valid());
I
Igor Canadi 已提交
5648
    delete itr;
5649

5650 5651 5652
    HistogramData hist_data = {0};
    options.statistics->histogramData(DB_WRITE, &hist_data);
    ASSERT_GT(hist_data.average, 0.0);
5653
  } while (ChangeOptions(kSkipNoSeekToLast));
5654 5655
}

5656 5657 5658 5659
namespace {
typedef std::map<std::string, std::string> KVMap;
}

J
jorlow@chromium.org 已提交
5660 5661
class ModelDB: public DB {
 public:
5662 5663 5664
  class ModelSnapshot : public Snapshot {
   public:
    KVMap map_;
I
Igor Canadi 已提交
5665

I
Igor Sugak 已提交
5666
    virtual SequenceNumber GetSequenceNumber() const override {
I
Igor Canadi 已提交
5667 5668 5669 5670
      // no need to call this
      assert(false);
      return 0;
    }
5671 5672
  };

5673
  explicit ModelDB(const Options& options) : options_(options) {}
5674
  using DB::Put;
5675
  virtual Status Put(const WriteOptions& o, ColumnFamilyHandle* cf,
I
Igor Sugak 已提交
5676
                     const Slice& k, const Slice& v) override {
5677
    WriteBatch batch;
I
Igor Canadi 已提交
5678
    batch.Put(cf, k, v);
5679
    return Write(o, &batch);
5680 5681
  }
  using DB::Delete;
5682
  virtual Status Delete(const WriteOptions& o, ColumnFamilyHandle* cf,
I
Igor Sugak 已提交
5683
                        const Slice& key) override {
5684
    WriteBatch batch;
I
Igor Canadi 已提交
5685
    batch.Delete(cf, key);
5686
    return Write(o, &batch);
5687
  }
A
Andres Noetzli 已提交
5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701
  using DB::SingleDelete;
  virtual Status SingleDelete(const WriteOptions& o, ColumnFamilyHandle* cf,
                              const Slice& key) override {
    WriteBatch batch;
    batch.SingleDelete(cf, key);
    return Write(o, &batch);
  }
  using DB::Merge;
  virtual Status Merge(const WriteOptions& o, ColumnFamilyHandle* cf,
                       const Slice& k, const Slice& v) override {
    WriteBatch batch;
    batch.Merge(cf, k, v);
    return Write(o, &batch);
  }
5702
  using DB::Get;
5703
  virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* cf,
I
Igor Sugak 已提交
5704
                     const Slice& key, std::string* value) override {
5705 5706 5707
    return Status::NotSupported(key);
  }

5708 5709 5710
  using DB::MultiGet;
  virtual std::vector<Status> MultiGet(
      const ReadOptions& options,
5711
      const std::vector<ColumnFamilyHandle*>& column_family,
I
Igor Sugak 已提交
5712 5713
      const std::vector<Slice>& keys,
      std::vector<std::string>* values) override {
5714 5715 5716
    std::vector<Status> s(keys.size(),
                          Status::NotSupported("Not implemented."));
    return s;
J
jorlow@chromium.org 已提交
5717
  }
5718

I
Islam AbdelRahman 已提交
5719
#ifndef ROCKSDB_LITE
5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731
  using DB::AddFile;
  virtual Status AddFile(ColumnFamilyHandle* column_family,
                         const ExternalSstFileInfo* file_path,
                         bool move_file) override {
    return Status::NotSupported("Not implemented.");
  }
  virtual Status AddFile(ColumnFamilyHandle* column_family,
                         const std::string& file_path,
                         bool move_file) override {
    return Status::NotSupported("Not implemented.");
  }

I
Igor Canadi 已提交
5732
  using DB::GetPropertiesOfAllTables;
I
Igor Sugak 已提交
5733 5734 5735
  virtual Status GetPropertiesOfAllTables(
      ColumnFamilyHandle* column_family,
      TablePropertiesCollection* props) override {
5736 5737
    return Status();
  }
5738

5739
  virtual Status GetPropertiesOfTablesInRange(
5740
      ColumnFamilyHandle* column_family, const Range* range, std::size_t n,
5741 5742 5743
      TablePropertiesCollection* props) override {
    return Status();
  }
I
Islam AbdelRahman 已提交
5744
#endif  // ROCKSDB_LITE
5745

5746
  using DB::KeyMayExist;
5747
  virtual bool KeyMayExist(const ReadOptions& options,
5748
                           ColumnFamilyHandle* column_family, const Slice& key,
I
Igor Sugak 已提交
5749 5750
                           std::string* value,
                           bool* value_found = nullptr) override {
5751 5752 5753
    if (value_found != nullptr) {
      *value_found = false;
    }
5754 5755
    return true; // Not Supported directly
  }
5756 5757
  using DB::NewIterator;
  virtual Iterator* NewIterator(const ReadOptions& options,
I
Igor Sugak 已提交
5758
                                ColumnFamilyHandle* column_family) override {
A
Abhishek Kona 已提交
5759
    if (options.snapshot == nullptr) {
J
jorlow@chromium.org 已提交
5760 5761 5762 5763 5764
      KVMap* saved = new KVMap;
      *saved = map_;
      return new ModelIter(saved, true);
    } else {
      const KVMap* snapshot_state =
5765
          &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
J
jorlow@chromium.org 已提交
5766 5767 5768
      return new ModelIter(snapshot_state, false);
    }
  }
5769 5770
  virtual Status NewIterators(
      const ReadOptions& options,
5771
      const std::vector<ColumnFamilyHandle*>& column_family,
I
Igor Sugak 已提交
5772
      std::vector<Iterator*>* iterators) override {
5773 5774
    return Status::NotSupported("Not supported yet");
  }
I
Igor Sugak 已提交
5775
  virtual const Snapshot* GetSnapshot() override {
5776 5777 5778
    ModelSnapshot* snapshot = new ModelSnapshot;
    snapshot->map_ = map_;
    return snapshot;
J
jorlow@chromium.org 已提交
5779 5780
  }

I
Igor Sugak 已提交
5781
  virtual void ReleaseSnapshot(const Snapshot* snapshot) override {
5782
    delete reinterpret_cast<const ModelSnapshot*>(snapshot);
J
jorlow@chromium.org 已提交
5783
  }
5784

I
Igor Sugak 已提交
5785 5786
  virtual Status Write(const WriteOptions& options,
                       WriteBatch* batch) override {
5787 5788 5789
    class Handler : public WriteBatch::Handler {
     public:
      KVMap* map_;
I
Igor Sugak 已提交
5790
      virtual void Put(const Slice& key, const Slice& value) override {
5791
        (*map_)[key.ToString()] = value.ToString();
J
jorlow@chromium.org 已提交
5792
      }
I
Igor Sugak 已提交
5793
      virtual void Merge(const Slice& key, const Slice& value) override {
5794 5795 5796
        // ignore merge for now
        //(*map_)[key.ToString()] = value.ToString();
      }
I
Igor Sugak 已提交
5797
      virtual void Delete(const Slice& key) override {
5798 5799 5800 5801 5802 5803
        map_->erase(key.ToString());
      }
    };
    Handler handler;
    handler.map_ = &map_;
    return batch->Iterate(&handler);
J
jorlow@chromium.org 已提交
5804 5805
  }

5806
  using DB::GetProperty;
5807
  virtual bool GetProperty(ColumnFamilyHandle* column_family,
I
Igor Sugak 已提交
5808
                           const Slice& property, std::string* value) override {
J
jorlow@chromium.org 已提交
5809 5810
    return false;
  }
5811 5812 5813 5814 5815
  using DB::GetIntProperty;
  virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                              const Slice& property, uint64_t* value) override {
    return false;
  }
5816 5817 5818 5819 5820
  using DB::GetAggregatedIntProperty;
  virtual bool GetAggregatedIntProperty(const Slice& property,
                                        uint64_t* value) override {
    return false;
  }
5821
  using DB::GetApproximateSizes;
5822
  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
5823 5824
                                   const Range* range, int n, uint64_t* sizes,
                                   bool include_memtable) override {
J
jorlow@chromium.org 已提交
5825 5826 5827 5828
    for (int i = 0; i < n; i++) {
      sizes[i] = 0;
    }
  }
5829
  using DB::CompactRange;
5830 5831 5832
  virtual Status CompactRange(const CompactRangeOptions& options,
                              ColumnFamilyHandle* column_family,
                              const Slice* start, const Slice* end) override {
L
Lei Jin 已提交
5833
    return Status::NotSupported("Not supported operation.");
G
Gabor Cselle 已提交
5834 5835
  }

5836 5837 5838 5839 5840 5841 5842 5843 5844
  using DB::CompactFiles;
  virtual Status CompactFiles(
      const CompactionOptions& compact_options,
      ColumnFamilyHandle* column_family,
      const std::vector<std::string>& input_file_names,
      const int output_level, const int output_path_id = -1) override {
    return Status::NotSupported("Not supported operation.");
  }

5845 5846 5847 5848 5849 5850 5851 5852
  Status PauseBackgroundWork() override {
    return Status::NotSupported("Not supported operation.");
  }

  Status ContinueBackgroundWork() override {
    return Status::NotSupported("Not supported operation.");
  }

5853 5854 5855 5856 5857
  Status EnableAutoCompaction(
      const std::vector<ColumnFamilyHandle*>& column_family_handles) override {
    return Status::NotSupported("Not supported operation.");
  }

5858
  using DB::NumberLevels;
I
Igor Sugak 已提交
5859 5860 5861
  virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
    return 1;
  }
5862

5863
  using DB::MaxMemCompactionLevel;
I
Igor Sugak 已提交
5864 5865
  virtual int MaxMemCompactionLevel(
      ColumnFamilyHandle* column_family) override {
5866
    return 1;
5867 5868
  }

5869
  using DB::Level0StopWriteTrigger;
I
Igor Sugak 已提交
5870 5871
  virtual int Level0StopWriteTrigger(
      ColumnFamilyHandle* column_family) override {
5872
    return -1;
5873 5874
  }

I
Igor Sugak 已提交
5875
  virtual const std::string& GetName() const override { return name_; }
I
Igor Canadi 已提交
5876

I
Igor Sugak 已提交
5877
  virtual Env* GetEnv() const override { return nullptr; }
5878

5879
  using DB::GetOptions;
I
Igor Sugak 已提交
5880 5881
  virtual const Options& GetOptions(
      ColumnFamilyHandle* column_family) const override {
I
Igor Canadi 已提交
5882 5883 5884
    return options_;
  }

5885 5886 5887
  using DB::GetDBOptions;
  virtual const DBOptions& GetDBOptions() const override { return options_; }

5888 5889
  using DB::Flush;
  virtual Status Flush(const rocksdb::FlushOptions& options,
I
Igor Sugak 已提交
5890
                       ColumnFamilyHandle* column_family) override {
H
heyongqiang 已提交
5891 5892 5893 5894
    Status ret;
    return ret;
  }

5895 5896 5897 5898
  virtual Status SyncWAL() override {
    return Status::OK();
  }

I
Islam AbdelRahman 已提交
5899
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
5900
  virtual Status DisableFileDeletions() override { return Status::OK(); }
I
Islam AbdelRahman 已提交
5901

I
Igor Sugak 已提交
5902
  virtual Status EnableFileDeletions(bool force) override {
5903 5904
    return Status::OK();
  }
5905
  virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size,
I
Igor Sugak 已提交
5906
                              bool flush_memtable = true) override {
5907 5908 5909
    return Status::OK();
  }

I
Igor Sugak 已提交
5910
  virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
5911 5912 5913
    return Status::OK();
  }

I
Igor Sugak 已提交
5914
  virtual Status DeleteFile(std::string name) override { return Status::OK(); }
5915

5916 5917 5918
  virtual Status GetUpdatesSince(
      rocksdb::SequenceNumber, unique_ptr<rocksdb::TransactionLogIterator>*,
      const TransactionLogIterator::ReadOptions&
I
Igor Sugak 已提交
5919
          read_options = TransactionLogIterator::ReadOptions()) override {
5920 5921 5922
    return Status::NotSupported("Not supported in Model DB");
  }

5923 5924
  virtual void GetColumnFamilyMetaData(
      ColumnFamilyHandle* column_family,
I
Igor Sugak 已提交
5925
      ColumnFamilyMetaData* metadata) override {}
I
Islam AbdelRahman 已提交
5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936
#endif  // ROCKSDB_LITE

  virtual Status GetDbIdentity(std::string& identity) const override {
    return Status::OK();
  }

  virtual SequenceNumber GetLatestSequenceNumber() const override { return 0; }

  virtual ColumnFamilyHandle* DefaultColumnFamily() const override {
    return nullptr;
  }
5937

J
jorlow@chromium.org 已提交
5938 5939 5940 5941 5942 5943 5944 5945 5946
 private:
  class ModelIter: public Iterator {
   public:
    ModelIter(const KVMap* map, bool owned)
        : map_(map), owned_(owned), iter_(map_->end()) {
    }
    ~ModelIter() {
      if (owned_) delete map_;
    }
I
Igor Sugak 已提交
5947 5948 5949
    virtual bool Valid() const override { return iter_ != map_->end(); }
    virtual void SeekToFirst() override { iter_ = map_->begin(); }
    virtual void SeekToLast() override {
J
jorlow@chromium.org 已提交
5950 5951 5952 5953 5954 5955
      if (map_->empty()) {
        iter_ = map_->end();
      } else {
        iter_ = map_->find(map_->rbegin()->first);
      }
    }
I
Igor Sugak 已提交
5956
    virtual void Seek(const Slice& k) override {
J
jorlow@chromium.org 已提交
5957 5958
      iter_ = map_->lower_bound(k.ToString());
    }
I
Igor Sugak 已提交
5959 5960
    virtual void Next() override { ++iter_; }
    virtual void Prev() override {
S
Stanislau Hlebik 已提交
5961 5962 5963 5964 5965 5966 5967
      if (iter_ == map_->begin()) {
        iter_ = map_->end();
        return;
      }
      --iter_;
    }

I
Igor Sugak 已提交
5968 5969 5970 5971
    virtual Slice key() const override { return iter_->first; }
    virtual Slice value() const override { return iter_->second; }
    virtual Status status() const override { return Status::OK(); }

J
jorlow@chromium.org 已提交
5972 5973 5974 5975 5976 5977 5978
   private:
    const KVMap* const map_;
    const bool owned_;  // Do we own map_
    KVMap::const_iterator iter_;
  };
  const Options options_;
  KVMap map_;
I
Igor Canadi 已提交
5979
  std::string name_ = "";
J
jorlow@chromium.org 已提交
5980 5981
};

J
Jim Paton 已提交
5982 5983 5984 5985 5986 5987 5988
static std::string RandomKey(Random* rnd, int minimum = 0) {
  int len;
  do {
    len = (rnd->OneIn(3)
           ? 1                // Short sometimes to encourage collisions
           : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
  } while (len < minimum);
J
jorlow@chromium.org 已提交
5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038
  return test::RandomKey(rnd, len);
}

static bool CompareIterators(int step,
                             DB* model,
                             DB* db,
                             const Snapshot* model_snap,
                             const Snapshot* db_snap) {
  ReadOptions options;
  options.snapshot = model_snap;
  Iterator* miter = model->NewIterator(options);
  options.snapshot = db_snap;
  Iterator* dbiter = db->NewIterator(options);
  bool ok = true;
  int count = 0;
  for (miter->SeekToFirst(), dbiter->SeekToFirst();
       ok && miter->Valid() && dbiter->Valid();
       miter->Next(), dbiter->Next()) {
    count++;
    if (miter->key().compare(dbiter->key()) != 0) {
      fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
              step,
              EscapeString(miter->key()).c_str(),
              EscapeString(dbiter->key()).c_str());
      ok = false;
      break;
    }

    if (miter->value().compare(dbiter->value()) != 0) {
      fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
              step,
              EscapeString(miter->key()).c_str(),
              EscapeString(miter->value()).c_str(),
              EscapeString(miter->value()).c_str());
      ok = false;
    }
  }

  if (ok) {
    if (miter->Valid() != dbiter->Valid()) {
      fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
              step, miter->Valid(), dbiter->Valid());
      ok = false;
    }
  }
  delete miter;
  delete dbiter;
  return ok;
}

I
Igor Sugak 已提交
6039
TEST_F(DBTest, Randomized) {
6040 6041
  anon::OptionsOverride options_override;
  options_override.skip_policy = kSkipNoSnapshot;
J
jorlow@chromium.org 已提交
6042
  Random rnd(test::RandomSeed());
S
Sanjay Ghemawat 已提交
6043
  do {
6044
    ModelDB model(CurrentOptions(options_override));
S
Sanjay Ghemawat 已提交
6045
    const int N = 10000;
A
Abhishek Kona 已提交
6046 6047
    const Snapshot* model_snap = nullptr;
    const Snapshot* db_snap = nullptr;
S
Sanjay Ghemawat 已提交
6048 6049 6050 6051
    std::string k, v;
    for (int step = 0; step < N; step++) {
      // TODO(sanjay): Test Get() works
      int p = rnd.Uniform(100);
J
Jim Paton 已提交
6052
      int minimum = 0;
6053
      if (option_config_ == kHashSkipList ||
6054
          option_config_ == kHashLinkList ||
6055
          option_config_ == kHashCuckoo ||
6056 6057 6058
          option_config_ == kPlainTableFirstBytePrefix ||
          option_config_ == kBlockBasedTableWithWholeKeyHashIndex ||
          option_config_ == kBlockBasedTableWithPrefixHashIndex) {
J
Jim Paton 已提交
6059 6060
        minimum = 1;
      }
S
Sanjay Ghemawat 已提交
6061
      if (p < 45) {                               // Put
J
Jim Paton 已提交
6062
        k = RandomKey(&rnd, minimum);
S
Sanjay Ghemawat 已提交
6063 6064 6065 6066 6067 6068 6069 6070
        v = RandomString(&rnd,
                         rnd.OneIn(20)
                         ? 100 + rnd.Uniform(100)
                         : rnd.Uniform(8));
        ASSERT_OK(model.Put(WriteOptions(), k, v));
        ASSERT_OK(db_->Put(WriteOptions(), k, v));

      } else if (p < 90) {                        // Delete
J
Jim Paton 已提交
6071
        k = RandomKey(&rnd, minimum);
S
Sanjay Ghemawat 已提交
6072 6073 6074 6075 6076 6077 6078 6079 6080
        ASSERT_OK(model.Delete(WriteOptions(), k));
        ASSERT_OK(db_->Delete(WriteOptions(), k));


      } else {                                    // Multi-element batch
        WriteBatch b;
        const int num = rnd.Uniform(8);
        for (int i = 0; i < num; i++) {
          if (i == 0 || !rnd.OneIn(10)) {
J
Jim Paton 已提交
6081
            k = RandomKey(&rnd, minimum);
S
Sanjay Ghemawat 已提交
6082 6083 6084 6085 6086 6087 6088 6089 6090 6091
          } else {
            // Periodically re-use the same key from the previous iter, so
            // we have multiple entries in the write batch for the same key
          }
          if (rnd.OneIn(2)) {
            v = RandomString(&rnd, rnd.Uniform(10));
            b.Put(k, v);
          } else {
            b.Delete(k);
          }
J
jorlow@chromium.org 已提交
6092
        }
S
Sanjay Ghemawat 已提交
6093 6094
        ASSERT_OK(model.Write(WriteOptions(), &b));
        ASSERT_OK(db_->Write(WriteOptions(), &b));
J
jorlow@chromium.org 已提交
6095 6096
      }

S
Sanjay Ghemawat 已提交
6097
      if ((step % 100) == 0) {
S
Stanislau Hlebik 已提交
6098 6099 6100 6101 6102 6103 6104 6105
        // For DB instances that use the hash index + block-based table, the
        // iterator will be invalid right when seeking a non-existent key, right
        // than return a key that is close to it.
        if (option_config_ != kBlockBasedTableWithWholeKeyHashIndex &&
            option_config_ != kBlockBasedTableWithPrefixHashIndex) {
          ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
          ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
        }
6106

S
Sanjay Ghemawat 已提交
6107 6108 6109
        // Save a snapshot from each DB this time that we'll use next
        // time we compare things, to make sure the current state is
        // preserved with the snapshot
A
Abhishek Kona 已提交
6110 6111
        if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
        if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
J
jorlow@chromium.org 已提交
6112

L
Lei Jin 已提交
6113

6114
        auto options = CurrentOptions(options_override);
L
Lei Jin 已提交
6115
        Reopen(options);
A
Abhishek Kona 已提交
6116
        ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
J
jorlow@chromium.org 已提交
6117

S
Sanjay Ghemawat 已提交
6118 6119 6120
        model_snap = model.GetSnapshot();
        db_snap = db_->GetSnapshot();
      }
S
Stanislau Hlebik 已提交
6121 6122

      if ((step % 2000) == 0) {
6123
        fprintf(stderr,
S
Stanislau Hlebik 已提交
6124 6125 6126
                "DBTest.Randomized, option ID: %d, step: %d out of %d\n",
                option_config_, step, N);
      }
J
jorlow@chromium.org 已提交
6127
    }
A
Abhishek Kona 已提交
6128 6129
    if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
    if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
6130
    // skip cuckoo hash as it does not support snapshot.
S
Stanislau Hlebik 已提交
6131 6132
  } while (ChangeOptions(kSkipDeletesFilterFirst | kSkipNoSeekToLast |
                         kSkipHashCuckoo));
J
jorlow@chromium.org 已提交
6133 6134
}

I
Igor Sugak 已提交
6135
TEST_F(DBTest, MultiGetSimple) {
6136
  do {
L
Lei Jin 已提交
6137
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152
    ASSERT_OK(Put(1, "k1", "v1"));
    ASSERT_OK(Put(1, "k2", "v2"));
    ASSERT_OK(Put(1, "k3", "v3"));
    ASSERT_OK(Put(1, "k4", "v4"));
    ASSERT_OK(Delete(1, "k4"));
    ASSERT_OK(Put(1, "k5", "v5"));
    ASSERT_OK(Delete(1, "no_key"));

    std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});

    std::vector<std::string> values(20, "Temporary data to be overwritten");
    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);

    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
    ASSERT_EQ(values.size(), keys.size());
6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164
    ASSERT_EQ(values[0], "v1");
    ASSERT_EQ(values[1], "v2");
    ASSERT_EQ(values[2], "v3");
    ASSERT_EQ(values[4], "v5");

    ASSERT_OK(s[0]);
    ASSERT_OK(s[1]);
    ASSERT_OK(s[2]);
    ASSERT_TRUE(s[3].IsNotFound());
    ASSERT_OK(s[4]);
    ASSERT_TRUE(s[5].IsNotFound());
  } while (ChangeCompactOptions());
6165 6166
}

I
Igor Sugak 已提交
6167
TEST_F(DBTest, MultiGetEmpty) {
6168
  do {
L
Lei Jin 已提交
6169
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
6170 6171 6172
    // Empty Key Set
    std::vector<Slice> keys;
    std::vector<std::string> values;
6173 6174 6175
    std::vector<ColumnFamilyHandle*> cfs;
    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
    ASSERT_EQ(s.size(), 0U);
6176 6177

    // Empty Database, Empty Key Set
L
Lei Jin 已提交
6178 6179 6180
    Options options = CurrentOptions();
    options.create_if_missing = true;
    DestroyAndReopen(options);
L
Lei Jin 已提交
6181
    CreateAndReopenWithCF({"pikachu"}, options);
6182 6183
    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
    ASSERT_EQ(s.size(), 0U);
6184 6185 6186 6187 6188

    // Empty Database, Search for Keys
    keys.resize(2);
    keys[0] = "a";
    keys[1] = "b";
6189 6190 6191
    cfs.push_back(handles_[0]);
    cfs.push_back(handles_[1]);
    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
6192 6193 6194
    ASSERT_EQ((int)s.size(), 2);
    ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
  } while (ChangeCompactOptions());
6195 6196
}

I
Islam AbdelRahman 已提交
6197
#ifndef ROCKSDB_LITE
I
Igor Canadi 已提交
6198
namespace {
T
Tyler Harter 已提交
6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220
void PrefixScanInit(DBTest *dbtest) {
  char buf[100];
  std::string keystr;
  const int small_range_sstfiles = 5;
  const int big_range_sstfiles = 5;

  // Generate 11 sst files with the following prefix ranges.
  // GROUP 0: [0,10]                              (level 1)
  // GROUP 1: [1,2], [2,3], [3,4], [4,5], [5, 6]  (level 0)
  // GROUP 2: [0,6], [0,7], [0,8], [0,9], [0,10]  (level 0)
  //
  // A seek with the previous API would do 11 random I/Os (to all the
  // files).  With the new API and a prefix filter enabled, we should
  // only do 2 random I/O, to the 2 files containing the key.

  // GROUP 0
  snprintf(buf, sizeof(buf), "%02d______:start", 0);
  keystr = std::string(buf);
  ASSERT_OK(dbtest->Put(keystr, keystr));
  snprintf(buf, sizeof(buf), "%02d______:end", 10);
  keystr = std::string(buf);
  ASSERT_OK(dbtest->Put(keystr, keystr));
6221
  dbtest->Flush();
6222 6223
  dbtest->dbfull()->CompactRange(CompactRangeOptions(), nullptr,
                                 nullptr);  // move to level 1
T
Tyler Harter 已提交
6224 6225 6226 6227 6228 6229 6230 6231 6232

  // GROUP 1
  for (int i = 1; i <= small_range_sstfiles; i++) {
    snprintf(buf, sizeof(buf), "%02d______:start", i);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
    snprintf(buf, sizeof(buf), "%02d______:end", i+1);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
6233
    dbtest->Flush();
T
Tyler Harter 已提交
6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244
  }

  // GROUP 2
  for (int i = 1; i <= big_range_sstfiles; i++) {
    snprintf(buf, sizeof(buf), "%02d______:start", 0);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
    snprintf(buf, sizeof(buf), "%02d______:end",
             small_range_sstfiles+i+1);
    keystr = std::string(buf);
    ASSERT_OK(dbtest->Put(keystr, keystr));
6245
    dbtest->Flush();
T
Tyler Harter 已提交
6246 6247
  }
}
I
Igor Canadi 已提交
6248
}  // namespace
T
Tyler Harter 已提交
6249

I
Igor Sugak 已提交
6250
TEST_F(DBTest, PrefixScan) {
V
Venkatesh Radhakrishnan 已提交
6251 6252
  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
             kSkipNoPrefix);
6253 6254 6255 6256 6257 6258 6259 6260 6261
  while (ChangeFilterOptions()) {
    int count;
    Slice prefix;
    Slice key;
    char buf[100];
    Iterator* iter;
    snprintf(buf, sizeof(buf), "03______:");
    prefix = Slice(buf, 8);
    key = Slice(buf, 9);
6262 6263
    ASSERT_EQ(key.difference_offset(prefix), 8);
    ASSERT_EQ(prefix.difference_offset(key), 8);
6264 6265 6266 6267 6268 6269 6270 6271 6272
    // db configs
    env_->count_random_reads_ = true;
    Options options = CurrentOptions();
    options.env = env_;
    options.prefix_extractor.reset(NewFixedPrefixTransform(8));
    options.disable_auto_compactions = true;
    options.max_background_compactions = 2;
    options.create_if_missing = true;
    options.memtable_factory.reset(NewHashSkipListRepFactory(16));
T
Tyler Harter 已提交
6273

6274 6275 6276 6277 6278
    BlockBasedTableOptions table_options;
    table_options.no_block_cache = true;
    table_options.filter_policy.reset(NewBloomFilterPolicy(10));
    table_options.whole_key_filtering = false;
    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
6279

6280
    // 11 RAND I/Os
L
Lei Jin 已提交
6281
    DestroyAndReopen(options);
6282 6283 6284 6285 6286 6287 6288 6289 6290
    PrefixScanInit(this);
    count = 0;
    env_->random_read_counter_.Reset();
    iter = db_->NewIterator(ReadOptions());
    for (iter->Seek(prefix); iter->Valid(); iter->Next()) {
      if (! iter->key().starts_with(prefix)) {
        break;
      }
      count++;
T
Tyler Harter 已提交
6291
    }
6292 6293 6294 6295 6296 6297
    ASSERT_OK(iter->status());
    delete iter;
    ASSERT_EQ(count, 2);
    ASSERT_EQ(env_->random_read_counter_.Read(), 2);
    Close();
  }  // end of while
V
Venkatesh Radhakrishnan 已提交
6298
  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
T
Tyler Harter 已提交
6299
}
I
Islam AbdelRahman 已提交
6300
#endif  // ROCKSDB_LITE
T
Tyler Harter 已提交
6301

I
Igor Sugak 已提交
6302
TEST_F(DBTest, BlockBasedTablePrefixIndexTest) {
6303 6304 6305 6306 6307 6308 6309 6310
  // create a DB with block prefix index
  BlockBasedTableOptions table_options;
  Options options = CurrentOptions();
  table_options.index_type = BlockBasedTableOptions::kHashSearch;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  options.prefix_extractor.reset(NewFixedPrefixTransform(1));


L
Lei Jin 已提交
6311
  Reopen(options);
6312 6313 6314 6315 6316 6317 6318 6319 6320 6321
  ASSERT_OK(Put("k1", "v1"));
  Flush();
  ASSERT_OK(Put("k2", "v2"));

  // Reopen it without prefix extractor, make sure everything still works.
  // RocksDB should just fall back to the binary index.
  table_options.index_type = BlockBasedTableOptions::kBinarySearch;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
  options.prefix_extractor.reset();

L
Lei Jin 已提交
6322
  Reopen(options);
6323 6324 6325 6326
  ASSERT_EQ("v1", Get("k1"));
  ASSERT_EQ("v2", Get("k2"));
}

I
Igor Sugak 已提交
6327
TEST_F(DBTest, ChecksumTest) {
I
xxHash  
Igor Canadi 已提交
6328 6329 6330 6331 6332
  BlockBasedTableOptions table_options;
  Options options = CurrentOptions();

  table_options.checksum = kCRC32c;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
6333
  Reopen(options);
I
xxHash  
Igor Canadi 已提交
6334 6335 6336 6337 6338 6339
  ASSERT_OK(Put("a", "b"));
  ASSERT_OK(Put("c", "d"));
  ASSERT_OK(Flush());  // table with crc checksum

  table_options.checksum = kxxHash;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
6340
  Reopen(options);
I
xxHash  
Igor Canadi 已提交
6341 6342 6343 6344 6345 6346
  ASSERT_OK(Put("e", "f"));
  ASSERT_OK(Put("g", "h"));
  ASSERT_OK(Flush());  // table with xxhash checksum

  table_options.checksum = kCRC32c;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
6347
  Reopen(options);
I
xxHash  
Igor Canadi 已提交
6348 6349 6350 6351
  ASSERT_EQ("b", Get("a"));
  ASSERT_EQ("d", Get("c"));
  ASSERT_EQ("f", Get("e"));
  ASSERT_EQ("h", Get("g"));
T
Tomislav Novak 已提交
6352

I
xxHash  
Igor Canadi 已提交
6353 6354
  table_options.checksum = kCRC32c;
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
L
Lei Jin 已提交
6355
  Reopen(options);
I
xxHash  
Igor Canadi 已提交
6356 6357 6358 6359 6360
  ASSERT_EQ("b", Get("a"));
  ASSERT_EQ("d", Get("c"));
  ASSERT_EQ("f", Get("e"));
  ASSERT_EQ("h", Get("g"));
}
I
Igor Canadi 已提交
6361

I
Islam AbdelRahman 已提交
6362
#ifndef ROCKSDB_LITE
6363
TEST_P(DBTestWithParam, FIFOCompactionTest) {
I
Igor Canadi 已提交
6364 6365 6366 6367 6368 6369
  for (int iter = 0; iter < 2; ++iter) {
    // first iteration -- auto compaction
    // second iteration -- manual compaction
    Options options;
    options.compaction_style = kCompactionStyleFIFO;
    options.write_buffer_size = 100 << 10;                             // 100KB
6370
    options.arena_block_size = 4096;
I
Igor Canadi 已提交
6371 6372 6373
    options.compaction_options_fifo.max_table_files_size = 500 << 10;  // 500KB
    options.compression = kNoCompression;
    options.create_if_missing = true;
6374
    options.max_subcompactions = max_subcompactions_;
I
Igor Canadi 已提交
6375 6376 6377
    if (iter == 1) {
      options.disable_auto_compactions = true;
    }
6378
    options = CurrentOptions(options);
L
Lei Jin 已提交
6379
    DestroyAndReopen(options);
I
Igor Canadi 已提交
6380 6381 6382

    Random rnd(301);
    for (int i = 0; i < 6; ++i) {
6383 6384
      for (int j = 0; j < 110; ++j) {
        ASSERT_OK(Put(ToString(i * 100 + j), RandomString(&rnd, 980)));
I
Igor Canadi 已提交
6385 6386
      }
      // flush should happen here
6387
      ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
I
Igor Canadi 已提交
6388 6389 6390 6391
    }
    if (iter == 0) {
      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    } else {
6392
      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
I
Igor Canadi 已提交
6393 6394 6395 6396 6397
    }
    // only 5 files should survive
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
    for (int i = 0; i < 50; ++i) {
      // these keys should be deleted in previous compaction
6398
      ASSERT_EQ("NOT_FOUND", Get(ToString(i)));
I
Igor Canadi 已提交
6399 6400 6401
    }
  }
}
I
Islam AbdelRahman 已提交
6402
#endif  // ROCKSDB_LITE
6403

6404
// verify that we correctly deprecated timeout_hint_us
I
Igor Sugak 已提交
6405
TEST_F(DBTest, SimpleWriteTimeoutTest) {
L
Lei Jin 已提交
6406
  WriteOptions write_opt;
6407
  write_opt.timeout_hint_us = 0;
6408 6409 6410
  ASSERT_OK(Put(Key(1), Key(1) + std::string(100, 'v'), write_opt));
  write_opt.timeout_hint_us = 10;
  ASSERT_NOK(Put(Key(1), Key(1) + std::string(100, 'v'), write_opt));
6411 6412
}

I
Islam AbdelRahman 已提交
6413
#ifndef ROCKSDB_LITE
L
Lei Jin 已提交
6414 6415
/*
 * This test is not reliable enough as it heavily depends on disk behavior.
L
Lei Jin 已提交
6416
 */
I
Igor Sugak 已提交
6417
TEST_F(DBTest, RateLimitingTest) {
L
Lei Jin 已提交
6418
  Options options = CurrentOptions();
L
Lei Jin 已提交
6419 6420 6421 6422 6423
  options.write_buffer_size = 1 << 20;         // 1MB
  options.level0_file_num_compaction_trigger = 2;
  options.target_file_size_base = 1 << 20;     // 1MB
  options.max_bytes_for_level_base = 4 << 20;  // 4MB
  options.max_bytes_for_level_multiplier = 4;
L
Lei Jin 已提交
6424 6425 6426
  options.compression = kNoCompression;
  options.create_if_missing = true;
  options.env = env_;
L
Lei Jin 已提交
6427
  options.IncreaseParallelism(4);
L
Lei Jin 已提交
6428
  DestroyAndReopen(options);
L
Lei Jin 已提交
6429

L
Lei Jin 已提交
6430 6431 6432
  WriteOptions wo;
  wo.disableWAL = true;

L
Lei Jin 已提交
6433 6434 6435
  // # no rate limiting
  Random rnd(301);
  uint64_t start = env_->NowMicros();
L
Lei Jin 已提交
6436 6437 6438 6439
  // Write ~96M data
  for (int64_t i = 0; i < (96 << 10); ++i) {
    ASSERT_OK(Put(RandomString(&rnd, 32),
                  RandomString(&rnd, (1 << 10) + 1), wo));
L
Lei Jin 已提交
6440 6441 6442 6443 6444 6445 6446
  }
  uint64_t elapsed = env_->NowMicros() - start;
  double raw_rate = env_->bytes_written_ * 1000000 / elapsed;
  Close();

  // # rate limiting with 0.7 x threshold
  options.rate_limiter.reset(
L
Lei Jin 已提交
6447
    NewGenericRateLimiter(static_cast<int64_t>(0.7 * raw_rate)));
L
Lei Jin 已提交
6448
  env_->bytes_written_ = 0;
L
Lei Jin 已提交
6449
  DestroyAndReopen(options);
L
Lei Jin 已提交
6450 6451

  start = env_->NowMicros();
L
Lei Jin 已提交
6452 6453 6454 6455
  // Write ~96M data
  for (int64_t i = 0; i < (96 << 10); ++i) {
    ASSERT_OK(Put(RandomString(&rnd, 32),
                  RandomString(&rnd, (1 << 10) + 1), wo));
L
Lei Jin 已提交
6456 6457 6458
  }
  elapsed = env_->NowMicros() - start;
  Close();
6459
  ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_);
L
Lei Jin 已提交
6460 6461
  double ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate;
  fprintf(stderr, "write rate ratio = %.2lf, expected 0.7\n", ratio);
L
Lei Jin 已提交
6462
  ASSERT_TRUE(ratio < 0.8);
L
Lei Jin 已提交
6463 6464 6465

  // # rate limiting with half of the raw_rate
  options.rate_limiter.reset(
L
Lei Jin 已提交
6466
    NewGenericRateLimiter(static_cast<int64_t>(raw_rate / 2)));
L
Lei Jin 已提交
6467
  env_->bytes_written_ = 0;
L
Lei Jin 已提交
6468
  DestroyAndReopen(options);
L
Lei Jin 已提交
6469 6470

  start = env_->NowMicros();
L
Lei Jin 已提交
6471 6472 6473 6474
  // Write ~96M data
  for (int64_t i = 0; i < (96 << 10); ++i) {
    ASSERT_OK(Put(RandomString(&rnd, 32),
                  RandomString(&rnd, (1 << 10) + 1), wo));
L
Lei Jin 已提交
6475 6476 6477
  }
  elapsed = env_->NowMicros() - start;
  Close();
6478
  ASSERT_EQ(options.rate_limiter->GetTotalBytesThrough(), env_->bytes_written_);
L
Lei Jin 已提交
6479 6480
  ratio = env_->bytes_written_ * 1000000 / elapsed / raw_rate;
  fprintf(stderr, "write rate ratio = %.2lf, expected 0.5\n", ratio);
6481
  ASSERT_LT(ratio, 0.6);
L
Lei Jin 已提交
6482 6483
}

I
Igor Sugak 已提交
6484
TEST_F(DBTest, TableOptionsSanitizeTest) {
6485 6486
  Options options = CurrentOptions();
  options.create_if_missing = true;
L
Lei Jin 已提交
6487
  DestroyAndReopen(options);
6488 6489 6490 6491
  ASSERT_EQ(db_->GetOptions().allow_mmap_reads, false);

  options.table_factory.reset(new PlainTableFactory());
  options.prefix_extractor.reset(NewNoopTransform());
L
Lei Jin 已提交
6492
  Destroy(options);
6493
  ASSERT_TRUE(!TryReopen(options).IsNotSupported());
6494 6495 6496 6497 6498

  // Test for check of prefix_extractor when hash index is used for
  // block-based table
  BlockBasedTableOptions to;
  to.index_type = BlockBasedTableOptions::kHashSearch;
6499
  options = CurrentOptions();
6500 6501
  options.create_if_missing = true;
  options.table_factory.reset(NewBlockBasedTableFactory(to));
L
Lei Jin 已提交
6502
  ASSERT_TRUE(TryReopen(options).IsInvalidArgument());
6503
  options.prefix_extractor.reset(NewFixedPrefixTransform(1));
L
Lei Jin 已提交
6504
  ASSERT_OK(TryReopen(options));
6505
}
I
Islam AbdelRahman 已提交
6506
#endif  // ROCKSDB_LITE
6507

I
Igor Sugak 已提交
6508
TEST_F(DBTest, SanitizeNumThreads) {
6509 6510
  for (int attempt = 0; attempt < 2; attempt++) {
    const size_t kTotalTasks = 8;
6511
    test::SleepingBackgroundTask sleeping_tasks[kTotalTasks];
6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522

    Options options = CurrentOptions();
    if (attempt == 0) {
      options.max_background_compactions = 3;
      options.max_background_flushes = 2;
    }
    options.create_if_missing = true;
    DestroyAndReopen(options);

    for (size_t i = 0; i < kTotalTasks; i++) {
      // Insert 5 tasks to low priority queue and 5 tasks to high priority queue
6523 6524
      env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
                     &sleeping_tasks[i],
6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547
                     (i < 4) ? Env::Priority::LOW : Env::Priority::HIGH);
    }

    // Wait 100 milliseconds for they are scheduled.
    env_->SleepForMicroseconds(100000);

    // pool size 3, total task 4. Queue size should be 1.
    ASSERT_EQ(1U, options.env->GetThreadPoolQueueLen(Env::Priority::LOW));
    // pool size 2, total task 4. Queue size should be 2.
    ASSERT_EQ(2U, options.env->GetThreadPoolQueueLen(Env::Priority::HIGH));

    for (size_t i = 0; i < kTotalTasks; i++) {
      sleeping_tasks[i].WakeUp();
      sleeping_tasks[i].WaitUntilDone();
    }

    ASSERT_OK(Put("abc", "def"));
    ASSERT_EQ("def", Get("abc"));
    Flush();
    ASSERT_EQ("def", Get("abc"));
  }
}

I
Igor Sugak 已提交
6548
TEST_F(DBTest, DBIteratorBoundTest) {
6549
  Options options = CurrentOptions();
6550 6551 6552 6553
  options.env = env_;
  options.create_if_missing = true;

  options.prefix_extractor = nullptr;
L
Lei Jin 已提交
6554
  DestroyAndReopen(options);
6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585
  ASSERT_OK(Put("a", "0"));
  ASSERT_OK(Put("foo", "bar"));
  ASSERT_OK(Put("foo1", "bar1"));
  ASSERT_OK(Put("g1", "0"));

  // testing basic case with no iterate_upper_bound and no prefix_extractor
  {
    ReadOptions ro;
    ro.iterate_upper_bound = nullptr;

    std::unique_ptr<Iterator> iter(db_->NewIterator(ro));

    iter->Seek("foo");

    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("foo")), 0);

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("foo1")), 0);

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("g1")), 0);
  }

  // testing iterate_upper_bound and forward iterator
  // to make sure it stops at bound
  {
    ReadOptions ro;
    // iterate_upper_bound points beyond the last expected entry
6586 6587
    Slice prefix("foo2");
    ro.iterate_upper_bound = &prefix;
6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603

    std::unique_ptr<Iterator> iter(db_->NewIterator(ro));

    iter->Seek("foo");

    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("foo")), 0);

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(("foo1")), 0);

    iter->Next();
    // should stop here...
    ASSERT_TRUE(!iter->Valid());
  }
6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616
  // Testing SeekToLast with iterate_upper_bound set
  {
    ReadOptions ro;

    Slice prefix("foo");
    ro.iterate_upper_bound = &prefix;

    std::unique_ptr<Iterator> iter(db_->NewIterator(ro));

    iter->SeekToLast();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("a")), 0);
  }
6617 6618 6619 6620

  // prefix is the first letter of the key
  options.prefix_extractor.reset(NewFixedPrefixTransform(1));

L
Lei Jin 已提交
6621
  DestroyAndReopen(options);
6622 6623 6624 6625 6626 6627 6628 6629 6630 6631
  ASSERT_OK(Put("a", "0"));
  ASSERT_OK(Put("foo", "bar"));
  ASSERT_OK(Put("foo1", "bar1"));
  ASSERT_OK(Put("g1", "0"));

  // testing with iterate_upper_bound and prefix_extractor
  // Seek target and iterate_upper_bound are not is same prefix
  // This should be an error
  {
    ReadOptions ro;
6632 6633
    Slice upper_bound("g");
    ro.iterate_upper_bound = &upper_bound;
6634 6635 6636 6637 6638

    std::unique_ptr<Iterator> iter(db_->NewIterator(ro));

    iter->Seek("foo");

6639 6640 6641 6642 6643 6644 6645 6646
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ("foo", iter->key().ToString());

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ("foo1", iter->key().ToString());

    iter->Next();
6647 6648 6649 6650 6651 6652 6653
    ASSERT_TRUE(!iter->Valid());
  }

  // testing that iterate_upper_bound prevents iterating over deleted items
  // if the bound has already reached
  {
    options.prefix_extractor = nullptr;
L
Lei Jin 已提交
6654
    DestroyAndReopen(options);
6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684
    ASSERT_OK(Put("a", "0"));
    ASSERT_OK(Put("b", "0"));
    ASSERT_OK(Put("b1", "0"));
    ASSERT_OK(Put("c", "0"));
    ASSERT_OK(Put("d", "0"));
    ASSERT_OK(Put("e", "0"));
    ASSERT_OK(Delete("c"));
    ASSERT_OK(Delete("d"));

    // base case with no bound
    ReadOptions ro;
    ro.iterate_upper_bound = nullptr;

    std::unique_ptr<Iterator> iter(db_->NewIterator(ro));

    iter->Seek("b");
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("b")), 0);

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(("b1")), 0);

    perf_context.Reset();
    iter->Next();

    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 2);

    // now testing with iterate_bound
6685 6686
    Slice prefix("c");
    ro.iterate_upper_bound = &prefix;
6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708

    iter.reset(db_->NewIterator(ro));

    perf_context.Reset();

    iter->Seek("b");
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Slice("b")), 0);

    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(("b1")), 0);

    iter->Next();
    // the iteration should stop as soon as the the bound key is reached
    // even though the key is deleted
    // hence internal_delete_skipped_count should be 0
    ASSERT_TRUE(!iter->Valid());
    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 0);
  }
}

I
Igor Sugak 已提交
6709
TEST_F(DBTest, WriteSingleThreadEntry) {
6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726
  std::vector<std::thread> threads;
  dbfull()->TEST_LockMutex();
  auto w = dbfull()->TEST_BeginWrite();
  threads.emplace_back([&] { Put("a", "b"); });
  env_->SleepForMicroseconds(10000);
  threads.emplace_back([&] { Flush(); });
  env_->SleepForMicroseconds(10000);
  dbfull()->TEST_UnlockMutex();
  dbfull()->TEST_LockMutex();
  dbfull()->TEST_EndWrite(w);
  dbfull()->TEST_UnlockMutex();

  for (auto& t : threads) {
    t.join();
  }
}

I
Igor Sugak 已提交
6727
TEST_F(DBTest, DisableDataSyncTest) {
S
sdong 已提交
6728
  env_->sync_counter_.store(0);
6729 6730 6731 6732 6733 6734
  // iter 0 -- no sync
  // iter 1 -- sync
  for (int iter = 0; iter < 2; ++iter) {
    Options options = CurrentOptions();
    options.disableDataSync = iter == 0;
    options.create_if_missing = true;
6735
    options.num_levels = 10;
6736
    options.env = env_;
L
Lei Jin 已提交
6737
    Reopen(options);
L
Lei Jin 已提交
6738
    CreateAndReopenWithCF({"pikachu"}, options);
6739 6740 6741 6742 6743 6744 6745 6746 6747

    MakeTables(10, "a", "z");
    Compact("a", "z");

    if (iter == 0) {
      ASSERT_EQ(env_->sync_counter_.load(), 0);
    } else {
      ASSERT_GT(env_->sync_counter_.load(), 0);
    }
L
Lei Jin 已提交
6748
    Destroy(options);
6749 6750
  }
}
6751

I
Islam AbdelRahman 已提交
6752
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
6753
TEST_F(DBTest, DynamicMemtableOptions) {
L
Lei Jin 已提交
6754 6755 6756
  const uint64_t k64KB = 1 << 16;
  const uint64_t k128KB = 1 << 17;
  const uint64_t k5KB = 5 * 1024;
6757
  const int kNumPutsBeforeWaitForFlush = 64;
L
Lei Jin 已提交
6758 6759 6760 6761
  Options options;
  options.env = env_;
  options.create_if_missing = true;
  options.compression = kNoCompression;
6762
  options.max_background_compactions = 1;
L
Lei Jin 已提交
6763
  options.write_buffer_size = k64KB;
6764
  options.arena_block_size = 16 * 1024;
L
Lei Jin 已提交
6765 6766 6767 6768 6769
  options.max_write_buffer_number = 2;
  // Don't trigger compact/slowdown/stop
  options.level0_file_num_compaction_trigger = 1024;
  options.level0_slowdown_writes_trigger = 1024;
  options.level0_stop_writes_trigger = 1024;
L
Lei Jin 已提交
6770
  DestroyAndReopen(options);
L
Lei Jin 已提交
6771

6772
  auto gen_l0_kb = [this, kNumPutsBeforeWaitForFlush](int size) {
L
Lei Jin 已提交
6773 6774
    Random rnd(301);
    for (int i = 0; i < size; i++) {
L
Lei Jin 已提交
6775
      ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
6776 6777 6778 6779 6780 6781 6782 6783 6784

      // The following condition prevents a race condition between flush jobs
      // acquiring work and this thread filling up multiple memtables. Without
      // this, the flush might produce less files than expected because
      // multiple memtables are flushed into a single L0 file. This race
      // condition affects assertion (A).
      if (i % kNumPutsBeforeWaitForFlush == kNumPutsBeforeWaitForFlush - 1) {
        dbfull()->TEST_WaitForFlushMemTable();
      }
L
Lei Jin 已提交
6785 6786 6787 6788
    }
    dbfull()->TEST_WaitForFlushMemTable();
  };

L
Lei Jin 已提交
6789
  // Test write_buffer_size
L
Lei Jin 已提交
6790 6791
  gen_l0_kb(64);
  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
6792
  ASSERT_LT(SizeAtLevel(0), k64KB + k5KB);
6793
  ASSERT_GT(SizeAtLevel(0), k64KB - k5KB * 2);
L
Lei Jin 已提交
6794 6795

  // Clean up L0
6796
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
6797 6798 6799
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

  // Increase buffer size
6800
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
6801 6802 6803 6804 6805 6806 6807
    {"write_buffer_size", "131072"},
  }));

  // The existing memtable is still 64KB in size, after it becomes immutable,
  // the next memtable will be 128KB in size. Write 256KB total, we should
  // have a 64KB L0 file, a 128KB L0 file, and a memtable with 64KB data
  gen_l0_kb(256);
6808
  ASSERT_EQ(NumTableFilesAtLevel(0), 2);  // (A)
6809
  ASSERT_LT(SizeAtLevel(0), k128KB + k64KB + 2 * k5KB);
6810
  ASSERT_GT(SizeAtLevel(0), k128KB + k64KB - 4 * k5KB);
L
Lei Jin 已提交
6811 6812 6813 6814 6815 6816

  // Test max_write_buffer_number
  // Block compaction thread, which will also block the flushes because
  // max_background_flushes == 0, so flushes are getting executed by the
  // compaction thread
  env_->SetBackgroundThreads(1, Env::LOW);
6817 6818
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
L
Lei Jin 已提交
6819 6820 6821 6822 6823
                 Env::Priority::LOW);
  // Start from scratch and disable compaction/flush. Flush can only happen
  // during compaction but trigger is pretty high
  options.max_background_flushes = 0;
  options.disable_auto_compactions = true;
L
Lei Jin 已提交
6824
  DestroyAndReopen(options);
L
Lei Jin 已提交
6825

6826 6827
  // Put until writes are stopped, bounded by 256 puts. We should see stop at
  // ~128KB
L
Lei Jin 已提交
6828 6829
  int count = 0;
  Random rnd(301);
S
sdong 已提交
6830 6831

  rocksdb::SyncPoint::GetInstance()->SetCallBack(
6832 6833
      "DBImpl::DelayWrite:Wait",
      [&](void* arg) { sleeping_task_low.WakeUp(); });
S
sdong 已提交
6834
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
L
Lei Jin 已提交
6835

6836 6837
  while (!sleeping_task_low.WokenUp() && count < 256) {
    ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024), WriteOptions()));
L
Lei Jin 已提交
6838 6839
    count++;
  }
6840 6841
  ASSERT_GT(static_cast<double>(count), 128 * 0.8);
  ASSERT_LT(static_cast<double>(count), 128 * 1.2);
L
Lei Jin 已提交
6842

6843
  sleeping_task_low.WaitUntilDone();
L
Lei Jin 已提交
6844 6845

  // Increase
6846
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
6847 6848 6849
    {"max_write_buffer_number", "8"},
  }));
  // Clean up memtable and L0
6850
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
6851

6852
  sleeping_task_low.Reset();
6853
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
L
Lei Jin 已提交
6854 6855
                 Env::Priority::LOW);
  count = 0;
6856 6857
  while (!sleeping_task_low.WokenUp() && count < 1024) {
    ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024), WriteOptions()));
L
Lei Jin 已提交
6858 6859
    count++;
  }
6860 6861
  // Windows fails this test. Will tune in the future and figure out
  // approp number
D
Dmitri Smirnov 已提交
6862
#ifndef OS_WIN
6863 6864
  ASSERT_GT(static_cast<double>(count), 512 * 0.8);
  ASSERT_LT(static_cast<double>(count), 512 * 1.2);
D
Dmitri Smirnov 已提交
6865
#endif
6866
  sleeping_task_low.WaitUntilDone();
L
Lei Jin 已提交
6867 6868

  // Decrease
6869
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
6870 6871 6872
    {"max_write_buffer_number", "4"},
  }));
  // Clean up memtable and L0
6873
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
6874

6875
  sleeping_task_low.Reset();
6876
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
L
Lei Jin 已提交
6877
                 Env::Priority::LOW);
S
sdong 已提交
6878

L
Lei Jin 已提交
6879
  count = 0;
6880 6881
  while (!sleeping_task_low.WokenUp() && count < 1024) {
    ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024), WriteOptions()));
L
Lei Jin 已提交
6882 6883
    count++;
  }
6884 6885
  // Windows fails this test. Will tune in the future and figure out
  // approp number
D
Dmitri Smirnov 已提交
6886
#ifndef OS_WIN
6887 6888
  ASSERT_GT(static_cast<double>(count), 256 * 0.8);
  ASSERT_LT(static_cast<double>(count), 266 * 1.2);
D
Dmitri Smirnov 已提交
6889
#endif
6890
  sleeping_task_low.WaitUntilDone();
S
sdong 已提交
6891 6892

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
L
Lei Jin 已提交
6893
}
I
Islam AbdelRahman 已提交
6894
#endif  // ROCKSDB_LITE
L
Lei Jin 已提交
6895

Y
Yueh-Hsuan Chiang 已提交
6896
#if ROCKSDB_USING_THREAD_STATUS
6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911
namespace {
void VerifyOperationCount(Env* env, ThreadStatus::OperationType op_type,
                          int expected_count) {
  int op_count = 0;
  std::vector<ThreadStatus> thread_list;
  ASSERT_OK(env->GetThreadList(&thread_list));
  for (auto thread : thread_list) {
    if (thread.operation_type == op_type) {
      op_count++;
    }
  }
  ASSERT_EQ(op_count, expected_count);
}
}  // namespace

I
Igor Sugak 已提交
6912
TEST_F(DBTest, GetThreadStatus) {
Y
Yueh-Hsuan Chiang 已提交
6913 6914
  Options options;
  options.env = env_;
6915 6916
  options.enable_thread_tracking = true;
  TryReopen(options);
Y
Yueh-Hsuan Chiang 已提交
6917 6918

  std::vector<ThreadStatus> thread_list;
6919
  Status s = env_->GetThreadList(&thread_list);
Y
Yueh-Hsuan Chiang 已提交
6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931

  for (int i = 0; i < 2; ++i) {
    // repeat the test with differet number of high / low priority threads
    const int kTestCount = 3;
    const unsigned int kHighPriCounts[kTestCount] = {3, 2, 5};
    const unsigned int kLowPriCounts[kTestCount] = {10, 15, 3};
    for (int test = 0; test < kTestCount; ++test) {
      // Change the number of threads in high / low priority pool.
      env_->SetBackgroundThreads(kHighPriCounts[test], Env::HIGH);
      env_->SetBackgroundThreads(kLowPriCounts[test], Env::LOW);
      // Wait to ensure the all threads has been registered
      env_->SleepForMicroseconds(100000);
6932
      s = env_->GetThreadList(&thread_list);
Y
Yueh-Hsuan Chiang 已提交
6933
      ASSERT_OK(s);
6934
      unsigned int thread_type_counts[ThreadStatus::NUM_THREAD_TYPES];
Y
Yueh-Hsuan Chiang 已提交
6935 6936
      memset(thread_type_counts, 0, sizeof(thread_type_counts));
      for (auto thread : thread_list) {
6937
        ASSERT_LT(thread.thread_type, ThreadStatus::NUM_THREAD_TYPES);
Y
Yueh-Hsuan Chiang 已提交
6938 6939 6940 6941
        thread_type_counts[thread.thread_type]++;
      }
      // Verify the total number of threades
      ASSERT_EQ(
6942 6943
          thread_type_counts[ThreadStatus::HIGH_PRIORITY] +
              thread_type_counts[ThreadStatus::LOW_PRIORITY],
Y
Yueh-Hsuan Chiang 已提交
6944 6945 6946
          kHighPriCounts[test] + kLowPriCounts[test]);
      // Verify the number of high-priority threads
      ASSERT_EQ(
6947
          thread_type_counts[ThreadStatus::HIGH_PRIORITY],
Y
Yueh-Hsuan Chiang 已提交
6948 6949 6950
          kHighPriCounts[test]);
      // Verify the number of low-priority threads
      ASSERT_EQ(
6951
          thread_type_counts[ThreadStatus::LOW_PRIORITY],
Y
Yueh-Hsuan Chiang 已提交
6952 6953 6954 6955 6956
          kLowPriCounts[test]);
    }
    if (i == 0) {
      // repeat the test with multiple column families
      CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options);
6957 6958
      env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(
          handles_, true);
Y
Yueh-Hsuan Chiang 已提交
6959 6960 6961
    }
  }
  db_->DropColumnFamily(handles_[2]);
6962
  delete handles_[2];
Y
Yueh-Hsuan Chiang 已提交
6963
  handles_.erase(handles_.begin() + 2);
6964 6965
  env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(
      handles_, true);
Y
Yueh-Hsuan Chiang 已提交
6966
  Close();
6967 6968
  env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(
      handles_, true);
6969 6970
}

I
Igor Sugak 已提交
6971
TEST_F(DBTest, DisableThreadStatus) {
6972 6973 6974 6975 6976 6977
  Options options;
  options.env = env_;
  options.enable_thread_tracking = false;
  TryReopen(options);
  CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options);
  // Verify non of the column family info exists
6978 6979
  env_->GetThreadStatusUpdater()->TEST_VerifyColumnFamilyInfoMap(
      handles_, false);
Y
Yueh-Hsuan Chiang 已提交
6980
}
6981

I
Igor Sugak 已提交
6982
TEST_F(DBTest, ThreadStatusFlush) {
6983 6984 6985 6986 6987 6988 6989
  Options options;
  options.env = env_;
  options.write_buffer_size = 100000;  // Small write buffer
  options.enable_thread_tracking = true;
  options = CurrentOptions(options);

  rocksdb::SyncPoint::GetInstance()->LoadDependency({
6990
      {"FlushJob::FlushJob()", "DBTest::ThreadStatusFlush:1"},
6991 6992
      {"DBTest::ThreadStatusFlush:2",
       "FlushJob::LogAndNotifyTableFileCreation()"},
6993 6994 6995 6996 6997 6998 6999 7000 7001 7002
  });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  CreateAndReopenWithCF({"pikachu"}, options);
  VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 0);

  ASSERT_OK(Put(1, "foo", "v1"));
  ASSERT_EQ("v1", Get(1, "foo"));
  VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 0);

7003 7004 7005 7006
  uint64_t num_running_flushes = 0;
  db_->GetIntProperty(DB::Properties::kNumRunningFlushes, &num_running_flushes);
  ASSERT_EQ(num_running_flushes, 0);

7007 7008
  Put(1, "k1", std::string(100000, 'x'));  // Fill memtable
  Put(1, "k2", std::string(100000, 'y'));  // Trigger flush
7009 7010 7011

  // The first sync point is to make sure there's one flush job
  // running when we perform VerifyOperationCount().
7012 7013
  TEST_SYNC_POINT("DBTest::ThreadStatusFlush:1");
  VerifyOperationCount(env_, ThreadStatus::OP_FLUSH, 1);
7014 7015
  db_->GetIntProperty(DB::Properties::kNumRunningFlushes, &num_running_flushes);
  ASSERT_EQ(num_running_flushes, 1);
7016 7017
  // This second sync point is to ensure the flush job will not
  // be completed until we already perform VerifyOperationCount().
7018 7019 7020 7021
  TEST_SYNC_POINT("DBTest::ThreadStatusFlush:2");
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}

7022
TEST_P(DBTestWithParam, ThreadStatusSingleCompaction) {
7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039
  const int kTestKeySize = 16;
  const int kTestValueSize = 984;
  const int kEntrySize = kTestKeySize + kTestValueSize;
  const int kEntriesPerBuffer = 100;
  Options options;
  options.create_if_missing = true;
  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  options.compaction_style = kCompactionStyleLevel;
  options.target_file_size_base = options.write_buffer_size;
  options.max_bytes_for_level_base = options.target_file_size_base * 2;
  options.max_bytes_for_level_multiplier = 2;
  options.compression = kNoCompression;
  options = CurrentOptions(options);
  options.env = env_;
  options.enable_thread_tracking = true;
  const int kNumL0Files = 4;
  options.level0_file_num_compaction_trigger = kNumL0Files;
7040
  options.max_subcompactions = max_subcompactions_;
7041 7042

  rocksdb::SyncPoint::GetInstance()->LoadDependency({
7043 7044 7045
      {"DBTest::ThreadStatusSingleCompaction:0", "DBImpl::BGWorkCompaction"},
      {"CompactionJob::Run():Start", "DBTest::ThreadStatusSingleCompaction:1"},
      {"DBTest::ThreadStatusSingleCompaction:2", "CompactionJob::Run():End"},
7046
  });
7047
  for (int tests = 0; tests < 2; ++tests) {
7048
    DestroyAndReopen(options);
7049 7050
    rocksdb::SyncPoint::GetInstance()->ClearTrace();
    rocksdb::SyncPoint::GetInstance()->EnableProcessing();
7051 7052

    Random rnd(301);
7053
    // The Put Phase.
7054 7055 7056 7057 7058 7059
    for (int file = 0; file < kNumL0Files; ++file) {
      for (int key = 0; key < kEntriesPerBuffer; ++key) {
        ASSERT_OK(Put(ToString(key + file * kEntriesPerBuffer),
                      RandomString(&rnd, kTestValueSize)));
      }
      Flush();
7060
    }
7061 7062
    // This makes sure a compaction won't be scheduled until
    // we have done with the above Put Phase.
7063 7064 7065 7066
    uint64_t num_running_compactions = 0;
    db_->GetIntProperty(DB::Properties::kNumRunningCompactions,
                        &num_running_compactions);
    ASSERT_EQ(num_running_compactions, 0);
7067
    TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:0");
7068 7069
    ASSERT_GE(NumTableFilesAtLevel(0),
              options.level0_file_num_compaction_trigger);
7070

7071
    // This makes sure at least one compaction is running.
7072
    TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:1");
7073

7074 7075
    if (options.enable_thread_tracking) {
      // expecting one single L0 to L1 compaction
7076
      VerifyOperationCount(env_, ThreadStatus::OP_COMPACTION, 1);
7077 7078
    } else {
      // If thread tracking is not enabled, compaction count should be 0.
7079
      VerifyOperationCount(env_, ThreadStatus::OP_COMPACTION, 0);
7080
    }
7081 7082 7083
    db_->GetIntProperty(DB::Properties::kNumRunningCompactions,
                        &num_running_compactions);
    ASSERT_EQ(num_running_compactions, 1);
7084
    // TODO(yhchiang): adding assert to verify each compaction stage.
7085
    TEST_SYNC_POINT("DBTest::ThreadStatusSingleCompaction:2");
7086 7087 7088

    // repeat the test with disabling thread tracking.
    options.enable_thread_tracking = false;
7089
    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
7090 7091 7092
  }
}

7093
TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
7094 7095
  Options options = CurrentOptions();
  options.max_background_flushes = 0;
7096
  options.max_subcompactions = max_subcompactions_;
7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126
  CreateAndReopenWithCF({"pikachu"}, options);

  // iter - 0 with 7 levels
  // iter - 1 with 3 levels
  for (int iter = 0; iter < 2; ++iter) {
    MakeTables(3, "p", "q", 1);
    ASSERT_EQ("1,1,1", FilesPerLevel(1));

    // Compaction range falls before files
    Compact(1, "", "c");
    ASSERT_EQ("1,1,1", FilesPerLevel(1));

    // Compaction range falls after files
    Compact(1, "r", "z");
    ASSERT_EQ("1,1,1", FilesPerLevel(1));

    // Compaction range overlaps files
    Compact(1, "p1", "p9");
    ASSERT_EQ("0,0,1", FilesPerLevel(1));

    // Populate a different range
    MakeTables(3, "c", "e", 1);
    ASSERT_EQ("1,1,2", FilesPerLevel(1));

    // Compact just the new range
    Compact(1, "b", "f");
    ASSERT_EQ("0,0,2", FilesPerLevel(1));

    // Compact all
    MakeTables(1, "a", "z", 1);
7127
    ASSERT_EQ("1,0,2", FilesPerLevel(1));
7128
    CancelAllBackgroundWork(db_);
7129
    db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
7130
    ASSERT_EQ("1,0,2", FilesPerLevel(1));
7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142

    if (iter == 0) {
      options = CurrentOptions();
      options.max_background_flushes = 0;
      options.num_levels = 3;
      options.create_if_missing = true;
      DestroyAndReopen(options);
      CreateAndReopenWithCF({"pikachu"}, options);
    }
  }
}

7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153
TEST_F(DBTest, PreShutdownFlush) {
  Options options = CurrentOptions();
  options.max_background_flushes = 0;
  CreateAndReopenWithCF({"pikachu"}, options);
  ASSERT_OK(Put(1, "key", "value"));
  CancelAllBackgroundWork(db_);
  Status s =
      db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
  ASSERT_TRUE(s.IsShutdownInProgress());
}

7154
TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) {
7155 7156 7157
  const int kTestKeySize = 16;
  const int kTestValueSize = 984;
  const int kEntrySize = kTestKeySize + kTestValueSize;
7158
  const int kEntriesPerBuffer = 40;
7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179
  const int kNumL0Files = 4;

  const int kHighPriCount = 3;
  const int kLowPriCount = 5;
  env_->SetBackgroundThreads(kHighPriCount, Env::HIGH);
  env_->SetBackgroundThreads(kLowPriCount, Env::LOW);

  Options options;
  options.create_if_missing = true;
  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  options.compaction_style = kCompactionStyleLevel;
  options.target_file_size_base = options.write_buffer_size;
  options.max_bytes_for_level_base =
      options.target_file_size_base * kNumL0Files;
  options.compression = kNoCompression;
  options = CurrentOptions(options);
  options.env = env_;
  options.enable_thread_tracking = true;
  options.level0_file_num_compaction_trigger = kNumL0Files;
  options.max_bytes_for_level_multiplier = 2;
  options.max_background_compactions = kLowPriCount;
7180 7181
  options.level0_stop_writes_trigger = 1 << 10;
  options.level0_slowdown_writes_trigger = 1 << 10;
7182
  options.max_subcompactions = max_subcompactions_;
7183 7184 7185 7186 7187 7188 7189

  TryReopen(options);
  Random rnd(301);

  std::vector<ThreadStatus> thread_list;
  // Delay both flush and compaction
  rocksdb::SyncPoint::GetInstance()->LoadDependency(
7190 7191
      {{"FlushJob::FlushJob()", "CompactionJob::Run():Start"},
       {"CompactionJob::Run():Start",
7192
        "DBTest::PreShutdownMultipleCompaction:Preshutdown"},
7193 7194
        {"CompactionJob::Run():Start",
        "DBTest::PreShutdownMultipleCompaction:VerifyCompaction"},
7195
       {"DBTest::PreShutdownMultipleCompaction:Preshutdown",
7196 7197
        "CompactionJob::Run():End"},
       {"CompactionJob::Run():End",
7198 7199 7200 7201 7202 7203 7204 7205
        "DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown"}});

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  // Make rocksdb busy
  int key = 0;
  // check how many threads are doing compaction using GetThreadList
  int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
7206
  for (int file = 0; file < 16 * kNumL0Files; ++file) {
7207 7208 7209 7210 7211 7212 7213 7214 7215 7216
    for (int k = 0; k < kEntriesPerBuffer; ++k) {
      ASSERT_OK(Put(ToString(key++), RandomString(&rnd, kTestValueSize)));
    }

    Status s = env_->GetThreadList(&thread_list);
    for (auto thread : thread_list) {
      operation_count[thread.operation_type]++;
    }

    // Speed up the test
7217 7218
    if (operation_count[ThreadStatus::OP_FLUSH] > 1 &&
        operation_count[ThreadStatus::OP_COMPACTION] >
7219 7220 7221
            0.6 * options.max_background_compactions) {
      break;
    }
7222 7223 7224
    if (file == 15 * kNumL0Files) {
      TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown");
    }
7225 7226 7227
  }

  TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:Preshutdown");
7228
  ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1);
7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242
  CancelAllBackgroundWork(db_);
  TEST_SYNC_POINT("DBTest::PreShutdownMultipleCompaction:VerifyPreshutdown");
  dbfull()->TEST_WaitForCompact();
  // Record the number of compactions at a time.
  for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) {
    operation_count[i] = 0;
  }
  Status s = env_->GetThreadList(&thread_list);
  for (auto thread : thread_list) {
    operation_count[thread.operation_type]++;
  }
  ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0);
}

7243
TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) {
7244 7245 7246
  const int kTestKeySize = 16;
  const int kTestValueSize = 984;
  const int kEntrySize = kTestKeySize + kTestValueSize;
7247
  const int kEntriesPerBuffer = 40;
7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268
  const int kNumL0Files = 4;

  const int kHighPriCount = 3;
  const int kLowPriCount = 5;
  env_->SetBackgroundThreads(kHighPriCount, Env::HIGH);
  env_->SetBackgroundThreads(kLowPriCount, Env::LOW);

  Options options;
  options.create_if_missing = true;
  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  options.compaction_style = kCompactionStyleLevel;
  options.target_file_size_base = options.write_buffer_size;
  options.max_bytes_for_level_base =
      options.target_file_size_base * kNumL0Files;
  options.compression = kNoCompression;
  options = CurrentOptions(options);
  options.env = env_;
  options.enable_thread_tracking = true;
  options.level0_file_num_compaction_trigger = kNumL0Files;
  options.max_bytes_for_level_multiplier = 2;
  options.max_background_compactions = kLowPriCount;
7269 7270
  options.level0_stop_writes_trigger = 1 << 10;
  options.level0_slowdown_writes_trigger = 1 << 10;
7271
  options.max_subcompactions = max_subcompactions_;
7272 7273 7274 7275 7276 7277 7278

  TryReopen(options);
  Random rnd(301);

  std::vector<ThreadStatus> thread_list;
  // Delay both flush and compaction
  rocksdb::SyncPoint::GetInstance()->LoadDependency(
7279
      {{"DBTest::PreShutdownCompactionMiddle:Preshutdown",
7280
        "CompactionJob::Run():Inprogress"},
7281 7282
        {"CompactionJob::Run():Start",
        "DBTest::PreShutdownCompactionMiddle:VerifyCompaction"},
7283 7284
       {"CompactionJob::Run():Inprogress", "CompactionJob::Run():End"},
       {"CompactionJob::Run():End",
7285
        "DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown"}});
7286 7287 7288 7289 7290 7291 7292

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  // Make rocksdb busy
  int key = 0;
  // check how many threads are doing compaction using GetThreadList
  int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
7293
  for (int file = 0; file < 16 * kNumL0Files; ++file) {
7294 7295 7296 7297 7298 7299 7300 7301 7302 7303
    for (int k = 0; k < kEntriesPerBuffer; ++k) {
      ASSERT_OK(Put(ToString(key++), RandomString(&rnd, kTestValueSize)));
    }

    Status s = env_->GetThreadList(&thread_list);
    for (auto thread : thread_list) {
      operation_count[thread.operation_type]++;
    }

    // Speed up the test
7304 7305
    if (operation_count[ThreadStatus::OP_FLUSH] > 1 &&
        operation_count[ThreadStatus::OP_COMPACTION] >
7306 7307 7308
            0.6 * options.max_background_compactions) {
      break;
    }
7309 7310 7311
    if (file == 15 * kNumL0Files) {
      TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyCompaction");
    }
7312 7313
  }

7314
  ASSERT_GE(operation_count[ThreadStatus::OP_COMPACTION], 1);
7315
  CancelAllBackgroundWork(db_);
7316 7317
  TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:Preshutdown");
  TEST_SYNC_POINT("DBTest::PreShutdownCompactionMiddle:VerifyPreshutdown");
7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329
  dbfull()->TEST_WaitForCompact();
  // Record the number of compactions at a time.
  for (int i = 0; i < ThreadStatus::NUM_OP_TYPES; ++i) {
    operation_count[i] = 0;
  }
  Status s = env_->GetThreadList(&thread_list);
  for (auto thread : thread_list) {
    operation_count[thread.operation_type]++;
  }
  ASSERT_EQ(operation_count[ThreadStatus::OP_COMPACTION], 0);
}

Y
Yueh-Hsuan Chiang 已提交
7330 7331
#endif  // ROCKSDB_USING_THREAD_STATUS

I
Islam AbdelRahman 已提交
7332
#ifndef ROCKSDB_LITE
7333 7334 7335 7336 7337 7338 7339
TEST_F(DBTest, FlushOnDestroy) {
  WriteOptions wo;
  wo.disableWAL = true;
  ASSERT_OK(Put("foo", "v1", wo));
  CancelAllBackgroundWork(db_);
}

7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371
namespace {
class OnFileDeletionListener : public EventListener {
 public:
  OnFileDeletionListener() :
      matched_count_(0),
      expected_file_name_("") {}

  void SetExpectedFileName(
      const std::string file_name) {
    expected_file_name_ = file_name;
  }

  void VerifyMatchedCount(size_t expected_value) {
    ASSERT_EQ(matched_count_, expected_value);
  }

  void OnTableFileDeleted(
      const TableFileDeletionInfo& info) override {
    if (expected_file_name_ != "") {
      ASSERT_EQ(expected_file_name_, info.file_path);
      expected_file_name_ = "";
      matched_count_++;
    }
  }

 private:
  size_t matched_count_;
  std::string expected_file_name_;
};

}  // namespace

I
Igor Sugak 已提交
7372
TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
I
Igor Canadi 已提交
7373
  if (!Snappy_Supported()) {
7374 7375
    return;
  }
7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403
  const int kNKeys = 120;
  int keys[kNKeys];
  for (int i = 0; i < kNKeys; i++) {
    keys[i] = i;
  }
  std::random_shuffle(std::begin(keys), std::end(keys));

  Random rnd(301);
  Options options;
  options.create_if_missing = true;
  options.db_write_buffer_size = 20480;
  options.write_buffer_size = 20480;
  options.max_write_buffer_number = 2;
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_stop_writes_trigger = 2;
  options.target_file_size_base = 2048;
  options.level_compaction_dynamic_level_bytes = true;
  options.max_bytes_for_level_base = 102400;
  options.max_bytes_for_level_multiplier = 4;
  options.max_background_compactions = 1;
  options.num_levels = 5;

  options.compression_per_level.resize(3);
  options.compression_per_level[0] = kNoCompression;
  options.compression_per_level[1] = kNoCompression;
  options.compression_per_level[2] = kSnappyCompression;

7404 7405 7406
  OnFileDeletionListener* listener = new OnFileDeletionListener();
  options.listeners.emplace_back(listener);

7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438
  DestroyAndReopen(options);

  // Insert more than 80K. L4 should be base level. Neither L0 nor L4 should
  // be compressed, so total data size should be more than 80K.
  for (int i = 0; i < 20; i++) {
    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
  }
  Flush();
  dbfull()->TEST_WaitForCompact();

  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
  ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(4), 20U * 4000U);

  // Insert 400KB. Some data will be compressed
  for (int i = 21; i < 120; i++) {
    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
  }
  Flush();
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
  ASSERT_LT(SizeAtLevel(0) + SizeAtLevel(3) + SizeAtLevel(4), 120U * 4000U);
  // Make sure data in files in L3 is not compacted by removing all files
  // in L4 and calculate number of rows
  ASSERT_OK(dbfull()->SetOptions({
      {"disable_auto_compactions", "true"},
  }));
  ColumnFamilyMetaData cf_meta;
  db_->GetColumnFamilyMetaData(&cf_meta);
  for (auto file : cf_meta.levels[4].files) {
7439
    listener->SetExpectedFileName(dbname_ + file.name);
7440 7441
    ASSERT_OK(dbfull()->DeleteFile(file.name));
  }
7442 7443
  listener->VerifyMatchedCount(cf_meta.levels[4].files.size());

7444 7445 7446 7447 7448 7449 7450 7451 7452
  int num_keys = 0;
  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    num_keys++;
  }
  ASSERT_OK(iter->status());
  ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(3), num_keys * 4000U);
}

I
Igor Sugak 已提交
7453
TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
7454 7455 7456
  if (!Snappy_Supported() || !LZ4_Supported() || !Zlib_Supported()) {
    return;
  }
7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472
  const int kNKeys = 500;
  int keys[kNKeys];
  for (int i = 0; i < kNKeys; i++) {
    keys[i] = i;
  }
  std::random_shuffle(std::begin(keys), std::end(keys));

  Random rnd(301);
  Options options;
  options.create_if_missing = true;
  options.db_write_buffer_size = 6000;
  options.write_buffer_size = 6000;
  options.max_write_buffer_number = 2;
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_stop_writes_trigger = 2;
7473
  options.soft_pending_compaction_bytes_limit = 1024 * 1024;
7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495

  // Use file size to distinguish levels
  // L1: 10, L2: 20, L3 40, L4 80
  // L0 is less than 30
  options.target_file_size_base = 10;
  options.target_file_size_multiplier = 2;

  options.level_compaction_dynamic_level_bytes = true;
  options.max_bytes_for_level_base = 200;
  options.max_bytes_for_level_multiplier = 8;
  options.max_background_compactions = 1;
  options.num_levels = 5;
  std::shared_ptr<mock::MockTableFactory> mtf(new mock::MockTableFactory);
  options.table_factory = mtf;

  options.compression_per_level.resize(3);
  options.compression_per_level[0] = kNoCompression;
  options.compression_per_level[1] = kLZ4Compression;
  options.compression_per_level[2] = kZlibCompression;

  DestroyAndReopen(options);
  // When base level is L4, L4 is LZ4.
7496 7497 7498
  std::atomic<int> num_zlib(0);
  std::atomic<int> num_lz4(0);
  std::atomic<int> num_no(0);
7499 7500 7501 7502
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
        if (compaction->output_level() == 4) {
7503
          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
7504
          num_lz4.fetch_add(1);
7505 7506
        }
      });
7507 7508 7509 7510 7511 7512
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
        auto* compression = reinterpret_cast<CompressionType*>(arg);
        ASSERT_TRUE(*compression == kNoCompression);
        num_no.fetch_add(1);
      });
7513 7514
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

7515 7516
  for (int i = 0; i < 100; i++) {
    ASSERT_OK(Put(Key(keys[i]), RandomString(&rnd, 200)));
7517 7518 7519 7520

    if (i % 25 == 0) {
      dbfull()->TEST_WaitForFlushMemTable();
    }
7521
  }
7522

7523
  Flush();
7524
  dbfull()->TEST_WaitForFlushMemTable();
7525
  dbfull()->TEST_WaitForCompact();
7526 7527
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
7528 7529 7530 7531

  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
7532
  ASSERT_GT(NumTableFilesAtLevel(4), 0);
7533 7534
  ASSERT_GT(num_no.load(), 2);
  ASSERT_GT(num_lz4.load(), 0);
7535
  int prev_num_files_l4 = NumTableFilesAtLevel(4);
7536 7537

  // After base level turn L4->L3, L3 becomes LZ4 and L4 becomes Zlib
7538 7539
  num_lz4.store(0);
  num_no.store(0);
7540 7541 7542 7543
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
        Compaction* compaction = reinterpret_cast<Compaction*>(arg);
        if (compaction->output_level() == 4 && compaction->start_level() == 3) {
7544
          ASSERT_TRUE(compaction->output_compression() == kZlibCompression);
7545
          num_zlib.fetch_add(1);
7546
        } else {
7547
          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
7548
          num_lz4.fetch_add(1);
7549 7550
        }
      });
7551 7552 7553 7554 7555 7556 7557
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
        auto* compression = reinterpret_cast<CompressionType*>(arg);
        ASSERT_TRUE(*compression == kNoCompression);
        num_no.fetch_add(1);
      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
7558

7559 7560 7561 7562 7563 7564 7565
  for (int i = 101; i < 500; i++) {
    ASSERT_OK(Put(Key(keys[i]), RandomString(&rnd, 200)));
    if (i % 100 == 99) {
      Flush();
      dbfull()->TEST_WaitForCompact();
    }
  }
7566 7567 7568

  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
7569 7570
  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
7571 7572
  ASSERT_GT(NumTableFilesAtLevel(3), 0);
  ASSERT_GT(NumTableFilesAtLevel(4), prev_num_files_l4);
7573 7574 7575
  ASSERT_GT(num_no.load(), 2);
  ASSERT_GT(num_lz4.load(), 0);
  ASSERT_GT(num_zlib.load(), 0);
7576 7577
}

7578
TEST_F(DBTest, DynamicCompactionOptions) {
7579 7580
  // minimum write buffer size is enforced at 64KB
  const uint64_t k32KB = 1 << 15;
7581 7582
  const uint64_t k64KB = 1 << 16;
  const uint64_t k128KB = 1 << 17;
7583
  const uint64_t k1MB = 1 << 20;
7584
  const uint64_t k4KB = 1 << 12;
7585 7586 7587 7588
  Options options;
  options.env = env_;
  options.create_if_missing = true;
  options.compression = kNoCompression;
7589
  options.soft_pending_compaction_bytes_limit = 1024 * 1024;
7590
  options.write_buffer_size = k64KB;
7591
  options.arena_block_size = 4 * k4KB;
7592 7593 7594
  options.max_write_buffer_number = 2;
  // Compaction related options
  options.level0_file_num_compaction_trigger = 3;
L
Lei Jin 已提交
7595 7596
  options.level0_slowdown_writes_trigger = 4;
  options.level0_stop_writes_trigger = 8;
7597 7598 7599
  options.max_grandparent_overlap_factor = 10;
  options.expanded_compaction_factor = 25;
  options.source_compaction_factor = 1;
7600
  options.target_file_size_base = k64KB;
7601
  options.target_file_size_multiplier = 1;
7602
  options.max_bytes_for_level_base = k128KB;
7603
  options.max_bytes_for_level_multiplier = 4;
L
Lei Jin 已提交
7604 7605 7606 7607

  // Block flush thread and disable compaction thread
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
L
Lei Jin 已提交
7608
  DestroyAndReopen(options);
7609

7610
  auto gen_l0_kb = [this](int start, int size, int stride) {
7611 7612
    Random rnd(301);
    for (int i = 0; i < size; i++) {
L
Lei Jin 已提交
7613
      ASSERT_OK(Put(Key(start + stride * i), RandomString(&rnd, 1024)));
7614 7615 7616 7617
    }
    dbfull()->TEST_WaitForFlushMemTable();
  };

7618 7619 7620
  // Write 3 files that have the same key range.
  // Since level0_file_num_compaction_trigger is 3, compaction should be
  // triggered. The compaction should result in one L1 file
7621
  gen_l0_kb(0, 64, 1);
7622
  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
7623
  gen_l0_kb(0, 64, 1);
7624
  ASSERT_EQ(NumTableFilesAtLevel(0), 2);
7625
  gen_l0_kb(0, 64, 1);
7626 7627 7628 7629
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ("0,1", FilesPerLevel());
  std::vector<LiveFileMetaData> metadata;
  db_->GetLiveFilesMetaData(&metadata);
7630
  ASSERT_EQ(1U, metadata.size());
7631 7632
  ASSERT_LE(metadata[0].size, k64KB + k4KB);
  ASSERT_GE(metadata[0].size, k64KB - k4KB);
7633

7634
  // Test compaction trigger and target_file_size_base
7635 7636 7637 7638
  // Reduce compaction trigger to 2, and reduce L1 file size to 32KB.
  // Writing to 64KB L0 files should trigger a compaction. Since these
  // 2 L0 files have the same key range, compaction merge them and should
  // result in 2 32KB L1 files.
7639
  ASSERT_OK(dbfull()->SetOptions({
7640
    {"level0_file_num_compaction_trigger", "2"},
7641
    {"target_file_size_base", ToString(k32KB) }
7642 7643
  }));

7644
  gen_l0_kb(0, 64, 1);
7645
  ASSERT_EQ("1,1", FilesPerLevel());
7646
  gen_l0_kb(0, 64, 1);
7647 7648 7649 7650
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ("0,2", FilesPerLevel());
  metadata.clear();
  db_->GetLiveFilesMetaData(&metadata);
7651
  ASSERT_EQ(2U, metadata.size());
7652 7653
  ASSERT_LE(metadata[0].size, k32KB + k4KB);
  ASSERT_GE(metadata[0].size, k32KB - k4KB);
7654 7655
  ASSERT_LE(metadata[1].size, k32KB + k4KB);
  ASSERT_GE(metadata[1].size, k32KB - k4KB);
7656

7657
  // Test max_bytes_for_level_base
7658 7659 7660
  // Increase level base size to 256KB and write enough data that will
  // fill L1 and L2. L1 size should be around 256KB while L2 size should be
  // around 256KB x 4.
7661
  ASSERT_OK(dbfull()->SetOptions({
7662
    {"max_bytes_for_level_base", ToString(k1MB) }
7663
  }));
7664

7665 7666 7667 7668
  // writing 96 x 64KB => 6 * 1024KB
  // (L1 + L2) = (1 + 4) * 1024KB
  for (int i = 0; i < 96; ++i) {
    gen_l0_kb(i, 64, 96);
7669 7670
  }
  dbfull()->TEST_WaitForCompact();
7671 7672 7673 7674 7675 7676
  ASSERT_GT(SizeAtLevel(1), k1MB / 2);
  ASSERT_LT(SizeAtLevel(1), k1MB + k1MB / 2);

  // Within (0.5, 1.5) of 4MB.
  ASSERT_GT(SizeAtLevel(2), 2 * k1MB);
  ASSERT_LT(SizeAtLevel(2), 6 * k1MB);
7677

7678
  // Test max_bytes_for_level_multiplier and
7679 7680 7681
  // max_bytes_for_level_base. Now, reduce both mulitplier and level base,
  // After filling enough data that can fit in L1 - L3, we should see L1 size
  // reduces to 128KB from 256KB which was asserted previously. Same for L2.
7682
  ASSERT_OK(dbfull()->SetOptions({
7683
    {"max_bytes_for_level_multiplier", "2"},
7684
    {"max_bytes_for_level_base", ToString(k128KB) }
7685 7686
  }));

7687 7688 7689 7690
  // writing 20 x 64KB = 10 x 128KB
  // (L1 + L2 + L3) = (1 + 2 + 4) * 128KB
  for (int i = 0; i < 20; ++i) {
    gen_l0_kb(i, 64, 32);
7691 7692
  }
  dbfull()->TEST_WaitForCompact();
7693 7694 7695
  uint64_t total_size =
    SizeAtLevel(1) + SizeAtLevel(2) + SizeAtLevel(3);
  ASSERT_TRUE(total_size < k128KB * 7 * 1.5);
L
Lei Jin 已提交
7696

7697 7698
  // Test level0_stop_writes_trigger.
  // Clean up memtable and L0. Block compaction threads. If continue to write
7699
  // and flush memtables. We should see put stop after 8 memtable flushes
7700
  // since level0_stop_writes_trigger = 8
7701
  dbfull()->TEST_FlushMemTable(true);
7702
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
7703
  // Block compaction
7704 7705
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
L
Lei Jin 已提交
7706
                 Env::Priority::LOW);
7707
  sleeping_task_low.WaitUntilSleeping();
L
Lei Jin 已提交
7708 7709 7710 7711
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  int count = 0;
  Random rnd(301);
  WriteOptions wo;
7712 7713
  while (count < 64) {
    ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024), wo));
L
Lei Jin 已提交
7714 7715
    dbfull()->TEST_FlushMemTable(true);
    count++;
7716 7717 7718 7719
    if (dbfull()->TEST_write_controler().IsStopped()) {
      sleeping_task_low.WakeUp();
      break;
    }
L
Lei Jin 已提交
7720
  }
7721
  // Stop trigger = 8
L
Lei Jin 已提交
7722 7723
  ASSERT_EQ(count, 8);
  // Unblock
7724
  sleeping_task_low.WaitUntilDone();
L
Lei Jin 已提交
7725

7726 7727
  // Now reduce level0_stop_writes_trigger to 6. Clear up memtables and L0.
  // Block compaction thread again. Perform the put and memtable flushes
7728
  // until we see the stop after 6 memtable flushes.
7729
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
7730 7731
    {"level0_stop_writes_trigger", "6"}
  }));
7732
  dbfull()->TEST_FlushMemTable(true);
7733
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
7734 7735
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

7736 7737
  // Block compaction again
  sleeping_task_low.Reset();
7738
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
L
Lei Jin 已提交
7739
                 Env::Priority::LOW);
7740
  sleeping_task_low.WaitUntilSleeping();
L
Lei Jin 已提交
7741
  count = 0;
7742 7743
  while (count < 64) {
    ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024), wo));
L
Lei Jin 已提交
7744 7745
    dbfull()->TEST_FlushMemTable(true);
    count++;
7746 7747 7748 7749
    if (dbfull()->TEST_write_controler().IsStopped()) {
      sleeping_task_low.WakeUp();
      break;
    }
L
Lei Jin 已提交
7750 7751 7752
  }
  ASSERT_EQ(count, 6);
  // Unblock
7753
  sleeping_task_low.WaitUntilDone();
L
Lei Jin 已提交
7754 7755

  // Test disable_auto_compactions
7756 7757 7758 7759
  // Compaction thread is unblocked but auto compaction is disabled. Write
  // 4 L0 files and compaction should be triggered. If auto compaction is
  // disabled, then TEST_WaitForCompact will be waiting for nothing. Number of
  // L0 files do not change after the call.
7760
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
7761 7762
    {"disable_auto_compactions", "true"}
  }));
7763
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
7764 7765 7766 7767
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

  for (int i = 0; i < 4; ++i) {
    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
7768
    // Wait for compaction so that put won't stop
L
Lei Jin 已提交
7769 7770 7771 7772 7773
    dbfull()->TEST_FlushMemTable(true);
  }
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(NumTableFilesAtLevel(0), 4);

7774 7775
  // Enable auto compaction and perform the same test, # of L0 files should be
  // reduced after compaction.
7776
  ASSERT_OK(dbfull()->SetOptions({
L
Lei Jin 已提交
7777 7778
    {"disable_auto_compactions", "false"}
  }));
7779
  dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
L
Lei Jin 已提交
7780 7781 7782 7783
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

  for (int i = 0; i < 4; ++i) {
    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
7784
    // Wait for compaction so that put won't stop
L
Lei Jin 已提交
7785 7786 7787 7788
    dbfull()->TEST_FlushMemTable(true);
  }
  dbfull()->TEST_WaitForCompact();
  ASSERT_LT(NumTableFilesAtLevel(0), 4);
7789
}
I
Islam AbdelRahman 已提交
7790
#endif  // ROCKSDB_LITE
7791

I
Igor Sugak 已提交
7792
TEST_F(DBTest, FileCreationRandomFailure) {
7793 7794 7795 7796 7797 7798 7799 7800
  Options options;
  options.env = env_;
  options.create_if_missing = true;
  options.write_buffer_size = 100000;  // Small write buffer
  options.target_file_size_base = 200000;
  options.max_bytes_for_level_base = 1000000;
  options.max_bytes_for_level_multiplier = 2;

L
Lei Jin 已提交
7801
  DestroyAndReopen(options);
7802 7803
  Random rnd(301);

7804
  const int kCDTKeysPerBuffer = 4;
7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844
  const int kTestSize = kCDTKeysPerBuffer * 4096;
  const int kTotalIteration = 100;
  // the second half of the test involves in random failure
  // of file creation.
  const int kRandomFailureTest = kTotalIteration / 2;
  std::vector<std::string> values;
  for (int i = 0; i < kTestSize; ++i) {
    values.push_back("NOT_FOUND");
  }
  for (int j = 0; j < kTotalIteration; ++j) {
    if (j == kRandomFailureTest) {
      env_->non_writeable_rate_.store(90);
    }
    for (int k = 0; k < kTestSize; ++k) {
      // here we expect some of the Put fails.
      std::string value = RandomString(&rnd, 100);
      Status s = Put(Key(k), Slice(value));
      if (s.ok()) {
        // update the latest successful put
        values[k] = value;
      }
      // But everything before we simulate the failure-test should succeed.
      if (j < kRandomFailureTest) {
        ASSERT_OK(s);
      }
    }
  }

  // If rocksdb does not do the correct job, internal assert will fail here.
  dbfull()->TEST_WaitForFlushMemTable();
  dbfull()->TEST_WaitForCompact();

  // verify we have the latest successful update
  for (int k = 0; k < kTestSize; ++k) {
    auto v = Get(Key(k));
    ASSERT_EQ(v, values[k]);
  }

  // reopen and reverify we have the latest successful update
  env_->non_writeable_rate_.store(0);
L
Lei Jin 已提交
7845
  Reopen(options);
7846 7847 7848 7849 7850 7851
  for (int k = 0; k < kTestSize; ++k) {
    auto v = Get(Key(k));
    ASSERT_EQ(v, values[k]);
  }
}

I
Islam AbdelRahman 已提交
7852
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
7853
TEST_F(DBTest, DynamicMiscOptions) {
7854 7855 7856 7857 7858 7859 7860
  // Test max_sequential_skip_in_iterations
  Options options;
  options.env = env_;
  options.create_if_missing = true;
  options.max_sequential_skip_in_iterations = 16;
  options.compression = kNoCompression;
  options.statistics = rocksdb::CreateDBStatistics();
L
Lei Jin 已提交
7861
  DestroyAndReopen(options);
7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885

  auto assert_reseek_count = [this, &options](int key_start, int num_reseek) {
    int key0 = key_start;
    int key1 = key_start + 1;
    int key2 = key_start + 2;
    Random rnd(301);
    ASSERT_OK(Put(Key(key0), RandomString(&rnd, 8)));
    for (int i = 0; i < 10; ++i) {
      ASSERT_OK(Put(Key(key1), RandomString(&rnd, 8)));
    }
    ASSERT_OK(Put(Key(key2), RandomString(&rnd, 8)));
    std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
    iter->Seek(Key(key1));
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Key(key1)), 0);
    iter->Next();
    ASSERT_TRUE(iter->Valid());
    ASSERT_EQ(iter->key().compare(Key(key2)), 0);
    ASSERT_EQ(num_reseek,
              TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION));
  };
  // No reseek
  assert_reseek_count(100, 0);

7886
  ASSERT_OK(dbfull()->SetOptions({
7887 7888 7889 7890 7891 7892 7893
    {"max_sequential_skip_in_iterations", "4"}
  }));
  // Clear memtable and make new option effective
  dbfull()->TEST_FlushMemTable(true);
  // Trigger reseek
  assert_reseek_count(200, 1);

7894
  ASSERT_OK(dbfull()->SetOptions({
7895 7896 7897 7898 7899 7900 7901
    {"max_sequential_skip_in_iterations", "16"}
  }));
  // Clear memtable and make new option effective
  dbfull()->TEST_FlushMemTable(true);
  // No reseek
  assert_reseek_count(300, 1);
}
I
Islam AbdelRahman 已提交
7902
#endif  // ROCKSDB_LITE
7903

I
Igor Sugak 已提交
7904
TEST_F(DBTest, DontDeletePendingOutputs) {
I
Igor Canadi 已提交
7905 7906 7907 7908 7909 7910 7911 7912
  Options options;
  options.env = env_;
  options.create_if_missing = true;
  DestroyAndReopen(options);

  // Every time we write to a table file, call FOF/POF with full DB scan. This
  // will make sure our pending_outputs_ protection work correctly
  std::function<void()> purge_obsolete_files_function = [&]() {
7913
    JobContext job_context(0);
I
Igor Canadi 已提交
7914 7915 7916 7917
    dbfull()->TEST_LockMutex();
    dbfull()->FindObsoleteFiles(&job_context, true /*force*/);
    dbfull()->TEST_UnlockMutex();
    dbfull()->PurgeObsoleteFiles(job_context);
Y
Yueh-Hsuan Chiang 已提交
7918
    job_context.Clean();
I
Igor Canadi 已提交
7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935
  };

  env_->table_write_callback_ = &purge_obsolete_files_function;

  for (int i = 0; i < 2; ++i) {
    ASSERT_OK(Put("a", "begin"));
    ASSERT_OK(Put("z", "end"));
    ASSERT_OK(Flush());
  }

  // If pending output guard does not work correctly, PurgeObsoleteFiles() will
  // delete the file that Compaction is trying to create, causing this: error
  // db/db_test.cc:975: IO error:
  // /tmp/rocksdbtest-1552237650/db_test/000009.sst: No such file or directory
  Compact("a", "b");
}

I
Islam AbdelRahman 已提交
7936
#ifndef ROCKSDB_LITE
I
Igor Sugak 已提交
7937
TEST_F(DBTest, DontDeleteMovedFile) {
I
Igor Canadi 已提交
7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966
  // This test triggers move compaction and verifies that the file is not
  // deleted when it's part of move compaction
  Options options = CurrentOptions();
  options.env = env_;
  options.create_if_missing = true;
  options.max_bytes_for_level_base = 1024 * 1024;  // 1 MB
  options.level0_file_num_compaction_trigger =
      2;  // trigger compaction when we have 2 files
  DestroyAndReopen(options);

  Random rnd(301);
  // Create two 1MB sst files
  for (int i = 0; i < 2; ++i) {
    // Create 1MB sst file
    for (int j = 0; j < 100; ++j) {
      ASSERT_OK(Put(Key(i * 50 + j), RandomString(&rnd, 10 * 1024)));
    }
    ASSERT_OK(Flush());
  }
  // this should execute both L0->L1 and L1->(move)->L2 compactions
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ("0,0,1", FilesPerLevel(0));

  // If the moved file is actually deleted (the move-safeguard in
  // ~Version::Version() is not there), we get this failure:
  // Corruption: Can't access /000009.sst
  Reopen(options);
}

I
Igor Sugak 已提交
7967
TEST_F(DBTest, OptimizeFiltersForHits) {
7968
  Options options = CurrentOptions();
7969 7970 7971
  options.write_buffer_size = 64 * 1024;
  options.arena_block_size = 4 * 1024;
  options.target_file_size_base = 64 * 1024;
7972 7973 7974
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_stop_writes_trigger = 4;
7975
  options.max_bytes_for_level_base = 256 * 1024;
7976 7977 7978
  options.max_write_buffer_number = 2;
  options.max_background_compactions = 8;
  options.max_background_flushes = 8;
7979
  options.compression = kNoCompression;
7980
  options.compaction_style = kCompactionStyleLevel;
7981
  options.level_compaction_dynamic_level_bytes = true;
7982 7983 7984 7985 7986 7987 7988 7989 7990 7991
  BlockBasedTableOptions bbto;
  bbto.filter_policy.reset(NewBloomFilterPolicy(10, true));
  bbto.whole_key_filtering = true;
  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
  options.optimize_filters_for_hits = true;
  options.statistics = rocksdb::CreateDBStatistics();
  CreateAndReopenWithCF({"mypikachu"}, options);

  int numkeys = 200000;

7992 7993 7994 7995 7996 7997 7998 7999
  // Generate randomly shuffled keys, so the updates are almost
  // random.
  std::vector<int> keys;
  keys.reserve(numkeys);
  for (int i = 0; i < numkeys; i += 2) {
    keys.push_back(i);
  }
  std::random_shuffle(std::begin(keys), std::end(keys));
8000

8001 8002 8003 8004 8005 8006 8007 8008 8009 8010
  int num_inserted = 0;
  for (int key : keys) {
    ASSERT_OK(Put(1, Key(key), "val"));
    if (++num_inserted % 1000 == 0) {
      dbfull()->TEST_WaitForFlushMemTable();
      dbfull()->TEST_WaitForCompact();
    }
  }
  ASSERT_OK(Put(1, Key(0), "val"));
  ASSERT_OK(Put(1, Key(numkeys), "val"));
8011 8012 8013
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();

8014 8015 8016 8017 8018 8019 8020 8021
  if (NumTableFilesAtLevel(0, 1) == 0) {
    // No Level 0 file. Create one.
    ASSERT_OK(Put(1, Key(0), "val"));
    ASSERT_OK(Put(1, Key(numkeys), "val"));
    ASSERT_OK(Flush(1));
    dbfull()->TEST_WaitForCompact();
  }

8022 8023 8024 8025 8026 8027 8028 8029
  for (int i = 1; i < numkeys; i += 2) {
    ASSERT_EQ(Get(1, Key(i)), "NOT_FOUND");
  }

  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L0));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP));

8030 8031 8032 8033
  // Now we have three sorted run, L0, L5 and L6 with most files in L6 have
  // no blooom filter. Most keys be checked bloom filters twice.
  ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 65000 * 2);
  ASSERT_LT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 120000 * 2);
8034 8035 8036 8037 8038

  for (int i = 0; i < numkeys; i += 2) {
    ASSERT_EQ(Get(1, Key(i)), "val");
  }
}
I
Islam AbdelRahman 已提交
8039
#endif  // ROCKSDB_LITE
8040

I
Igor Sugak 已提交
8041
TEST_F(DBTest, L0L1L2AndUpHitCounter) {
8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078
  Options options = CurrentOptions();
  options.write_buffer_size = 32 * 1024;
  options.target_file_size_base = 32 * 1024;
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_stop_writes_trigger = 4;
  options.max_bytes_for_level_base = 64 * 1024;
  options.max_write_buffer_number = 2;
  options.max_background_compactions = 8;
  options.max_background_flushes = 8;
  options.statistics = rocksdb::CreateDBStatistics();
  CreateAndReopenWithCF({"mypikachu"}, options);

  int numkeys = 20000;
  for (int i = 0; i < numkeys; i++) {
    ASSERT_OK(Put(1, Key(i), "val"));
  }
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L0));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L1));
  ASSERT_EQ(0, TestGetTickerCount(options, GET_HIT_L2_AND_UP));

  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();

  for (int i = 0; i < numkeys; i++) {
    ASSERT_EQ(Get(1, Key(i)), "val");
  }

  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L0), 100);
  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L1), 100);
  ASSERT_GT(TestGetTickerCount(options, GET_HIT_L2_AND_UP), 100);

  ASSERT_EQ(numkeys, TestGetTickerCount(options, GET_HIT_L0) +
                         TestGetTickerCount(options, GET_HIT_L1) +
                         TestGetTickerCount(options, GET_HIT_L2_AND_UP));
}

I
Igor Sugak 已提交
8079
TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
8080 8081 8082 8083 8084 8085 8086
  // iter 0 -- zlib
  // iter 1 -- bzip2
  // iter 2 -- lz4
  // iter 3 -- lz4HC
  CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
                                    kLZ4Compression,  kLZ4HCCompression};
  for (int iter = 0; iter < 4; ++iter) {
8087 8088 8089
    if (!CompressionTypeSupported(compressions[iter])) {
      continue;
    }
8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123
    // first_table_version 1 -- generate with table_version == 1, read with
    // table_version == 2
    // first_table_version 2 -- generate with table_version == 2, read with
    // table_version == 1
    for (int first_table_version = 1; first_table_version <= 2;
         ++first_table_version) {
      BlockBasedTableOptions table_options;
      table_options.format_version = first_table_version;
      table_options.filter_policy.reset(NewBloomFilterPolicy(10));
      Options options = CurrentOptions();
      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
      options.create_if_missing = true;
      options.compression = compressions[iter];
      DestroyAndReopen(options);

      int kNumKeysWritten = 100000;

      Random rnd(301);
      for (int i = 0; i < kNumKeysWritten; ++i) {
        // compressible string
        ASSERT_OK(Put(Key(i), RandomString(&rnd, 128) + std::string(128, 'a')));
      }

      table_options.format_version = first_table_version == 1 ? 2 : 1;
      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
      Reopen(options);
      for (int i = 0; i < kNumKeysWritten; ++i) {
        auto r = Get(Key(i));
        ASSERT_EQ(r.substr(128), std::string(128, 'a'));
      }
    }
  }
}

I
Igor Sugak 已提交
8124
TEST_F(DBTest, MutexWaitStats) {
8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138
  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
  CreateAndReopenWithCF({"pikachu"}, options);
  const int64_t kMutexWaitDelay = 100;
  ThreadStatusUtil::TEST_SetStateDelay(
      ThreadStatus::STATE_MUTEX_WAIT, kMutexWaitDelay);
  ASSERT_OK(Put("hello", "rocksdb"));
  ASSERT_GE(TestGetTickerCount(
            options, DB_MUTEX_WAIT_MICROS), kMutexWaitDelay);
  ThreadStatusUtil::TEST_SetStateDelay(
      ThreadStatus::STATE_MUTEX_WAIT, 0);
}

I
Islam AbdelRahman 已提交
8139
#ifndef ROCKSDB_LITE
I
Igor Canadi 已提交
8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153
// This reproduces a bug where we don't delete a file because when it was
// supposed to be deleted, it was blocked by pending_outputs
// Consider:
// 1. current file_number is 13
// 2. compaction (1) starts, blocks deletion of all files starting with 13
// (pending outputs)
// 3. file 13 is created by compaction (2)
// 4. file 13 is consumed by compaction (3) and file 15 was created. Since file
// 13 has no references, it is put into VersionSet::obsolete_files_
// 5. FindObsoleteFiles() gets file 13 from VersionSet::obsolete_files_. File 13
// is deleted from obsolete_files_ set.
// 6. PurgeObsoleteFiles() tries to delete file 13, but this file is blocked by
// pending outputs since compaction (1) is still running. It is not deleted and
// it is not present in obsolete_files_ anymore. Therefore, we never delete it.
I
Igor Sugak 已提交
8154
TEST_F(DBTest, DeleteObsoleteFilesPendingOutputs) {
I
Igor Canadi 已提交
8155 8156 8157 8158 8159 8160 8161 8162
  Options options = CurrentOptions();
  options.env = env_;
  options.write_buffer_size = 2 * 1024 * 1024;     // 2 MB
  options.max_bytes_for_level_base = 1024 * 1024;  // 1 MB
  options.level0_file_num_compaction_trigger =
      2;  // trigger compaction when we have 2 files
  options.max_background_flushes = 2;
  options.max_background_compactions = 2;
8163 8164 8165 8166

  OnFileDeletionListener* listener = new OnFileDeletionListener();
  options.listeners.emplace_back(listener);

I
Igor Canadi 已提交
8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181
  Reopen(options);

  Random rnd(301);
  // Create two 1MB sst files
  for (int i = 0; i < 2; ++i) {
    // Create 1MB sst file
    for (int j = 0; j < 100; ++j) {
      ASSERT_OK(Put(Key(i * 50 + j), RandomString(&rnd, 10 * 1024)));
    }
    ASSERT_OK(Flush());
  }
  // this should execute both L0->L1 and L1->(move)->L2 compactions
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ("0,0,1", FilesPerLevel(0));

8182
  test::SleepingBackgroundTask blocking_thread;
I
Igor Canadi 已提交
8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214
  port::Mutex mutex_;
  bool already_blocked(false);

  // block the flush
  std::function<void()> block_first_time = [&]() {
    bool blocking = false;
    {
      MutexLock l(&mutex_);
      if (!already_blocked) {
        blocking = true;
        already_blocked = true;
      }
    }
    if (blocking) {
      blocking_thread.DoSleep();
    }
  };
  env_->table_write_callback_ = &block_first_time;
  // Create 1MB sst file
  for (int j = 0; j < 256; ++j) {
    ASSERT_OK(Put(Key(j), RandomString(&rnd, 10 * 1024)));
  }
  // this should trigger a flush, which is blocked with block_first_time
  // pending_file is protecting all the files created after

  ASSERT_OK(dbfull()->TEST_CompactRange(2, nullptr, nullptr));

  ASSERT_EQ("0,0,0,1", FilesPerLevel(0));
  std::vector<LiveFileMetaData> metadata;
  db_->GetLiveFilesMetaData(&metadata);
  ASSERT_EQ(metadata.size(), 1U);
  auto file_on_L2 = metadata[0].name;
8215
  listener->SetExpectedFileName(dbname_ + file_on_L2);
I
Igor Canadi 已提交
8216

8217 8218
  ASSERT_OK(dbfull()->TEST_CompactRange(3, nullptr, nullptr, nullptr,
                                        true /* disallow trivial move */));
I
Igor Canadi 已提交
8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230
  ASSERT_EQ("0,0,0,0,1", FilesPerLevel(0));

  // finish the flush!
  blocking_thread.WakeUp();
  blocking_thread.WaitUntilDone();
  dbfull()->TEST_WaitForFlushMemTable();
  ASSERT_EQ("1,0,0,0,1", FilesPerLevel(0));

  metadata.clear();
  db_->GetLiveFilesMetaData(&metadata);
  ASSERT_EQ(metadata.size(), 2U);

8231
  // This file should have been deleted during last compaction
A
agiardullo 已提交
8232
  ASSERT_EQ(Status::NotFound(), env_->FileExists(dbname_ + file_on_L2));
8233
  listener->VerifyMatchedCount(1);
I
Igor Canadi 已提交
8234
}
I
Islam AbdelRahman 已提交
8235
#endif  // ROCKSDB_LITE
I
Igor Canadi 已提交
8236

I
Igor Sugak 已提交
8237
TEST_F(DBTest, CloseSpeedup) {
8238 8239
  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleLevel;
8240 8241
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
8242 8243 8244 8245 8246 8247 8248 8249
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = 4;
  options.max_bytes_for_level_base = 400 * 1024;
  options.max_write_buffer_number = 16;

  // Block background threads
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
8250 8251
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
8252
                 Env::Priority::LOW);
8253 8254 8255
  test::SleepingBackgroundTask sleeping_task_high;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
                 &sleeping_task_high, Env::Priority::HIGH);
8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290

  std::vector<std::string> filenames;
  env_->GetChildren(dbname_, &filenames);
  // Delete archival files.
  for (size_t i = 0; i < filenames.size(); ++i) {
    env_->DeleteFile(dbname_ + "/" + filenames[i]);
  }
  env_->DeleteDir(dbname_);
  DestroyAndReopen(options);

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
  Random rnd(301);
  int key_idx = 0;

  // First three 110KB files are not going to level 2
  // After that, (100K, 200K)
  for (int num = 0; num < 5; num++) {
    GenerateNewFile(&rnd, &key_idx, true);
  }

  ASSERT_EQ(0, GetSstFileCount(dbname_));

  Close();
  ASSERT_EQ(0, GetSstFileCount(dbname_));

  // Unblock background threads
  sleeping_task_high.WakeUp();
  sleeping_task_high.WaitUntilDone();
  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilDone();

  Destroy(options);
}
8291

8292
class DelayedMergeOperator : public MergeOperator {
8293 8294 8295 8296 8297
 private:
  DBTest* db_test_;

 public:
  explicit DelayedMergeOperator(DBTest* d) : db_test_(d) {}
8298 8299 8300 8301
  virtual bool FullMerge(const Slice& key, const Slice* existing_value,
                         const std::deque<std::string>& operand_list,
                         std::string* new_value,
                         Logger* logger) const override {
S
sdong 已提交
8302
    db_test_->env_->addon_time_.fetch_add(1000);
8303
    *new_value = "";
8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317
    return true;
  }

  virtual const char* Name() const override { return "DelayedMergeOperator"; }
};

TEST_F(DBTest, MergeTestTime) {
  std::string one, two, three;
  PutFixed64(&one, 1);
  PutFixed64(&two, 2);
  PutFixed64(&three, 3);

  // Enable time profiling
  SetPerfLevel(kEnableTime);
S
sdong 已提交
8318
  this->env_->addon_time_.store(0);
8319 8320
  this->env_->time_elapse_only_sleep_ = true;
  this->env_->no_sleep_ = true;
8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340
  Options options;
  options = CurrentOptions(options);
  options.statistics = rocksdb::CreateDBStatistics();
  options.merge_operator.reset(new DelayedMergeOperator(this));
  DestroyAndReopen(options);

  ASSERT_EQ(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 0);
  db_->Put(WriteOptions(), "foo", one);
  ASSERT_OK(Flush());
  ASSERT_OK(db_->Merge(WriteOptions(), "foo", two));
  ASSERT_OK(Flush());
  ASSERT_OK(db_->Merge(WriteOptions(), "foo", three));
  ASSERT_OK(Flush());

  ReadOptions opt;
  opt.verify_checksums = true;
  opt.snapshot = nullptr;
  std::string result;
  db_->Get(opt, "foo", &result);

8341
  ASSERT_EQ(1000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME));
8342 8343 8344 8345 8346 8347 8348 8349 8350 8351

  ReadOptions read_options;
  std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
  int count = 0;
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    ASSERT_OK(iter->status());
    ++count;
  }

  ASSERT_EQ(1, count);
8352
  ASSERT_EQ(2000000, TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME));
8353 8354 8355
#if ROCKSDB_USING_THREAD_STATUS
  ASSERT_GT(TestGetTickerCount(options, FLUSH_WRITE_BYTES), 0);
#endif  // ROCKSDB_USING_THREAD_STATUS
8356
  this->env_->time_elapse_only_sleep_ = false;
8357 8358
}

I
Islam AbdelRahman 已提交
8359
#ifndef ROCKSDB_LITE
8360
TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
8361 8362 8363 8364 8365 8366 8367
  SetPerfLevel(kEnableTime);
  Options options;
  options = CurrentOptions(options);
  options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
  options.statistics = rocksdb::CreateDBStatistics();
  options.merge_operator.reset(new DelayedMergeOperator(this));
  options.compaction_style = kCompactionStyleUniversal;
8368
  options.max_subcompactions = max_subcompactions_;
8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380
  DestroyAndReopen(options);

  for (int i = 0; i < 1000; i++) {
    ASSERT_OK(db_->Merge(WriteOptions(), "foo", "TEST"));
    ASSERT_OK(Flush());
  }
  dbfull()->TEST_WaitForFlushMemTable();
  dbfull()->TEST_WaitForCompact();

  ASSERT_NE(TestGetTickerCount(options, MERGE_OPERATION_TOTAL_TIME), 0);
}

8381
TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
8382 8383 8384 8385 8386 8387
  Options options;
  options.compaction_filter_factory =
      std::make_shared<DelayFilterFactory>(this);
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
8388
  options.max_subcompactions = max_subcompactions_;
8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399
  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // put some data
  for (int table = 0; table < 4; ++table) {
    for (int i = 0; i < 10 + table; ++i) {
      Put(ToString(table * 100 + i), "val");
    }
    Flush();
  }

8400
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
8401 8402 8403 8404 8405 8406 8407 8408 8409
  ASSERT_EQ(0U, CountLiveFiles());

  Reopen(options);

  Iterator* itr = db_->NewIterator(ReadOptions());
  itr->SeekToFirst();
  ASSERT_NE(TestGetTickerCount(options, FILTER_OPERATION_TOTAL_TIME), 0);
  delete itr;
}
I
Islam AbdelRahman 已提交
8410
#endif  // ROCKSDB_LITE
8411

8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426
TEST_F(DBTest, TestLogCleanup) {
  Options options = CurrentOptions();
  options.write_buffer_size = 64 * 1024;  // very small
  // only two memtables allowed ==> only two log files
  options.max_write_buffer_number = 2;
  Reopen(options);

  for (int i = 0; i < 100000; ++i) {
    Put(Key(i), "val");
    // only 2 memtables will be alive, so logs_to_free needs to always be below
    // 2
    ASSERT_LT(dbfull()->TEST_LogsToFreeSize(), static_cast<size_t>(3));
  }
}

I
Islam AbdelRahman 已提交
8427
#ifndef ROCKSDB_LITE
8428 8429 8430 8431 8432 8433 8434 8435 8436 8437
TEST_F(DBTest, EmptyCompactedDB) {
  Options options;
  options.max_open_files = -1;
  options = CurrentOptions(options);
  Close();
  ASSERT_OK(ReadOnlyReopen(options));
  Status s = Put("new", "value");
  ASSERT_TRUE(s.IsNotSupported());
  Close();
}
I
Islam AbdelRahman 已提交
8438
#endif  // ROCKSDB_LITE
8439

8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470
class CountingDeleteTabPropCollector : public TablePropertiesCollector {
 public:
  const char* Name() const override { return "CountingDeleteTabPropCollector"; }

  Status AddUserKey(const Slice& user_key, const Slice& value, EntryType type,
                    SequenceNumber seq, uint64_t file_size) override {
    if (type == kEntryDelete) {
      num_deletes_++;
    }
    return Status::OK();
  }

  bool NeedCompact() const override { return num_deletes_ > 10; }

  UserCollectedProperties GetReadableProperties() const override {
    return UserCollectedProperties{};
  }

  Status Finish(UserCollectedProperties* properties) override {
    *properties =
        UserCollectedProperties{{"num_delete", ToString(num_deletes_)}};
    return Status::OK();
  }

 private:
  uint32_t num_deletes_ = 0;
};

class CountingDeleteTabPropCollectorFactory
    : public TablePropertiesCollectorFactory {
 public:
8471 8472
  virtual TablePropertiesCollector* CreateTablePropertiesCollector(
      TablePropertiesCollectorFactory::Context context) override {
8473 8474 8475 8476 8477 8478 8479
    return new CountingDeleteTabPropCollector();
  }
  const char* Name() const override {
    return "CountingDeleteTabPropCollectorFactory";
  }
};

I
Islam AbdelRahman 已提交
8480
#ifndef ROCKSDB_LITE
8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493
TEST_F(DBTest, TablePropertiesNeedCompactTest) {
  Random rnd(301);

  Options options;
  options.create_if_missing = true;
  options.write_buffer_size = 4096;
  options.max_write_buffer_number = 8;
  options.level0_file_num_compaction_trigger = 2;
  options.level0_slowdown_writes_trigger = 2;
  options.level0_stop_writes_trigger = 4;
  options.target_file_size_base = 2048;
  options.max_bytes_for_level_base = 10240;
  options.max_bytes_for_level_multiplier = 4;
8494
  options.soft_pending_compaction_bytes_limit = 1024 * 1024;
8495 8496
  options.num_levels = 8;

8497 8498
  std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
      std::make_shared<CountingDeleteTabPropCollectorFactory>();
8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556
  options.table_properties_collector_factories.resize(1);
  options.table_properties_collector_factories[0] = collector_factory;

  DestroyAndReopen(options);

  const int kMaxKey = 1000;
  for (int i = 0; i < kMaxKey; i++) {
    ASSERT_OK(Put(Key(i), RandomString(&rnd, 102)));
    ASSERT_OK(Put(Key(kMaxKey + i), RandomString(&rnd, 102)));
  }
  Flush();
  dbfull()->TEST_WaitForCompact();
  if (NumTableFilesAtLevel(0) == 1) {
    // Clear Level 0 so that when later flush a file with deletions,
    // we don't trigger an organic compaction.
    ASSERT_OK(Put(Key(0), ""));
    ASSERT_OK(Put(Key(kMaxKey * 2), ""));
    Flush();
    dbfull()->TEST_WaitForCompact();
  }
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);

  {
    int c = 0;
    std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
    iter->Seek(Key(kMaxKey - 100));
    while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) {
      iter->Next();
      ++c;
    }
    ASSERT_EQ(c, 200);
  }

  Delete(Key(0));
  for (int i = kMaxKey - 100; i < kMaxKey + 100; i++) {
    Delete(Key(i));
  }
  Delete(Key(kMaxKey * 2));

  Flush();
  dbfull()->TEST_WaitForCompact();

  {
    SetPerfLevel(kEnableCount);
    perf_context.Reset();
    int c = 0;
    std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
    iter->Seek(Key(kMaxKey - 100));
    while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) {
      iter->Next();
    }
    ASSERT_EQ(c, 0);
    ASSERT_LT(perf_context.internal_delete_skipped_count, 30u);
    ASSERT_LT(perf_context.internal_key_skipped_count, 30u);
    SetPerfLevel(kDisable);
  }
}

8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611
TEST_F(DBTest, NeedCompactHintPersistentTest) {
  Random rnd(301);

  Options options;
  options.create_if_missing = true;
  options.max_write_buffer_number = 8;
  options.level0_file_num_compaction_trigger = 10;
  options.level0_slowdown_writes_trigger = 10;
  options.level0_stop_writes_trigger = 10;
  options.disable_auto_compactions = true;

  std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
      std::make_shared<CountingDeleteTabPropCollectorFactory>();
  options.table_properties_collector_factories.resize(1);
  options.table_properties_collector_factories[0] = collector_factory;

  DestroyAndReopen(options);

  const int kMaxKey = 100;
  for (int i = 0; i < kMaxKey; i++) {
    ASSERT_OK(Put(Key(i), ""));
  }
  Flush();
  dbfull()->TEST_WaitForFlushMemTable();

  for (int i = 1; i < kMaxKey - 1; i++) {
    Delete(Key(i));
  }
  Flush();
  dbfull()->TEST_WaitForFlushMemTable();
  ASSERT_EQ(NumTableFilesAtLevel(0), 2);

  // Restart the DB. Although number of files didn't reach
  // options.level0_file_num_compaction_trigger, compaction should
  // still be triggered because of the need-compaction hint.
  options.disable_auto_compactions = false;
  Reopen(options);
  dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  {
    SetPerfLevel(kEnableCount);
    perf_context.Reset();
    int c = 0;
    std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
    for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
      c++;
    }
    ASSERT_EQ(c, 2);
    ASSERT_EQ(perf_context.internal_delete_skipped_count, 0);
    // We iterate every key twice. Is it a bug?
    ASSERT_LE(perf_context.internal_key_skipped_count, 2);
    SetPerfLevel(kDisable);
  }
}

8612
TEST_F(DBTest, SuggestCompactRangeTest) {
8613 8614 8615
  class CompactionFilterFactoryGetContext : public CompactionFilterFactory {
   public:
    virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
I
Igor Canadi 已提交
8616
        const CompactionFilter::Context& context) override {
8617 8618 8619 8620 8621 8622 8623 8624 8625
      saved_context = context;
      std::unique_ptr<CompactionFilter> empty_filter;
      return empty_filter;
    }
    const char* Name() const override {
      return "CompactionFilterFactoryGetContext";
    }
    static bool IsManual(CompactionFilterFactory* compaction_filter_factory) {
      return reinterpret_cast<CompactionFilterFactoryGetContext*>(
8626
                 compaction_filter_factory)->saved_context.is_manual_compaction;
8627 8628 8629 8630
    }
    CompactionFilter::Context saved_context;
  };

8631
  Options options = CurrentOptions();
8632 8633
  options.memtable_factory.reset(
      new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
8634
  options.compaction_style = kCompactionStyleLevel;
8635 8636
  options.compaction_filter_factory.reset(
      new CompactionFilterFactoryGetContext());
8637
  options.write_buffer_size = 200 << 10;
8638
  options.arena_block_size = 4 << 10;
8639
  options.level0_file_num_compaction_trigger = 4;
8640
  options.num_levels = 4;
8641 8642 8643 8644
  options.compression = kNoCompression;
  options.max_bytes_for_level_base = 450 << 10;
  options.target_file_size_base = 98 << 10;
  options.max_grandparent_overlap_factor = 1 << 20;  // inf
8645 8646 8647 8648 8649 8650

  Reopen(options);

  Random rnd(301);

  for (int num = 0; num < 3; num++) {
8651
    GenerateNewRandomFile(&rnd);
8652 8653
  }

8654 8655 8656 8657
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("0,4", FilesPerLevel(0));
  ASSERT_TRUE(!CompactionFilterFactoryGetContext::IsManual(
                   options.compaction_filter_factory.get()));
8658

8659
  GenerateNewRandomFile(&rnd);
8660 8661
  ASSERT_EQ("1,4", FilesPerLevel(0));

8662 8663
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("2,4", FilesPerLevel(0));
8664

8665 8666
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("3,4", FilesPerLevel(0));
8667

8668 8669
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("0,4,4", FilesPerLevel(0));
8670

8671
  GenerateNewRandomFile(&rnd);
8672 8673
  ASSERT_EQ("1,4,4", FilesPerLevel(0));

8674 8675
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("2,4,4", FilesPerLevel(0));
8676

8677 8678
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("3,4,4", FilesPerLevel(0));
8679

8680 8681
  GenerateNewRandomFile(&rnd);
  ASSERT_EQ("0,4,8", FilesPerLevel(0));
8682

8683
  GenerateNewRandomFile(&rnd);
8684 8685 8686 8687 8688 8689 8690 8691
  ASSERT_EQ("1,4,8", FilesPerLevel(0));

  // compact it three times
  for (int i = 0; i < 3; ++i) {
    ASSERT_OK(experimental::SuggestCompactRange(db_, nullptr, nullptr));
    dbfull()->TEST_WaitForCompact();
  }

8692 8693 8694
  // All files are compacted
  ASSERT_EQ(0, NumTableFilesAtLevel(0));
  ASSERT_EQ(0, NumTableFilesAtLevel(1));
8695

8696
  GenerateNewRandomFile(&rnd);
8697
  ASSERT_EQ(1, NumTableFilesAtLevel(0));
8698 8699 8700 8701 8702 8703 8704

  // nonoverlapping with the file on level 0
  Slice start("a"), end("b");
  ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end));
  dbfull()->TEST_WaitForCompact();

  // should not compact the level 0 file
8705
  ASSERT_EQ(1, NumTableFilesAtLevel(0));
8706 8707 8708 8709 8710

  start = Slice("j");
  end = Slice("m");
  ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end));
  dbfull()->TEST_WaitForCompact();
8711 8712
  ASSERT_TRUE(CompactionFilterFactoryGetContext::IsManual(
      options.compaction_filter_factory.get()));
8713 8714

  // now it should compact the level 0 file
8715 8716
  ASSERT_EQ(0, NumTableFilesAtLevel(0));
  ASSERT_EQ(1, NumTableFilesAtLevel(1));
8717 8718
}

8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773
TEST_F(DBTest, PromoteL0) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.write_buffer_size = 10 * 1024 * 1024;
  DestroyAndReopen(options);

  // non overlapping ranges
  std::vector<std::pair<int32_t, int32_t>> ranges = {
      {81, 160}, {0, 80}, {161, 240}, {241, 320}};

  int32_t value_size = 10 * 1024;  // 10 KB

  Random rnd(301);
  std::map<int32_t, std::string> values;
  for (const auto& range : ranges) {
    for (int32_t j = range.first; j < range.second; j++) {
      values[j] = RandomString(&rnd, value_size);
      ASSERT_OK(Put(Key(j), values[j]));
    }
    ASSERT_OK(Flush());
  }

  int32_t level0_files = NumTableFilesAtLevel(0, 0);
  ASSERT_EQ(level0_files, ranges.size());
  ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0);  // No files in L1

  // Promote L0 level to L2.
  ASSERT_OK(experimental::PromoteL0(db_, db_->DefaultColumnFamily(), 2));
  // We expect that all the files were trivially moved from L0 to L2
  ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0);
  ASSERT_EQ(NumTableFilesAtLevel(2, 0), level0_files);

  for (const auto& kv : values) {
    ASSERT_EQ(Get(Key(kv.first)), kv.second);
  }
}

TEST_F(DBTest, PromoteL0Failure) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.write_buffer_size = 10 * 1024 * 1024;
  DestroyAndReopen(options);

  // Produce two L0 files with overlapping ranges.
  ASSERT_OK(Put(Key(0), ""));
  ASSERT_OK(Put(Key(3), ""));
  ASSERT_OK(Flush());
  ASSERT_OK(Put(Key(1), ""));
  ASSERT_OK(Flush());

  Status status;
  // Fails because L0 has overlapping files.
  status = experimental::PromoteL0(db_, db_->DefaultColumnFamily());
  ASSERT_TRUE(status.IsInvalidArgument());

8774
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
8775 8776 8777 8778 8779 8780 8781 8782 8783
  // Now there is a file in L1.
  ASSERT_GE(NumTableFilesAtLevel(1, 0), 1);

  ASSERT_OK(Put(Key(5), ""));
  ASSERT_OK(Flush());
  // Fails because L1 is non-empty.
  status = experimental::PromoteL0(db_, db_->DefaultColumnFamily());
  ASSERT_TRUE(status.IsInvalidArgument());
}
I
Islam AbdelRahman 已提交
8784
#endif  // ROCKSDB_LITE
8785

8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801
// Github issue #596
TEST_F(DBTest, HugeNumberOfLevels) {
  Options options = CurrentOptions();
  options.write_buffer_size = 2 * 1024 * 1024;         // 2MB
  options.max_bytes_for_level_base = 2 * 1024 * 1024;  // 2MB
  options.num_levels = 12;
  options.max_background_compactions = 10;
  options.max_bytes_for_level_multiplier = 2;
  options.level_compaction_dynamic_level_bytes = true;
  DestroyAndReopen(options);

  Random rnd(301);
  for (int i = 0; i < 300000; ++i) {
    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
  }

8802
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
8803 8804
}

8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817
// Github issue #595
// Large write batch with column families
TEST_F(DBTest, LargeBatchWithColumnFamilies) {
  Options options;
  options.env = env_;
  options = CurrentOptions(options);
  options.write_buffer_size = 100000;  // Small write buffer
  CreateAndReopenWithCF({"pikachu"}, options);
  int64_t j = 0;
  for (int i = 0; i < 5; i++) {
    for (int pass = 1; pass <= 3; pass++) {
      WriteBatch batch;
      size_t write_size = 1024 * 1024 * (5 + i);
Y
yuslepukhin 已提交
8818
      fprintf(stderr, "prepare: %" ROCKSDB_PRIszt " MB, pass:%d\n", (write_size / 1024 / 1024),
8819 8820 8821
              pass);
      for (;;) {
        std::string data(3000, j++ % 127 + 20);
S
sdong 已提交
8822
        data += ToString(j);
8823 8824 8825 8826 8827
        batch.Put(handles_[0], Slice(data), Slice(data));
        if (batch.GetDataSize() > write_size) {
          break;
        }
      }
Y
yuslepukhin 已提交
8828
      fprintf(stderr, "write: %" ROCKSDB_PRIszt " MB\n", (batch.GetDataSize() / 1024 / 1024));
8829 8830 8831 8832 8833 8834 8835 8836
      ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
      fprintf(stderr, "done\n");
    }
  }
  // make sure we can re-open it.
  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
}

8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899
// Make sure that Flushes can proceed in parallel with CompactRange()
TEST_F(DBTest, FlushesInParallelWithCompactRange) {
  // iter == 0 -- leveled
  // iter == 1 -- leveled, but throw in a flush between two levels compacting
  // iter == 2 -- universal
  for (int iter = 0; iter < 3; ++iter) {
    Options options = CurrentOptions();
    if (iter < 2) {
      options.compaction_style = kCompactionStyleLevel;
    } else {
      options.compaction_style = kCompactionStyleUniversal;
    }
    options.write_buffer_size = 110 << 10;
    options.level0_file_num_compaction_trigger = 4;
    options.num_levels = 4;
    options.compression = kNoCompression;
    options.max_bytes_for_level_base = 450 << 10;
    options.target_file_size_base = 98 << 10;
    options.max_write_buffer_number = 2;

    DestroyAndReopen(options);

    Random rnd(301);
    for (int num = 0; num < 14; num++) {
      GenerateNewRandomFile(&rnd);
    }

    if (iter == 1) {
    rocksdb::SyncPoint::GetInstance()->LoadDependency(
        {{"DBImpl::RunManualCompaction()::1",
          "DBTest::FlushesInParallelWithCompactRange:1"},
         {"DBTest::FlushesInParallelWithCompactRange:2",
          "DBImpl::RunManualCompaction()::2"}});
    } else {
      rocksdb::SyncPoint::GetInstance()->LoadDependency(
          {{"CompactionJob::Run():Start",
            "DBTest::FlushesInParallelWithCompactRange:1"},
           {"DBTest::FlushesInParallelWithCompactRange:2",
            "CompactionJob::Run():End"}});
    }
    rocksdb::SyncPoint::GetInstance()->EnableProcessing();

    std::vector<std::thread> threads;
    threads.emplace_back([&]() { Compact("a", "z"); });

    TEST_SYNC_POINT("DBTest::FlushesInParallelWithCompactRange:1");

    // this has to start a flush. if flushes are blocked, this will try to
    // create
    // 3 memtables, and that will fail because max_write_buffer_number is 2
    for (int num = 0; num < 3; num++) {
      GenerateNewRandomFile(&rnd, /* nowait */ true);
    }

    TEST_SYNC_POINT("DBTest::FlushesInParallelWithCompactRange:2");

    for (auto& t : threads) {
      t.join();
    }
    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  }
}

S
sdong 已提交
8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929
TEST_F(DBTest, DelayedWriteRate) {
  Options options;
  options.env = env_;
  env_->no_sleep_ = true;
  options = CurrentOptions(options);
  options.write_buffer_size = 100000;  // Small write buffer
  options.max_write_buffer_number = 256;
  options.disable_auto_compactions = true;
  options.level0_file_num_compaction_trigger = 3;
  options.level0_slowdown_writes_trigger = 3;
  options.level0_stop_writes_trigger = 999999;
  options.delayed_write_rate = 200000;  // About 200KB/s limited rate

  CreateAndReopenWithCF({"pikachu"}, options);

  for (int i = 0; i < 3; i++) {
    Put(Key(i), std::string(10000, 'x'));
    Flush();
  }

  // These writes will be slowed down to 1KB/s
  size_t estimated_total_size = 0;
  Random rnd(301);
  for (int i = 0; i < 3000; i++) {
    auto rand_num = rnd.Uniform(20);
    // Spread the size range to more.
    size_t entry_size = rand_num * rand_num * rand_num;
    WriteOptions wo;
    Put(Key(i), std::string(entry_size, 'x'), wo);
    estimated_total_size += entry_size + 20;
C
charsyam 已提交
8930
    // Occasionally sleep a while
S
sdong 已提交
8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943
    if (rnd.Uniform(20) == 6) {
      env_->SleepForMicroseconds(2666);
    }
  }
  uint64_t estimated_sleep_time =
      estimated_total_size / options.delayed_write_rate * 1000000U;
  ASSERT_GT(env_->addon_time_.load(), estimated_sleep_time * 0.8);
  ASSERT_LT(env_->addon_time_.load(), estimated_sleep_time * 1.1);

  env_->no_sleep_ = false;
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}

8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959
TEST_F(DBTest, HardLimit) {
  Options options;
  options.env = env_;
  env_->SetBackgroundThreads(1, Env::LOW);
  options = CurrentOptions(options);
  options.max_write_buffer_number = 256;
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 * 1024;
  options.level0_file_num_compaction_trigger = 4;
  options.level0_slowdown_writes_trigger = 999999;
  options.level0_stop_writes_trigger = 999999;
  options.hard_pending_compaction_bytes_limit = 800 << 10;
  options.max_bytes_for_level_base = 10000000000u;
  options.max_background_compactions = 1;

  env_->SetBackgroundThreads(1, Env::LOW);
I
Igor Canadi 已提交
8960 8961
  test::SleepingBackgroundTask sleeping_task_low;
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990
                 Env::Priority::LOW);

  CreateAndReopenWithCF({"pikachu"}, options);

  std::atomic<int> callback_count(0);
  rocksdb::SyncPoint::GetInstance()->SetCallBack("DBImpl::DelayWrite:Wait",
                                                 [&](void* arg) {
                                                   callback_count.fetch_add(1);
                                                   sleeping_task_low.WakeUp();
                                                 });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  Random rnd(301);
  int key_idx = 0;
  for (int num = 0; num < 5; num++) {
    GenerateNewFile(&rnd, &key_idx, true);
  }

  ASSERT_EQ(0, callback_count.load());

  for (int num = 0; num < 5; num++) {
    GenerateNewFile(&rnd, &key_idx, true);
    dbfull()->TEST_WaitForFlushMemTable();
  }
  ASSERT_GE(callback_count.load(), 1);

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}

I
Islam AbdelRahman 已提交
8991
#ifndef ROCKSDB_LITE
S
sdong 已提交
8992 8993 8994 8995 8996 8997
TEST_F(DBTest, SoftLimit) {
  Options options;
  options.env = env_;
  options = CurrentOptions(options);
  options.write_buffer_size = 100000;  // Small write buffer
  options.max_write_buffer_number = 256;
8998
  options.level0_file_num_compaction_trigger = 1;
S
sdong 已提交
8999 9000
  options.level0_slowdown_writes_trigger = 3;
  options.level0_stop_writes_trigger = 999999;
9001 9002
  options.delayed_write_rate = 20000;  // About 200KB/s limited rate
  options.soft_pending_compaction_bytes_limit = 200000;
S
sdong 已提交
9003 9004
  options.target_file_size_base = 99999999;  // All into one file
  options.max_bytes_for_level_base = 50000;
9005 9006
  options.max_bytes_for_level_multiplier = 10;
  options.max_background_compactions = 1;
S
sdong 已提交
9007 9008 9009 9010 9011
  options.compression = kNoCompression;

  Reopen(options);
  Put(Key(0), "");

9012 9013 9014 9015 9016
  test::SleepingBackgroundTask sleeping_task_low;
  // Block compactions
  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::LOW);
  sleeping_task_low.WaitUntilSleeping();
S
sdong 已提交
9017

9018
  // Create 3 L0 files, making score of L0 to be 3.
S
sdong 已提交
9019 9020 9021
  for (int i = 0; i < 3; i++) {
    Put(Key(i), std::string(5000, 'x'));
    Put(Key(100 - i), std::string(5000, 'x'));
9022
    // Flush the file. File size is around 30KB.
S
sdong 已提交
9023 9024
    Flush();
  }
9025 9026 9027 9028 9029 9030 9031
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());

  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilDone();
  sleeping_task_low.Reset();
  dbfull()->TEST_WaitForCompact();

S
sdong 已提交
9032
  // Now there is one L1 file but doesn't trigger soft_rate_limit
9033
  // The L1 file size is around 30KB.
S
sdong 已提交
9034
  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
9035
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
S
sdong 已提交
9036

9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051
  // Only allow one compactin going through.
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "BackgroundCallCompaction:0", [&](void* arg) {
        // Schedule a sleeping task.
        sleeping_task_low.Reset();
        env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
                       &sleeping_task_low, Env::Priority::LOW);
      });

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::LOW);
  sleeping_task_low.WaitUntilSleeping();
  // Create 3 L0 files, making score of L0 to be 3
S
sdong 已提交
9052 9053 9054
  for (int i = 0; i < 3; i++) {
    Put(Key(10 + i), std::string(5000, 'x'));
    Put(Key(90 - i), std::string(5000, 'x'));
9055
    // Flush the file. File size is around 30KB.
S
sdong 已提交
9056 9057
    Flush();
  }
9058 9059 9060 9061 9062 9063 9064 9065 9066 9067

  // Wake up sleep task to enable compaction to run and waits
  // for it to go to sleep state again to make sure one compaction
  // goes through.
  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilSleeping();

  // Now there is one L1 file (around 60KB) which exceeds 50KB base by 10KB
  // Given level multiplier 10, estimated pending compaction is around 100KB
  // doesn't trigger soft_pending_compaction_bytes_limit
S
sdong 已提交
9068
  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
9069
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
S
sdong 已提交
9070

9071 9072 9073 9074 9075 9076
  // Create 3 L0 files, making score of L0 to be 3, higher than L0.
  for (int i = 0; i < 3; i++) {
    Put(Key(20 + i), std::string(5000, 'x'));
    Put(Key(80 - i), std::string(5000, 'x'));
    // Flush the file. File size is around 30KB.
    Flush();
S
sdong 已提交
9077
  }
9078 9079 9080 9081 9082
  // Wake up sleep task to enable compaction to run and waits
  // for it to go to sleep state again to make sure one compaction
  // goes through.
  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilSleeping();
S
sdong 已提交
9083

9084 9085 9086 9087 9088
  // Now there is one L1 file (around 90KB) which exceeds 50KB base by 40KB
  // Given level multiplier 10, estimated pending compaction is around 400KB
  // triggerring soft_pending_compaction_bytes_limit
  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
S
sdong 已提交
9089

9090 9091 9092 9093
  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilSleeping();

  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
S
sdong 已提交
9094 9095 9096 9097 9098 9099

  // shrink level base so L2 will hit soft limit easier.
  ASSERT_OK(dbfull()->SetOptions({
      {"max_bytes_for_level_base", "5000"},
  }));

9100 9101 9102
  Put("", "");
  Flush();
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
S
sdong 已提交
9103

9104
  sleeping_task_low.WaitUntilSleeping();
S
sdong 已提交
9105
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
9106 9107
  sleeping_task_low.WakeUp();
  sleeping_task_low.WaitUntilDone();
S
sdong 已提交
9108
}
I
Islam AbdelRahman 已提交
9109
#endif  // ROCKSDB_LITE
S
sdong 已提交
9110

9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130
TEST_F(DBTest, FailWhenCompressionNotSupportedTest) {
  CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
                                    kLZ4Compression,  kLZ4HCCompression};
  for (int iter = 0; iter < 4; ++iter) {
    if (!CompressionTypeSupported(compressions[iter])) {
      // not supported, we should fail the Open()
      Options options = CurrentOptions();
      options.compression = compressions[iter];
      ASSERT_TRUE(!TryReopen(options).ok());
      // Try if CreateColumnFamily also fails
      options.compression = kNoCompression;
      ASSERT_OK(TryReopen(options));
      ColumnFamilyOptions cf_options(options);
      cf_options.compression = compressions[iter];
      ColumnFamilyHandle* handle;
      ASSERT_TRUE(!db_->CreateColumnFamily(cf_options, "name", &handle).ok());
    }
  }
}

I
Islam AbdelRahman 已提交
9131
#ifndef ROCKSDB_LITE
9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149
TEST_F(DBTest, RowCache) {
  Options options = CurrentOptions();
  options.statistics = rocksdb::CreateDBStatistics();
  options.row_cache = NewLRUCache(8192);
  DestroyAndReopen(options);

  ASSERT_OK(Put("foo", "bar"));
  ASSERT_OK(Flush());

  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 0);
  ASSERT_EQ(Get("foo"), "bar");
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 0);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
  ASSERT_EQ(Get("foo"), "bar");
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
}
I
Islam AbdelRahman 已提交
9150
#endif  // ROCKSDB_LITE
9151

9152 9153
// TODO(3.13): fix the issue of Seek() + Prev() which might not necessary
//             return the biggest key which is smaller than the seek key.
9154
TEST_F(DBTest, PrevAfterMerge) {
9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176
  Options options;
  options.create_if_missing = true;
  options.merge_operator = MergeOperators::CreatePutOperator();
  DestroyAndReopen(options);

  // write three entries with different keys using Merge()
  WriteOptions wopts;
  db_->Merge(wopts, "1", "data1");
  db_->Merge(wopts, "2", "data2");
  db_->Merge(wopts, "3", "data3");

  std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));

  it->Seek("2");
  ASSERT_TRUE(it->Valid());
  ASSERT_EQ("2", it->key().ToString());

  it->Prev();
  ASSERT_TRUE(it->Valid());
  ASSERT_EQ("1", it->key().ToString());
}

9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210
TEST_F(DBTest, DeletingOldWalAfterDrop) {
  rocksdb::SyncPoint::GetInstance()->LoadDependency(
      { { "Test:AllowFlushes", "DBImpl::BGWorkFlush" },
        { "DBImpl::BGWorkFlush:done", "Test:WaitForFlush"} });
  rocksdb::SyncPoint::GetInstance()->ClearTrace();

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  Options options = CurrentOptions();
  options.max_total_wal_size = 8192;
  options.compression = kNoCompression;
  options.write_buffer_size = 1 << 20;
  options.level0_file_num_compaction_trigger = (1<<30);
  options.level0_slowdown_writes_trigger = (1<<30);
  options.level0_stop_writes_trigger = (1<<30);
  options.disable_auto_compactions = true;
  DestroyAndReopen(options);
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  CreateColumnFamilies({"cf1", "cf2"}, options);
  ASSERT_OK(Put(0, "key1", DummyString(8192)));
  ASSERT_OK(Put(0, "key2", DummyString(8192)));
  // the oldest wal should now be getting_flushed
  ASSERT_OK(db_->DropColumnFamily(handles_[0]));
  // all flushes should now do nothing because their CF is dropped
  TEST_SYNC_POINT("Test:AllowFlushes");
  TEST_SYNC_POINT("Test:WaitForFlush");
  uint64_t lognum1 = dbfull()->TEST_LogfileNumber();
  ASSERT_OK(Put(1, "key3", DummyString(8192)));
  ASSERT_OK(Put(1, "key4", DummyString(8192)));
  // new wal should have been created
  uint64_t lognum2 = dbfull()->TEST_LogfileNumber();
  EXPECT_GT(lognum2, lognum1);
}

I
Islam AbdelRahman 已提交
9211
#ifndef ROCKSDB_LITE
I
Islam AbdelRahman 已提交
9212 9213 9214 9215 9216
TEST_F(DBTest, RateLimitedDelete) {
  rocksdb::SyncPoint::GetInstance()->LoadDependency({
      {"DBTest::RateLimitedDelete:1",
       "DeleteSchedulerImpl::BackgroundEmptyTrash"},
  });
9217 9218 9219 9220 9221

  std::vector<uint64_t> penalties;
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "DeleteSchedulerImpl::BackgroundEmptyTrash:Wait",
      [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
I
Islam AbdelRahman 已提交
9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();

  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.env = env_;

  std::string trash_dir = test::TmpDir(env_) + "/trash";
  int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
  Status s;
  options.delete_scheduler.reset(NewDeleteScheduler(
      env_, trash_dir, rate_bytes_per_sec, nullptr, false, &s));
  ASSERT_OK(s);

  Destroy(last_options_);
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
  ASSERT_OK(TryReopen(options));
  // Create 4 files in L0
  for (char v = 'a'; v <= 'd'; v++) {
    ASSERT_OK(Put("Key2", DummyString(1024, v)));
    ASSERT_OK(Put("Key3", DummyString(1024, v)));
    ASSERT_OK(Put("Key4", DummyString(1024, v)));
    ASSERT_OK(Put("Key1", DummyString(1024, v)));
    ASSERT_OK(Put("Key4", DummyString(1024, v)));
    ASSERT_OK(Flush());
  }
  // We created 4 sst files in L0
  ASSERT_EQ("4", FilesPerLevel(0));

  std::vector<LiveFileMetaData> metadata;
  db_->GetLiveFilesMetaData(&metadata);

  // Compaction will move the 4 files in L0 to trash and create 1 L1 file
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("0,1", FilesPerLevel(0));

9257
  uint64_t delete_start_time = env_->NowMicros();
I
Islam AbdelRahman 已提交
9258 9259
  // Hold BackgroundEmptyTrash
  TEST_SYNC_POINT("DBTest::RateLimitedDelete:1");
9260
  options.delete_scheduler->WaitForEmptyTrash();
I
Islam AbdelRahman 已提交
9261 9262
  uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;

9263 9264 9265 9266 9267 9268 9269 9270 9271 9272
  uint64_t total_files_size = 0;
  uint64_t expected_penlty = 0;
  ASSERT_EQ(penalties.size(), metadata.size());
  for (size_t i = 0; i < metadata.size(); i++) {
    total_files_size += metadata[i].size;
    expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec);
    ASSERT_EQ(expected_penlty, penalties[i]);
  }
  ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);

I
Islam AbdelRahman 已提交
9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}

// Create a DB with 2 db_paths, and generate multiple files in the 2
// db_paths using CompactRangeOptions, make sure that files that were
// deleted from first db_path were deleted using DeleteScheduler and
// files in the second path were not.
TEST_F(DBTest, DeleteSchedulerMultipleDBPaths) {
  int bg_delete_file = 0;
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "DeleteSchedulerImpl::DeleteTrashFile:DeleteFile",
      [&](void* arg) { bg_delete_file++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.db_paths.emplace_back(dbname_, 1024 * 100);
  options.db_paths.emplace_back(dbname_ + "_2", 1024 * 100);
  options.env = env_;

  std::string trash_dir = test::TmpDir(env_) + "/trash";
  int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
  Status s;
  options.delete_scheduler.reset(NewDeleteScheduler(
      env_, trash_dir, rate_bytes_per_sec, nullptr, false, &s));
  ASSERT_OK(s);

  DestroyAndReopen(options);

  // Create 4 files in L0
  for (int i = 0; i < 4; i++) {
    ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'A')));
    ASSERT_OK(Flush());
  }
  // We created 4 sst files in L0
  ASSERT_EQ("4", FilesPerLevel(0));
  // Compaction will delete files from L0 in first db path and generate a new
  // file in L1 in second db path
  CompactRangeOptions compact_options;
  compact_options.target_path_id = 1;
  Slice begin("Key0");
  Slice end("Key3");
  ASSERT_OK(db_->CompactRange(compact_options, &begin, &end));
  ASSERT_EQ("0,1", FilesPerLevel(0));

  // Create 4 files in L0
  for (int i = 4; i < 8; i++) {
    ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'B')));
    ASSERT_OK(Flush());
  }
  ASSERT_EQ("4,1", FilesPerLevel(0));

  // Compaction will delete files from L0 in first db path and generate a new
  // file in L1 in second db path
  begin = "Key4";
  end  = "Key7";
  ASSERT_OK(db_->CompactRange(compact_options, &begin, &end));
  ASSERT_EQ("0,2", FilesPerLevel(0));

9332
  options.delete_scheduler->WaitForEmptyTrash();
I
Islam AbdelRahman 已提交
9333 9334 9335 9336 9337 9338 9339
  ASSERT_EQ(bg_delete_file, 8);

  compact_options.bottommost_level_compaction =
      BottommostLevelCompaction::kForce;
  ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
  ASSERT_EQ("0,1", FilesPerLevel(0));

9340
  options.delete_scheduler->WaitForEmptyTrash();
I
Islam AbdelRahman 已提交
9341 9342 9343 9344 9345
  ASSERT_EQ(bg_delete_file, 8);

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}

9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365
TEST_F(DBTest, DestroyDBWithRateLimitedDelete) {
  int bg_delete_file = 0;
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "DeleteSchedulerImpl::DeleteTrashFile:DeleteFile",
      [&](void* arg) { bg_delete_file++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.env = env_;
  DestroyAndReopen(options);

  // Create 4 files in L0
  for (int i = 0; i < 4; i++) {
    ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'A')));
    ASSERT_OK(Flush());
  }
  // We created 4 sst files in L0
  ASSERT_EQ("4", FilesPerLevel(0));

C
charsyam 已提交
9366
  // Close DB and destroy it using DeleteScheduler
9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379
  Close();
  std::string trash_dir = test::TmpDir(env_) + "/trash";
  int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
  Status s;
  options.delete_scheduler.reset(NewDeleteScheduler(
      env_, trash_dir, rate_bytes_per_sec, nullptr, false, &s));
  ASSERT_OK(s);
  ASSERT_OK(DestroyDB(dbname_, options));

  options.delete_scheduler->WaitForEmptyTrash();
  // We have deleted the 4 sst files in the delete_scheduler
  ASSERT_EQ(bg_delete_file, 4);
}
I
Islam AbdelRahman 已提交
9380
#endif  // ROCKSDB_LITE
9381

9382 9383 9384 9385 9386 9387 9388
TEST_F(DBTest, UnsupportedManualSync) {
  DestroyAndReopen(CurrentOptions());
  env_->is_wal_sync_thread_safe_.store(false);
  Status s = db_->SyncWAL();
  ASSERT_TRUE(s.IsNotSupported());
}

I
Islam AbdelRahman 已提交
9389
#ifndef ROCKSDB_LITE
9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445
TEST_F(DBTest, OpenDBWithInfiniteMaxOpenFiles) {
  // Open DB with infinite max open files
  //  - First iteration use 1 thread to open files
  //  - Second iteration use 5 threads to open files
  for (int iter = 0; iter < 2; iter++) {
    Options options;
    options.create_if_missing = true;
    options.write_buffer_size = 100000;
    options.disable_auto_compactions = true;
    options.max_open_files = -1;
    if (iter == 0) {
      options.max_file_opening_threads = 1;
    } else {
      options.max_file_opening_threads = 5;
    }
    options = CurrentOptions(options);
    DestroyAndReopen(options);

    // Create 12 Files in L0 (then move then to L2)
    for (int i = 0; i < 12; i++) {
      std::string k = "L2_" + Key(i);
      ASSERT_OK(Put(k, k + std::string(1000, 'a')));
      ASSERT_OK(Flush());
    }
    CompactRangeOptions compact_options;
    compact_options.change_level = true;
    compact_options.target_level = 2;
    db_->CompactRange(compact_options, nullptr, nullptr);

    // Create 12 Files in L0
    for (int i = 0; i < 12; i++) {
      std::string k = "L0_" + Key(i);
      ASSERT_OK(Put(k, k + std::string(1000, 'a')));
      ASSERT_OK(Flush());
    }
    Close();

    // Reopening the DB will load all exisitng files
    Reopen(options);
    ASSERT_EQ("12,0,12", FilesPerLevel(0));
    std::vector<std::vector<FileMetaData>> files;
    dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files);

    for (const auto& level : files) {
      for (const auto& file : level) {
        ASSERT_TRUE(file.table_reader_handle != nullptr);
      }
    }

    for (int i = 0; i < 12; i++) {
      ASSERT_EQ(Get("L0_" + Key(i)), "L0_" + Key(i) + std::string(1000, 'a'));
      ASSERT_EQ(Get("L2_" + Key(i)), "L2_" + Key(i) + std::string(1000, 'a'));
    }
  }
}

9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622
TEST_F(DBTest, GetTotalSstFilesSize) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.compression = kNoCompression;
  DestroyAndReopen(options);
  // Generate 5 files in L0
  for (int i = 0; i < 5; i++) {
    for (int j = 0; j < 10; j++) {
      std::string val = "val_file_" + ToString(i);
      ASSERT_OK(Put(Key(j), val));
    }
    Flush();
  }
  ASSERT_EQ("5", FilesPerLevel(0));

  std::vector<LiveFileMetaData> live_files_meta;
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 5);
  uint64_t single_file_size = live_files_meta[0].size;

  uint64_t live_sst_files_size = 0;
  uint64_t total_sst_files_size = 0;
  for (const auto& file_meta : live_files_meta) {
    live_sst_files_size += file_meta.size;
  }

  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 5
  // Total SST files = 5
  ASSERT_EQ(live_sst_files_size, 5 * single_file_size);
  ASSERT_EQ(total_sst_files_size, 5 * single_file_size);

  // hold current version
  std::unique_ptr<Iterator> iter1(dbfull()->NewIterator(ReadOptions()));

  // Compact 5 files into 1 file in L0
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("0,1", FilesPerLevel(0));

  live_files_meta.clear();
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 1);

  live_sst_files_size = 0;
  total_sst_files_size = 0;
  for (const auto& file_meta : live_files_meta) {
    live_sst_files_size += file_meta.size;
  }
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 1 (compacted file)
  // Total SST files = 6 (5 original files + compacted file)
  ASSERT_EQ(live_sst_files_size, 1 * single_file_size);
  ASSERT_EQ(total_sst_files_size, 6 * single_file_size);

  // hold current version
  std::unique_ptr<Iterator> iter2(dbfull()->NewIterator(ReadOptions()));

  // Delete all keys and compact, this will delete all live files
  for (int i = 0; i < 10; i++) {
    ASSERT_OK(Delete(Key(i)));
  }
  Flush();
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("", FilesPerLevel(0));

  live_files_meta.clear();
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 0);

  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 0
  // Total SST files = 6 (5 original files + compacted file)
  ASSERT_EQ(total_sst_files_size, 6 * single_file_size);

  iter1.reset();
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 0
  // Total SST files = 1 (compacted file)
  ASSERT_EQ(total_sst_files_size, 1 * single_file_size);

  iter2.reset();
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 0
  // Total SST files = 0
  ASSERT_EQ(total_sst_files_size, 0);
}

TEST_F(DBTest, GetTotalSstFilesSizeVersionsFilesShared) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.compression = kNoCompression;
  DestroyAndReopen(options);
  // Generate 5 files in L0
  for (int i = 0; i < 5; i++) {
    ASSERT_OK(Put(Key(i), "val"));
    Flush();
  }
  ASSERT_EQ("5", FilesPerLevel(0));

  std::vector<LiveFileMetaData> live_files_meta;
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 5);
  uint64_t single_file_size = live_files_meta[0].size;

  uint64_t live_sst_files_size = 0;
  uint64_t total_sst_files_size = 0;
  for (const auto& file_meta : live_files_meta) {
    live_sst_files_size += file_meta.size;
  }

  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));

  // Live SST files = 5
  // Total SST files = 5
  ASSERT_EQ(live_sst_files_size, 5 * single_file_size);
  ASSERT_EQ(total_sst_files_size, 5 * single_file_size);

  // hold current version
  std::unique_ptr<Iterator> iter1(dbfull()->NewIterator(ReadOptions()));

  // Compaction will do trivial move from L0 to L1
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("0,5", FilesPerLevel(0));

  live_files_meta.clear();
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 5);

  live_sst_files_size = 0;
  total_sst_files_size = 0;
  for (const auto& file_meta : live_files_meta) {
    live_sst_files_size += file_meta.size;
  }
  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 5
  // Total SST files = 5 (used in 2 version)
  ASSERT_EQ(live_sst_files_size, 5 * single_file_size);
  ASSERT_EQ(total_sst_files_size, 5 * single_file_size);

  // hold current version
  std::unique_ptr<Iterator> iter2(dbfull()->NewIterator(ReadOptions()));

  // Delete all keys and compact, this will delete all live files
  for (int i = 0; i < 5; i++) {
    ASSERT_OK(Delete(Key(i)));
  }
  Flush();
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_EQ("", FilesPerLevel(0));

  live_files_meta.clear();
  dbfull()->GetLiveFilesMetaData(&live_files_meta);
  ASSERT_EQ(live_files_meta.size(), 0);

  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 0
  // Total SST files = 5 (used in 2 version)
  ASSERT_EQ(total_sst_files_size, 5 * single_file_size);

  iter1.reset();
  iter2.reset();

  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size",
                                       &total_sst_files_size));
  // Live SST files = 0
  // Total SST files = 0
  ASSERT_EQ(total_sst_files_size, 0);
}

9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 9642 9643 9644 9645 9646 9647 9648 9649 9650 9651 9652 9653 9654 9655 9656 9657 9658 9659 9660 9661 9662 9663 9664 9665 9666 9667 9668 9669 9670 9671 9672 9673 9674 9675 9676 9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719
TEST_F(DBTest, AddExternalSstFile) {
  do {
    std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
    env_->CreateDir(sst_files_folder);
    Options options = CurrentOptions();
    options.env = env_;
    const ImmutableCFOptions ioptions(options);

    SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);

    // file1.sst (0 => 99)
    std::string file1 = sst_files_folder + "file1.sst";
    ASSERT_OK(sst_file_writer.Open(file1));
    for (int k = 0; k < 100; k++) {
      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
    }
    ExternalSstFileInfo file1_info;
    Status s = sst_file_writer.Finish(&file1_info);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(file1_info.file_path, file1);
    ASSERT_EQ(file1_info.num_entries, 100);
    ASSERT_EQ(file1_info.smallest_key, Key(0));
    ASSERT_EQ(file1_info.largest_key, Key(99));
    // sst_file_writer already finished, cannot add this value
    s = sst_file_writer.Add(Key(100), "bad_val");
    ASSERT_FALSE(s.ok()) << s.ToString();

    // file2.sst (100 => 199)
    std::string file2 = sst_files_folder + "file2.sst";
    ASSERT_OK(sst_file_writer.Open(file2));
    for (int k = 100; k < 200; k++) {
      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
    }
    // Cannot add this key because it's not after last added key
    s = sst_file_writer.Add(Key(99), "bad_val");
    ASSERT_FALSE(s.ok()) << s.ToString();
    ExternalSstFileInfo file2_info;
    s = sst_file_writer.Finish(&file2_info);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(file2_info.file_path, file2);
    ASSERT_EQ(file2_info.num_entries, 100);
    ASSERT_EQ(file2_info.smallest_key, Key(100));
    ASSERT_EQ(file2_info.largest_key, Key(199));

    // file3.sst (195 => 299)
    // This file values overlap with file2 values
    std::string file3 = sst_files_folder + "file3.sst";
    ASSERT_OK(sst_file_writer.Open(file3));
    for (int k = 195; k < 300; k++) {
      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
    }
    ExternalSstFileInfo file3_info;
    s = sst_file_writer.Finish(&file3_info);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(file3_info.file_path, file3);
    ASSERT_EQ(file3_info.num_entries, 105);
    ASSERT_EQ(file3_info.smallest_key, Key(195));
    ASSERT_EQ(file3_info.largest_key, Key(299));

    // file4.sst (30 => 39)
    // This file values overlap with file1 values
    std::string file4 = sst_files_folder + "file4.sst";
    ASSERT_OK(sst_file_writer.Open(file4));
    for (int k = 30; k < 40; k++) {
      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
    }
    ExternalSstFileInfo file4_info;
    s = sst_file_writer.Finish(&file4_info);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(file4_info.file_path, file4);
    ASSERT_EQ(file4_info.num_entries, 10);
    ASSERT_EQ(file4_info.smallest_key, Key(30));
    ASSERT_EQ(file4_info.largest_key, Key(39));

    // file5.sst (400 => 499)
    std::string file5 = sst_files_folder + "file5.sst";
    ASSERT_OK(sst_file_writer.Open(file5));
    for (int k = 400; k < 500; k++) {
      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
    }
    ExternalSstFileInfo file5_info;
    s = sst_file_writer.Finish(&file5_info);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(file5_info.file_path, file5);
    ASSERT_EQ(file5_info.num_entries, 100);
    ASSERT_EQ(file5_info.smallest_key, Key(400));
    ASSERT_EQ(file5_info.largest_key, Key(499));

    DestroyAndReopen(options);
    // Add file using file path
    s = db_->AddFile(file1);
    ASSERT_TRUE(s.ok()) << s.ToString();
    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
    for (int k = 0; k < 100; k++) {
      ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
    }

9720 9721 9722 9723 9724 9725 9726 9727 9728
    // Add file while holding a snapshot will fail
    const Snapshot* s1 = db_->GetSnapshot();
    if (s1 != nullptr) {
      ASSERT_NOK(db_->AddFile(&file2_info));
      db_->ReleaseSnapshot(s1);
    }
    // We can add the file after releaseing the snapshot
    ASSERT_OK(db_->AddFile(&file2_info));

9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747
    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
    for (int k = 0; k < 200; k++) {
      ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
    }

    // This file have overlapping values with the exisitng data
    s = db_->AddFile(file3);
    ASSERT_FALSE(s.ok()) << s.ToString();

    // This file have overlapping values with the exisitng data
    s = db_->AddFile(&file4_info);
    ASSERT_FALSE(s.ok()) << s.ToString();

    // Overwrite values of keys divisible by 5
    for (int k = 0; k < 200; k += 5) {
      ASSERT_OK(Put(Key(k), Key(k) + "_val_new"));
    }
    ASSERT_NE(db_->GetLatestSequenceNumber(), 0U);

9748 9749
    // Key range of file5 (400 => 499) dont overlap with any keys in DB
    ASSERT_OK(db_->AddFile(file5));
9750 9751 9752 9753 9754 9755 9756 9757 9758 9759

    // Make sure values are correct before and after flush/compaction
    for (int i = 0; i < 2; i++) {
      for (int k = 0; k < 200; k++) {
        std::string value = Key(k) + "_val";
        if (k % 5 == 0) {
          value += "_new";
        }
        ASSERT_EQ(Get(Key(k)), value);
      }
9760 9761 9762 9763
      for (int k = 400; k < 500; k++) {
        std::string value = Key(k) + "_val";
        ASSERT_EQ(Get(Key(k)), value);
      }
9764 9765 9766 9767
      ASSERT_OK(Flush());
      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
    }

9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788 9789 9790
    Close();
    options.disable_auto_compactions = true;
    Reopen(options);

    // Delete keys in range (400 => 499)
    for (int k = 400; k < 500; k++) {
      ASSERT_OK(Delete(Key(k)));
    }
    // We deleted range (400 => 499) but cannot add file5 because
    // of the range tombstones
    ASSERT_NOK(db_->AddFile(file5));

    // Compacting the DB will remove the tombstones
    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));

    // Now we can add the file
    ASSERT_OK(db_->AddFile(file5));

    // Verify values of file5 in DB
    for (int k = 400; k < 500; k++) {
      std::string value = Key(k) + "_val";
      ASSERT_EQ(Get(Key(k)), value);
    }
9791 9792 9793 9794 9795 9796 9797 9798 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814 9815 9816 9817 9818 9819 9820 9821 9822 9823 9824 9825 9826 9827 9828 9829 9830 9831 9832 9833 9834 9835 9836 9837 9838 9839 9840 9841 9842 9843 9844 9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855 9856 9857 9858 9859 9860 9861 9862 9863 9864 9865 9866 9867 9868 9869 9870 9871 9872 9873 9874 9875 9876 9877 9878 9879 9880 9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896 9897 9898 9899 9900 9901 9902 9903 9904 9905 9906 9907 9908 9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962
  } while (ChangeOptions(kSkipPlainTable | kSkipUniversalCompaction |
                         kSkipFIFOCompaction));
}

TEST_F(DBTest, AddExternalSstFileNoCopy) {
  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
  env_->CreateDir(sst_files_folder);
  Options options = CurrentOptions();
  options.env = env_;
  const ImmutableCFOptions ioptions(options);

  SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);

  // file1.sst (0 => 99)
  std::string file1 = sst_files_folder + "file1.sst";
  ASSERT_OK(sst_file_writer.Open(file1));
  for (int k = 0; k < 100; k++) {
    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
  }
  ExternalSstFileInfo file1_info;
  Status s = sst_file_writer.Finish(&file1_info);
  ASSERT_TRUE(s.ok()) << s.ToString();
  ASSERT_EQ(file1_info.file_path, file1);
  ASSERT_EQ(file1_info.num_entries, 100);
  ASSERT_EQ(file1_info.smallest_key, Key(0));
  ASSERT_EQ(file1_info.largest_key, Key(99));

  // file2.sst (100 => 299)
  std::string file2 = sst_files_folder + "file2.sst";
  ASSERT_OK(sst_file_writer.Open(file2));
  for (int k = 100; k < 300; k++) {
    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
  }
  ExternalSstFileInfo file2_info;
  s = sst_file_writer.Finish(&file2_info);
  ASSERT_TRUE(s.ok()) << s.ToString();
  ASSERT_EQ(file2_info.file_path, file2);
  ASSERT_EQ(file2_info.num_entries, 200);
  ASSERT_EQ(file2_info.smallest_key, Key(100));
  ASSERT_EQ(file2_info.largest_key, Key(299));

  // file3.sst (110 => 124) .. overlap with file2.sst
  std::string file3 = sst_files_folder + "file3.sst";
  ASSERT_OK(sst_file_writer.Open(file3));
  for (int k = 110; k < 125; k++) {
    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
  }
  ExternalSstFileInfo file3_info;
  s = sst_file_writer.Finish(&file3_info);
  ASSERT_TRUE(s.ok()) << s.ToString();
  ASSERT_EQ(file3_info.file_path, file3);
  ASSERT_EQ(file3_info.num_entries, 15);
  ASSERT_EQ(file3_info.smallest_key, Key(110));
  ASSERT_EQ(file3_info.largest_key, Key(124));

  s = db_->AddFile(&file1_info, true /* move file */);
  ASSERT_TRUE(s.ok()) << s.ToString();
  ASSERT_EQ(Status::NotFound(), env_->FileExists(file1));

  s = db_->AddFile(&file2_info, false /* copy file */);
  ASSERT_TRUE(s.ok()) << s.ToString();
  ASSERT_OK(env_->FileExists(file2));

  // This file have overlapping values with the exisitng data
  s = db_->AddFile(&file3_info, true /* move file */);
  ASSERT_FALSE(s.ok()) << s.ToString();
  ASSERT_OK(env_->FileExists(file3));

  for (int k = 0; k < 300; k++) {
    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
  }
}

TEST_F(DBTest, AddExternalSstFileMultiThreaded) {
  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
  // Bulk load 10 files every file contain 1000 keys
  int num_files = 10;
  int keys_per_file = 1000;

  // Generate file names
  std::vector<std::string> file_names;
  for (int i = 0; i < num_files; i++) {
    std::string file_name = "file_" + ToString(i) + ".sst";
    file_names.push_back(sst_files_folder + file_name);
  }

  do {
    env_->CreateDir(sst_files_folder);
    Options options = CurrentOptions();
    const ImmutableCFOptions ioptions(options);

    std::atomic<int> thread_num(0);
    std::function<void()> write_file_func = [&]() {
      int file_idx = thread_num.fetch_add(1);
      int range_start = file_idx * keys_per_file;
      int range_end = range_start + keys_per_file;

      SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);

      ASSERT_OK(sst_file_writer.Open(file_names[file_idx]));

      for (int k = range_start; k < range_end; k++) {
        ASSERT_OK(sst_file_writer.Add(Key(k), Key(k)));
      }

      Status s = sst_file_writer.Finish();
      ASSERT_TRUE(s.ok()) << s.ToString();
    };
    // Write num_files files in parallel
    std::vector<std::thread> sst_writer_threads;
    for (int i = 0; i < num_files; ++i) {
      sst_writer_threads.emplace_back(write_file_func);
    }

    for (auto& t : sst_writer_threads) {
      t.join();
    }

    fprintf(stderr, "Wrote %d files (%d keys)\n", num_files,
            num_files * keys_per_file);

    thread_num.store(0);
    std::atomic<int> files_added(0);
    std::function<void()> load_file_func = [&]() {
      // We intentionally add every file twice, and assert that it was added
      // only once and the other add failed
      int thread_id = thread_num.fetch_add(1);
      int file_idx = thread_id / 2;
      // sometimes we use copy, sometimes link .. the result should be the same
      bool move_file = (thread_id % 3 == 0);

      Status s = db_->AddFile(file_names[file_idx], move_file);
      if (s.ok()) {
        files_added++;
      }
    };
    // Bulk load num_files files in parallel
    std::vector<std::thread> add_file_threads;
    DestroyAndReopen(options);
    for (int i = 0; i < num_files * 2; ++i) {
      add_file_threads.emplace_back(load_file_func);
    }

    for (auto& t : add_file_threads) {
      t.join();
    }
    ASSERT_EQ(files_added.load(), num_files);
    fprintf(stderr, "Loaded %d files (%d keys)\n", num_files,
            num_files * keys_per_file);

    // Overwrite values of keys divisible by 100
    for (int k = 0; k < num_files * keys_per_file; k += 100) {
      std::string key = Key(k);
      Status s = Put(key, key + "_new");
      ASSERT_TRUE(s.ok());
    }

    for (int i = 0; i < 2; i++) {
      // Make sure the values are correct before and after flush/compaction
      for (int k = 0; k < num_files * keys_per_file; ++k) {
        std::string key = Key(k);
        std::string value = (k % 100 == 0) ? (key + "_new") : key;
        ASSERT_EQ(Get(key), value);
      }
      ASSERT_OK(Flush());
      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
    }

    fprintf(stderr, "Verified %d values\n", num_files * keys_per_file);
  } while (ChangeOptions(kSkipPlainTable | kSkipUniversalCompaction |
                         kSkipFIFOCompaction));
}
9963 9964 9965 9966 9967 9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997 9998 9999 10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032 10033 10034 10035 10036 10037 10038 10039 10040 10041 10042 10043 10044 10045 10046 10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063

TEST_F(DBTest, AddExternalSstFileOverlappingRanges) {
  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
  Random rnd(301);
  do {
    env_->CreateDir(sst_files_folder);
    Options options = CurrentOptions();
    DestroyAndReopen(options);
    const ImmutableCFOptions ioptions(options);
    SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);

    printf("Option config = %d\n", option_config_);
    std::vector<std::pair<int, int>> key_ranges;
    for (int i = 0; i < 500; i++) {
      int range_start = rnd.Uniform(20000);
      int keys_per_range = 10 + rnd.Uniform(41);

      key_ranges.emplace_back(range_start, range_start + keys_per_range);
    }

    int memtable_add = 0;
    int success_add_file = 0;
    int failed_add_file = 0;
    std::map<std::string, std::string> true_data;
    for (size_t i = 0; i < key_ranges.size(); i++) {
      int range_start = key_ranges[i].first;
      int range_end = key_ranges[i].second;

      Status s;
      std::string range_val = "range_" + ToString(i);

      // For 20% of ranges we use DB::Put, for 80% we use DB::AddFile
      if (i && i % 5 == 0) {
        // Use DB::Put to insert range (insert into memtable)
        range_val += "_put";
        for (int k = range_start; k <= range_end; k++) {
          s = Put(Key(k), range_val);
          ASSERT_OK(s);
        }
        memtable_add++;
      } else {
        // Use DB::AddFile to insert range
        range_val += "_add_file";

        // Generate the file containing the range
        std::string file_name = sst_files_folder + env_->GenerateUniqueId();
        ASSERT_OK(sst_file_writer.Open(file_name));
        for (int k = range_start; k <= range_end; k++) {
          s = sst_file_writer.Add(Key(k), range_val);
          ASSERT_OK(s);
        }
        ExternalSstFileInfo file_info;
        s = sst_file_writer.Finish(&file_info);
        ASSERT_OK(s);

        // Insert the generated file
        s = db_->AddFile(&file_info);

        auto it = true_data.lower_bound(Key(range_start));
        if (it != true_data.end() && it->first <= Key(range_end)) {
          // This range overlap with data already exist in DB
          ASSERT_NOK(s);
          failed_add_file++;
        } else {
          ASSERT_OK(s);
          success_add_file++;
        }
      }

      if (s.ok()) {
        // Update true_data map to include the new inserted data
        for (int k = range_start; k <= range_end; k++) {
          true_data[Key(k)] = range_val;
        }
      }

      // Flush / Compact the DB
      if (i && i % 50 == 0) {
        Flush();
      }
      if (i && i % 75 == 0) {
        db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
      }
    }

    printf(
        "Total: %zu ranges\n"
        "AddFile()|Success: %d ranges\n"
        "AddFile()|RangeConflict: %d ranges\n"
        "Put(): %d ranges\n",
        key_ranges.size(), success_add_file, failed_add_file, memtable_add);

    // Verify the correctness of the data
    for (const auto& kv : true_data) {
      ASSERT_EQ(Get(kv.first), kv.second);
    }
    printf("keys/values verified\n");
  } while (ChangeOptions(kSkipPlainTable | kSkipUniversalCompaction |
                         kSkipFIFOCompaction));
}

I
Islam AbdelRahman 已提交
10064
#endif  // ROCKSDB_LITE
10065

D
dyniusz 已提交
10066 10067 10068 10069 10070 10071 10072 10073 10074 10075 10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096 10097 10098 10099 10100 10101 10102 10103 10104 10105 10106
// 1 Create some SST files by inserting K-V pairs into DB
// 2 Close DB and change suffix from ".sst" to ".ldb" for every other SST file
// 3 Open DB and check if all key can be read
TEST_F(DBTest, SSTsWithLdbSuffixHandling) {
  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.num_levels = 4;
  DestroyAndReopen(options);

  Random rnd(301);
  int key_id = 0;
  for (int i = 0; i < 10; ++i) {
    GenerateNewFile(&rnd, &key_id, false);
  }
  Flush();
  Close();
  int const num_files = GetSstFileCount(dbname_);
  ASSERT_GT(num_files, 0);

  std::vector<std::string> filenames;
  GetSstFiles(dbname_, &filenames);
  int num_ldb_files = 0;
  for (unsigned int i = 0; i < filenames.size(); ++i) {
    if (i & 1) {
      continue;
    }
    std::string const rdb_name = dbname_ + "/" + filenames[i];
    std::string const ldb_name = Rocks2LevelTableFileName(rdb_name);
    ASSERT_TRUE(env_->RenameFile(rdb_name, ldb_name).ok());
    ++num_ldb_files;
  }
  ASSERT_GT(num_ldb_files, 0);
  ASSERT_EQ(num_files, GetSstFileCount(dbname_));

  Reopen(options);
  for (int k = 0; k < key_id; ++k) {
    ASSERT_NE("NOT_FOUND", Get(Key(k)));
  }
  Destroy(options);
}

10107 10108
INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
                        ::testing::Values(1, 4));
10109

10110 10111 10112 10113 10114 10115 10116
TEST_F(DBTest, PauseBackgroundWorkTest) {
  Options options;
  options.write_buffer_size = 100000;  // Small write buffer
  options = CurrentOptions(options);
  Reopen(options);

  std::vector<std::thread> threads;
10117
  std::atomic<bool> done(false);
10118 10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129 10130 10131 10132 10133 10134 10135 10136
  db_->PauseBackgroundWork();
  threads.emplace_back([&]() {
    Random rnd(301);
    for (int i = 0; i < 10000; ++i) {
      Put(RandomString(&rnd, 10), RandomString(&rnd, 10));
    }
    done.store(true);
  });
  env_->SleepForMicroseconds(200000);
  // make sure the thread is not done
  ASSERT_EQ(false, done.load());
  db_->ContinueBackgroundWork();
  for (auto& t : threads) {
    t.join();
  }
  // now it's done
  ASSERT_EQ(true, done.load());
}

10137
#ifndef ROCKSDB_LITE
10138
namespace {
S
sdong 已提交
10139 10140 10141 10142 10143 10144 10145 10146 10147
void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
                          const std::vector<Slice>& keys_must_not_exist) {
  // Ensure that expected keys exist
  std::vector<std::string> values;
  if (keys_must_exist.size() > 0) {
    std::vector<Status> status_list =
        db->MultiGet(ReadOptions(), keys_must_exist, &values);
    for (size_t i = 0; i < keys_must_exist.size(); i++) {
      ASSERT_OK(status_list[i]);
10148
    }
S
sdong 已提交
10149
  }
10150

S
sdong 已提交
10151 10152 10153 10154 10155 10156
  // Ensure that given keys don't exist
  if (keys_must_not_exist.size() > 0) {
    std::vector<Status> status_list =
        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
    for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
      ASSERT_TRUE(status_list[i].IsNotFound());
10157 10158
    }
  }
S
sdong 已提交
10159
}
10160

S
sdong 已提交
10161
}  // namespace
10162

10163
TEST_F(DBTest, WalFilterTest) {
10164
  class TestWalFilter : public WalFilter {
S
sdong 已提交
10165
   private:
10166
    // Processing option that is requested to be applied at the given index
10167 10168 10169
    WalFilter::WalProcessingOption wal_processing_option_;
    // Index at which to apply wal_processing_option_
    // At other indexes default wal_processing_option::kContinueProcessing is
10170
    // returned.
10171
    size_t apply_option_at_record_index_;
10172
    // Current record index, incremented with each record encountered.
10173
    size_t current_record_index_;
10174

S
sdong 已提交
10175 10176 10177 10178 10179 10180 10181 10182 10183 10184
   public:
    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
                  size_t apply_option_for_record_index)
        : wal_processing_option_(wal_processing_option),
          apply_option_at_record_index_(apply_option_for_record_index),
          current_record_index_(0) {}

    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
                                          WriteBatch* new_batch,
                                          bool* batch_changed) const override {
10185
      WalFilter::WalProcessingOption option_to_return;
10186

10187 10188
      if (current_record_index_ == apply_option_at_record_index_) {
        option_to_return = wal_processing_option_;
S
sdong 已提交
10189
      } else {
10190
        option_to_return = WalProcessingOption::kContinueProcessing;
10191 10192 10193 10194 10195
      }

      // Filter is passed as a const object for RocksDB to not modify the
      // object, however we modify it for our own purpose here and hence
      // cast the constness away.
10196
      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
10197

10198
      return option_to_return;
10199 10200
    }

S
sdong 已提交
10201
    virtual const char* Name() const override { return "TestWalFilter"; }
10202 10203 10204
  };

  // Create 3 batches with two keys each
10205
  std::vector<std::vector<std::string>> batch_keys(3);
10206

10207 10208 10209 10210 10211 10212
  batch_keys[0].push_back("key1");
  batch_keys[0].push_back("key2");
  batch_keys[1].push_back("key3");
  batch_keys[1].push_back("key4");
  batch_keys[2].push_back("key5");
  batch_keys[2].push_back("key6");
10213 10214

  // Test with all WAL processing options
S
sdong 已提交
10215 10216 10217 10218
  for (int option = 0;
       option < static_cast<int>(
                    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
       option++) {
10219 10220
    Options options = OptionsForLogIterTest();
    DestroyAndReopen(options);
S
sdong 已提交
10221
    CreateAndReopenWithCF({"pikachu"}, options);
10222

10223
    // Write given keys in given batches
10224
    for (size_t i = 0; i < batch_keys.size(); i++) {
10225
      WriteBatch batch;
10226 10227
      for (size_t j = 0; j < batch_keys[i].size(); j++) {
        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
10228 10229 10230 10231
      }
      dbfull()->Write(WriteOptions(), &batch);
    }

10232
    WalFilter::WalProcessingOption wal_processing_option =
S
sdong 已提交
10233
        static_cast<WalFilter::WalProcessingOption>(option);
10234

10235
    // Create a test filter that would apply wal_processing_option at the first
10236
    // record
10237
    size_t apply_option_for_record_index = 1;
S
sdong 已提交
10238 10239
    TestWalFilter test_wal_filter(wal_processing_option,
                                  apply_option_for_record_index);
10240 10241 10242

    // Reopen database with option to use WAL filter
    options = OptionsForLogIterTest();
10243
    options.wal_filter = &test_wal_filter;
S
sdong 已提交
10244 10245
    Status status =
        TryReopenWithColumnFamilies({"default", "pikachu"}, options);
10246
    if (wal_processing_option ==
S
sdong 已提交
10247
        WalFilter::WalProcessingOption::kCorruptedRecord) {
10248 10249 10250 10251
      assert(!status.ok());
      // In case of corruption we can turn off paranoid_checks to reopen
      // databse
      options.paranoid_checks = false;
S
sdong 已提交
10252
      ReopenWithColumnFamilies({"default", "pikachu"}, options);
10253 10254 10255 10256 10257 10258
    } else {
      assert(status.ok());
    }

    // Compute which keys we expect to be found
    // and which we expect not to be found after recovery.
10259 10260 10261
    std::vector<Slice> keys_must_exist;
    std::vector<Slice> keys_must_not_exist;
    switch (wal_processing_option) {
S
sdong 已提交
10262 10263
      case WalFilter::WalProcessingOption::kCorruptedRecord:
      case WalFilter::WalProcessingOption::kContinueProcessing: {
10264
        fprintf(stderr, "Testing with complete WAL processing\n");
S
sdong 已提交
10265
        // we expect all records to be processed
10266 10267 10268
        for (size_t i = 0; i < batch_keys.size(); i++) {
          for (size_t j = 0; j < batch_keys[i].size(); j++) {
            keys_must_exist.push_back(Slice(batch_keys[i][j]));
10269 10270
          }
        }
10271 10272 10273
        break;
      }
      case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
S
sdong 已提交
10274 10275 10276
        fprintf(stderr,
                "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
                apply_option_for_record_index);
10277
        // We expect the record with apply_option_for_record_index to be not
10278
        // found.
10279 10280 10281 10282
        for (size_t i = 0; i < batch_keys.size(); i++) {
          for (size_t j = 0; j < batch_keys[i].size(); j++) {
            if (i == apply_option_for_record_index) {
              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
S
sdong 已提交
10283
            } else {
10284
              keys_must_exist.push_back(Slice(batch_keys[i][j]));
10285 10286 10287
            }
          }
        }
10288 10289 10290
        break;
      }
      case WalFilter::WalProcessingOption::kStopReplay: {
S
sdong 已提交
10291 10292 10293 10294
        fprintf(stderr,
                "Testing with stopping replay from record %" ROCKSDB_PRIszt
                "\n",
                apply_option_for_record_index);
10295
        // We expect records beyond apply_option_for_record_index to be not
10296
        // found.
10297 10298 10299 10300
        for (size_t i = 0; i < batch_keys.size(); i++) {
          for (size_t j = 0; j < batch_keys[i].size(); j++) {
            if (i >= apply_option_for_record_index) {
              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
S
sdong 已提交
10301
            } else {
10302
              keys_must_exist.push_back(Slice(batch_keys[i][j]));
10303 10304 10305
            }
          }
        }
10306
        break;
10307
      }
10308
      default:
S
sdong 已提交
10309
        assert(false);  // unhandled case
10310
    }
10311

10312
    bool checked_after_reopen = false;
10313

10314
    while (true) {
10315 10316
      // Ensure that expected keys exists
      // and not expected keys don't exist after recovery
10317
      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
10318

10319
      if (checked_after_reopen) {
10320 10321 10322
        break;
      }

S
sdong 已提交
10323
      // reopen database again to make sure previous log(s) are not used
10324
      //(even if they were skipped)
S
sdong 已提交
10325
      // reopn database with option to use WAL filter
10326
      options = OptionsForLogIterTest();
S
sdong 已提交
10327
      ReopenWithColumnFamilies({"default", "pikachu"}, options);
10328

10329
      checked_after_reopen = true;
10330 10331 10332 10333 10334 10335
    }
  }
}

TEST_F(DBTest, WalFilterTestWithChangeBatch) {
  class ChangeBatchHandler : public WriteBatch::Handler {
S
sdong 已提交
10336
   private:
10337
    // Batch to insert keys in
10338
    WriteBatch* new_write_batch_;
10339
    // Number of keys to add in the new batch
10340
    size_t num_keys_to_add_in_new_batch_;
10341
    // Number of keys added to new batch
10342
    size_t num_keys_added_;
S
sdong 已提交
10343 10344 10345 10346 10347 10348 10349

   public:
    ChangeBatchHandler(WriteBatch* new_write_batch,
                       size_t num_keys_to_add_in_new_batch)
        : new_write_batch_(new_write_batch),
          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
          num_keys_added_(0) {}
10350
    virtual void Put(const Slice& key, const Slice& value) override {
10351 10352 10353
      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
        new_write_batch_->Put(key, value);
        ++num_keys_added_;
10354 10355 10356 10357 10358
      }
    }
  };

  class TestWalFilterWithChangeBatch : public WalFilter {
S
sdong 已提交
10359
   private:
10360
    // Index at which to start changing records
10361
    size_t change_records_from_index_;
10362
    // Number of keys to add in the new batch
10363
    size_t num_keys_to_add_in_new_batch_;
10364
    // Current record index, incremented with each record encountered.
10365
    size_t current_record_index_;
10366

S
sdong 已提交
10367 10368 10369 10370 10371 10372 10373 10374 10375 10376
   public:
    TestWalFilterWithChangeBatch(size_t change_records_from_index,
                                 size_t num_keys_to_add_in_new_batch)
        : change_records_from_index_(change_records_from_index),
          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
          current_record_index_(0) {}

    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
                                          WriteBatch* new_batch,
                                          bool* batch_changed) const override {
10377 10378
      if (current_record_index_ >= change_records_from_index_) {
        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
10379 10380 10381 10382 10383 10384 10385
        batch.Iterate(&handler);
        *batch_changed = true;
      }

      // Filter is passed as a const object for RocksDB to not modify the
      // object, however we modify it for our own purpose here and hence
      // cast the constness away.
S
sdong 已提交
10386 10387
      (const_cast<TestWalFilterWithChangeBatch*>(this)
           ->current_record_index_)++;
10388 10389 10390 10391 10392 10393 10394 10395 10396

      return WalProcessingOption::kContinueProcessing;
    }

    virtual const char* Name() const override {
      return "TestWalFilterWithChangeBatch";
    }
  };

10397
  std::vector<std::vector<std::string>> batch_keys(3);
10398

10399 10400 10401 10402 10403 10404
  batch_keys[0].push_back("key1");
  batch_keys[0].push_back("key2");
  batch_keys[1].push_back("key3");
  batch_keys[1].push_back("key4");
  batch_keys[2].push_back("key5");
  batch_keys[2].push_back("key6");
10405

10406 10407
  Options options = OptionsForLogIterTest();
  DestroyAndReopen(options);
S
sdong 已提交
10408
  CreateAndReopenWithCF({"pikachu"}, options);
10409 10410

  // Write given keys in given batches
10411
  for (size_t i = 0; i < batch_keys.size(); i++) {
10412
    WriteBatch batch;
10413 10414
    for (size_t j = 0; j < batch_keys[i].size(); j++) {
      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
10415 10416 10417 10418
    }
    dbfull()->Write(WriteOptions(), &batch);
  }

10419
  // Create a test filter that would apply wal_processing_option at the first
10420
  // record
10421 10422 10423
  size_t change_records_from_index = 1;
  size_t num_keys_to_add_in_new_batch = 1;
  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
S
sdong 已提交
10424
      change_records_from_index, num_keys_to_add_in_new_batch);
10425 10426 10427

  // Reopen database with option to use WAL filter
  options = OptionsForLogIterTest();
10428
  options.wal_filter = &test_wal_filter_with_change_batch;
S
sdong 已提交
10429
  ReopenWithColumnFamilies({"default", "pikachu"}, options);
10430

10431
  // Ensure that all keys exist before change_records_from_index_
10432 10433
  // And after that index only single key exists
  // as our filter adds only single key for each batch
10434 10435
  std::vector<Slice> keys_must_exist;
  std::vector<Slice> keys_must_not_exist;
10436

10437 10438 10439 10440
  for (size_t i = 0; i < batch_keys.size(); i++) {
    for (size_t j = 0; j < batch_keys[i].size(); j++) {
      if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
S
sdong 已提交
10441
      } else {
10442
        keys_must_exist.push_back(Slice(batch_keys[i][j]));
10443 10444 10445
      }
    }
  }
10446

10447
  bool checked_after_reopen = false;
10448

10449
  while (true) {
10450 10451
    // Ensure that expected keys exists
    // and not expected keys don't exist after recovery
10452
    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
10453

10454
    if (checked_after_reopen) {
10455 10456 10457
      break;
    }

S
sdong 已提交
10458
    // reopen database again to make sure previous log(s) are not used
10459
    //(even if they were skipped)
S
sdong 已提交
10460
    // reopn database with option to use WAL filter
10461
    options = OptionsForLogIterTest();
S
sdong 已提交
10462
    ReopenWithColumnFamilies({"default", "pikachu"}, options);
10463

10464 10465 10466 10467 10468 10469
    checked_after_reopen = true;
  }
}

TEST_F(DBTest, WalFilterTestWithChangeBatchExtraKeys) {
  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
S
sdong 已提交
10470 10471 10472 10473
   public:
    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
                                          WriteBatch* new_batch,
                                          bool* batch_changed) const override {
10474 10475 10476 10477 10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495
      *new_batch = batch;
      new_batch->Put("key_extra", "value_extra");
      *batch_changed = true;
      return WalProcessingOption::kContinueProcessing;
    }

    virtual const char* Name() const override {
      return "WalFilterTestWithChangeBatchExtraKeys";
    }
  };

  std::vector<std::vector<std::string>> batch_keys(3);

  batch_keys[0].push_back("key1");
  batch_keys[0].push_back("key2");
  batch_keys[1].push_back("key3");
  batch_keys[1].push_back("key4");
  batch_keys[2].push_back("key5");
  batch_keys[2].push_back("key6");

  Options options = OptionsForLogIterTest();
  DestroyAndReopen(options);
S
sdong 已提交
10496
  CreateAndReopenWithCF({"pikachu"}, options);
10497 10498 10499 10500 10501 10502 10503 10504

  // Write given keys in given batches
  for (size_t i = 0; i < batch_keys.size(); i++) {
    WriteBatch batch;
    for (size_t j = 0; j < batch_keys[i].size(); j++) {
      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
    }
    dbfull()->Write(WriteOptions(), &batch);
10505
  }
10506 10507 10508 10509 10510 10511 10512

  // Create a test filter that would add extra keys
  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;

  // Reopen database with option to use WAL filter
  options = OptionsForLogIterTest();
  options.wal_filter = &test_wal_filter_extra_keys;
S
sdong 已提交
10513
  Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
10514 10515 10516 10517 10518
  ASSERT_TRUE(status.IsNotSupported());

  // Reopen without filter, now reopen should succeed - previous
  // attempt to open must not have altered the db.
  options = OptionsForLogIterTest();
S
sdong 已提交
10519
  ReopenWithColumnFamilies({"default", "pikachu"}, options);
10520 10521

  std::vector<Slice> keys_must_exist;
S
sdong 已提交
10522
  std::vector<Slice> keys_must_not_exist;  // empty vector
10523 10524 10525 10526 10527 10528 10529 10530

  for (size_t i = 0; i < batch_keys.size(); i++) {
    for (size_t j = 0; j < batch_keys[i].size(); j++) {
      keys_must_exist.push_back(Slice(batch_keys[i][j]));
    }
  }

  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
10531
}
10532

S
sdong 已提交
10533
#endif  // ROCKSDB_LITE
P
Praveen Rao 已提交
10534

I
Islam AbdelRahman 已提交
10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550 10551 10552 10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571 10572 10573 10574 10575
#ifndef ROCKSDB_LITE
class BloomStatsTestWithParam
    : public DBTest,
      public testing::WithParamInterface<std::tuple<bool, bool>> {
 public:
  BloomStatsTestWithParam() {
    use_block_table_ = std::get<0>(GetParam());
    use_block_based_builder_ = std::get<1>(GetParam());

    options_.create_if_missing = true;
    options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
    options_.memtable_prefix_bloom_bits = 8 * 1024;
    if (use_block_table_) {
      BlockBasedTableOptions table_options;
      table_options.hash_index_allow_collision = false;
      table_options.filter_policy.reset(
          NewBloomFilterPolicy(10, use_block_based_builder_));
      options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
    } else {
      PlainTableOptions table_options;
      options_.table_factory.reset(NewPlainTableFactory(table_options));
    }

    perf_context.Reset();
    DestroyAndReopen(options_);
  }

  ~BloomStatsTestWithParam() {
    perf_context.Reset();
    Destroy(options_);
  }

  // Required if inheriting from testing::WithParamInterface<>
  static void SetUpTestCase() {}
  static void TearDownTestCase() {}

  bool use_block_table_;
  bool use_block_based_builder_;
  Options options_;
};

10576 10577 10578 10579 10580 10581 10582 10583 10584 10585 10586 10587 10588 10589 10590 10591 10592 10593 10594 10595 10596 10597 10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625 10626 10627 10628 10629 10630 10631 10632 10633 10634 10635 10636 10637 10638 10639 10640 10641 10642 10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657 10658 10659 10660 10661 10662 10663 10664
// 1 Insert 2 K-V pairs into DB
// 2 Call Get() for both keys - expext memtable bloom hit stat to be 2
// 3 Call Get() for nonexisting key - expect memtable bloom miss stat to be 1
// 4 Call Flush() to create SST
// 5 Call Get() for both keys - expext SST bloom hit stat to be 2
// 6 Call Get() for nonexisting key - expect SST bloom miss stat to be 1
// Test both: block and plain SST
TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
  std::string key1("AAAA");
  std::string key2("RXDB");  // not in DB
  std::string key3("ZBRA");
  std::string value1("Value1");
  std::string value3("Value3");

  ASSERT_OK(Put(key1, value1, WriteOptions()));
  ASSERT_OK(Put(key3, value3, WriteOptions()));

  // check memtable bloom stats
  ASSERT_EQ(value1, Get(key1));
  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
  ASSERT_EQ(value3, Get(key3));
  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);

  ASSERT_EQ("NOT_FOUND", Get(key2));
  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);

  // sanity checks
  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);

  Flush();

  // sanity checks
  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);

  // check SST bloom stats
  // NOTE: hits per get differs because of code paths differences
  // in BlockBasedTable::Get()
  int hits_per_get = use_block_table_ && !use_block_based_builder_ ? 2 : 1;
  ASSERT_EQ(value1, Get(key1));
  ASSERT_EQ(hits_per_get, perf_context.bloom_sst_hit_count);
  ASSERT_EQ(value3, Get(key3));
  ASSERT_EQ(2 * hits_per_get, perf_context.bloom_sst_hit_count);

  ASSERT_EQ("NOT_FOUND", Get(key2));
  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
}

// Same scenario as in BloomStatsTest but using an iterator
TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
  std::string key1("AAAA");
  std::string key2("RXDB");  // not in DB
  std::string key3("ZBRA");
  std::string value1("Value1");
  std::string value3("Value3");

  ASSERT_OK(Put(key1, value1, WriteOptions()));
  ASSERT_OK(Put(key3, value3, WriteOptions()));

  unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));

  // check memtable bloom stats
  iter->Seek(key1);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value1, iter->value().ToString());
  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);

  iter->Seek(key3);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value3, iter->value().ToString());
  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);

  iter->Seek(key2);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(!iter->Valid());
  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);

  Flush();

  iter.reset(dbfull()->NewIterator(ReadOptions()));

10665
  // Check SST bloom stats
10666 10667 10668 10669 10670 10671 10672 10673 10674 10675 10676 10677 10678 10679 10680 10681 10682 10683 10684 10685 10686 10687 10688
  iter->Seek(key1);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value1, iter->value().ToString());
  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);

  iter->Seek(key3);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(iter->Valid());
  ASSERT_EQ(value3, iter->value().ToString());
  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);

  iter->Seek(key2);
  ASSERT_OK(iter->status());
  ASSERT_TRUE(!iter->Valid());
  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
}

INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,
                        ::testing::Values(std::make_tuple(true, true),
                                          std::make_tuple(true, false),
                                          std::make_tuple(false, false)));
I
Islam AbdelRahman 已提交
10689
#endif  // ROCKSDB_LITE
10690
}  // namespace rocksdb
J
jorlow@chromium.org 已提交
10691

D
Dmitri Smirnov 已提交
10692

J
jorlow@chromium.org 已提交
10693
int main(int argc, char** argv) {
10694
  rocksdb::port::InstallStackTraceHandler();
I
Igor Sugak 已提交
10695 10696
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
J
jorlow@chromium.org 已提交
10697
}