db_impl.h 25.5 KB
Newer Older
1 2 3 4 5
//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
J
jorlow@chromium.org 已提交
6 7 8
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
9
#pragma once
K
Kai Liu 已提交
10

H
Haobo Xu 已提交
11
#include <atomic>
12
#include <deque>
13
#include <limits>
J
jorlow@chromium.org 已提交
14
#include <set>
T
Tomislav Novak 已提交
15
#include <utility>
16
#include <vector>
I
Igor Canadi 已提交
17
#include <string>
K
kailiu 已提交
18

J
jorlow@chromium.org 已提交
19 20 21
#include "db/dbformat.h"
#include "db/log_writer.h"
#include "db/snapshot.h"
22
#include "db/column_family.h"
23
#include "db/version_edit.h"
K
Kai Liu 已提交
24 25
#include "memtable_list.h"
#include "port/port.h"
26 27 28 29
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/transaction_log.h"
30
#include "util/autovector.h"
31
#include "util/stop_watch.h"
32
#include "util/thread_local.h"
33
#include "util/scoped_arena_iterator.h"
I
Igor Canadi 已提交
34
#include "db/internal_stats.h"
35
#include "db/write_controller.h"
I
Igor Canadi 已提交
36
#include "db/flush_scheduler.h"
I
Igor Canadi 已提交
37
#include "db/write_thread.h"
38

39
namespace rocksdb {
J
jorlow@chromium.org 已提交
40 41 42 43 44 45

class MemTable;
class TableCache;
class Version;
class VersionEdit;
class VersionSet;
D
Danny Guo 已提交
46
class CompactionFilterV2;
47
class Arena;
J
jorlow@chromium.org 已提交
48 49 50

class DBImpl : public DB {
 public:
I
Igor Canadi 已提交
51
  DBImpl(const DBOptions& options, const std::string& dbname);
J
jorlow@chromium.org 已提交
52 53 54
  virtual ~DBImpl();

  // Implementations of the DB interface
55 56
  using DB::Put;
  virtual Status Put(const WriteOptions& options,
57
                     ColumnFamilyHandle* column_family, const Slice& key,
58 59 60
                     const Slice& value);
  using DB::Merge;
  virtual Status Merge(const WriteOptions& options,
61 62
                       ColumnFamilyHandle* column_family, const Slice& key,
                       const Slice& value);
63 64
  using DB::Delete;
  virtual Status Delete(const WriteOptions& options,
65
                        ColumnFamilyHandle* column_family, const Slice& key);
66
  using DB::Write;
J
jorlow@chromium.org 已提交
67
  virtual Status Write(const WriteOptions& options, WriteBatch* updates);
68
  using DB::Get;
J
jorlow@chromium.org 已提交
69
  virtual Status Get(const ReadOptions& options,
70
                     ColumnFamilyHandle* column_family, const Slice& key,
J
jorlow@chromium.org 已提交
71
                     std::string* value);
72 73 74
  using DB::MultiGet;
  virtual std::vector<Status> MultiGet(
      const ReadOptions& options,
75
      const std::vector<ColumnFamilyHandle*>& column_family,
76
      const std::vector<Slice>& keys, std::vector<std::string>* values);
77

78
  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
79
                                    const std::string& column_family,
80 81
                                    ColumnFamilyHandle** handle);
  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family);
82

83 84 85 86
  // Returns false if key doesn't exist in the database and true if it may.
  // If value_found is not passed in as null, then return the value if found in
  // memory. On return, if value was found, then value_found will be set to true
  // , otherwise false.
87
  using DB::KeyMayExist;
88
  virtual bool KeyMayExist(const ReadOptions& options,
89 90
                           ColumnFamilyHandle* column_family, const Slice& key,
                           std::string* value, bool* value_found = nullptr);
91 92
  using DB::NewIterator;
  virtual Iterator* NewIterator(const ReadOptions& options,
93
                                ColumnFamilyHandle* column_family);
94 95
  virtual Status NewIterators(
      const ReadOptions& options,
I
Igor Canadi 已提交
96
      const std::vector<ColumnFamilyHandle*>& column_families,
97
      std::vector<Iterator*>* iterators);
J
jorlow@chromium.org 已提交
98 99
  virtual const Snapshot* GetSnapshot();
  virtual void ReleaseSnapshot(const Snapshot* snapshot);
100
  using DB::GetProperty;
101
  virtual bool GetProperty(ColumnFamilyHandle* column_family,
102
                           const Slice& property, std::string* value);
103 104 105
  using DB::GetIntProperty;
  virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                              const Slice& property, uint64_t* value) override;
106
  using DB::GetApproximateSizes;
107
  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
108 109
                                   const Range* range, int n, uint64_t* sizes);
  using DB::CompactRange;
110
  virtual Status CompactRange(ColumnFamilyHandle* column_family,
111
                              const Slice* begin, const Slice* end,
112 113
                              bool reduce_level = false, int target_level = -1,
                              uint32_t target_path_id = 0);
114

115 116 117 118
  using DB::SetOptions;
  bool SetOptions(ColumnFamilyHandle* column_family,
      const std::unordered_map<std::string, std::string>& options_map);

119
  using DB::NumberLevels;
120
  virtual int NumberLevels(ColumnFamilyHandle* column_family);
121
  using DB::MaxMemCompactionLevel;
122
  virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family);
123
  using DB::Level0StopWriteTrigger;
124
  virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family);
I
Igor Canadi 已提交
125
  virtual const std::string& GetName() const;
126
  virtual Env* GetEnv() const;
127
  using DB::GetOptions;
128
  virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const;
129 130
  using DB::Flush;
  virtual Status Flush(const FlushOptions& options,
131
                       ColumnFamilyHandle* column_family);
I
Igor Canadi 已提交
132 133 134 135

  virtual SequenceNumber GetLatestSequenceNumber() const;

#ifndef ROCKSDB_LITE
136
  virtual Status DisableFileDeletions();
137
  virtual Status EnableFileDeletions(bool force);
138
  virtual int IsFileDeletionsEnabled() const;
I
Igor Canadi 已提交
139
  // All the returned filenames start with "/"
140
  virtual Status GetLiveFiles(std::vector<std::string>&,
141 142
                              uint64_t* manifest_file_size,
                              bool flush_memtable = true);
143
  virtual Status GetSortedWalFiles(VectorLogPtr& files);
I
Igor Canadi 已提交
144

145 146 147 148
  virtual Status GetUpdatesSince(
      SequenceNumber seq_number, unique_ptr<TransactionLogIterator>* iter,
      const TransactionLogIterator::ReadOptions&
          read_options = TransactionLogIterator::ReadOptions());
149 150
  virtual Status DeleteFile(std::string name);

151
  virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
I
Igor Canadi 已提交
152
#endif  // ROCKSDB_LITE
153

I
Igor Canadi 已提交
154 155 156 157
  // checks if all live files exist on file system and that their file sizes
  // match to our in-memory records
  virtual Status CheckConsistency();

158 159
  virtual Status GetDbIdentity(std::string& identity);

I
Igor Canadi 已提交
160
  Status RunManualCompaction(ColumnFamilyData* cfd, int input_level,
161 162
                             int output_level, uint32_t output_path_id,
                             const Slice* begin, const Slice* end);
163

164
#ifndef ROCKSDB_LITE
J
jorlow@chromium.org 已提交
165
  // Extra methods (for testing) that are not in the public DB interface
I
Igor Canadi 已提交
166
  // Implemented in db_impl_debug.cc
J
jorlow@chromium.org 已提交
167

168
  // Compact any files in the named level that overlap [*begin, *end]
169 170
  Status TEST_CompactRange(int level, const Slice* begin, const Slice* end,
                           ColumnFamilyHandle* column_family = nullptr);
J
jorlow@chromium.org 已提交
171

172
  // Force current memtable contents to be flushed.
173
  Status TEST_FlushMemTable(bool wait = true);
J
jorlow@chromium.org 已提交
174

175
  // Wait for memtable compaction
176
  Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);
177 178 179 180

  // Wait for any compaction
  Status TEST_WaitForCompact();

J
jorlow@chromium.org 已提交
181 182 183
  // Return an internal iterator over the current state of the database.
  // The keys of this iterator are internal keys (see format.h).
  // The returned iterator should be deleted when no longer needed.
184 185
  Iterator* TEST_NewInternalIterator(
      Arena* arena, ColumnFamilyHandle* column_family = nullptr);
J
jorlow@chromium.org 已提交
186

187 188
  // Return the maximum overlapping data (in bytes) at next level for any
  // file at a level >= 1.
189 190
  int64_t TEST_MaxNextLevelOverlappingBytes(ColumnFamilyHandle* column_family =
                                                nullptr);
191

A
Abhishek Kona 已提交
192 193
  // Return the current manifest file no.
  uint64_t TEST_Current_Manifest_FileNo();
194 195 196 197

  // Trigger's a background call for testing.
  void TEST_PurgeObsoleteteWAL();

198
  // get total level0 file size. Only for testing.
199
  uint64_t TEST_GetLevel0TotalSize();
200

201 202 203 204 205
  void TEST_SetDefaultTimeToCheck(uint64_t default_interval_to_delete_obsolete_WAL)
  {
    default_interval_to_delete_obsolete_WAL_ = default_interval_to_delete_obsolete_WAL;
  }

206 207
  void TEST_GetFilesMetaData(ColumnFamilyHandle* column_family,
                             std::vector<std::vector<FileMetaData>>* metadata);
208

I
Igor Canadi 已提交
209 210 211 212
  Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
                              SequenceNumber* sequence);

  Status TEST_ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
213 214 215 216 217 218 219 220 221 222 223

  void TEST_LockMutex();

  void TEST_UnlockMutex();

  // REQUIRES: mutex locked
  void* TEST_BeginWrite();

  // REQUIRES: mutex locked
  // pass the pointer that you got from TEST_BeginWrite()
  void TEST_EndWrite(void* w);
I
Igor Canadi 已提交
224 225
#endif  // NDEBUG

226 227 228 229 230 231 232 233 234 235 236
  // Structure to store information for candidate files to delete.
  struct CandidateFileInfo {
    std::string file_name;
    uint32_t path_id;
    CandidateFileInfo(std::string name, uint32_t path)
        : file_name(name), path_id(path) {}
    bool operator==(const CandidateFileInfo& other) const {
      return file_name == other.file_name && path_id == other.path_id;
    }
  };

I
Igor Canadi 已提交
237
  // needed for CleanupIteratorState
I
Igor Canadi 已提交
238 239
  struct DeletionState {
    inline bool HaveSomethingToDelete() const {
K
kailiu 已提交
240
      return  candidate_files.size() ||
I
Igor Canadi 已提交
241 242 243
        sst_delete_files.size() ||
        log_delete_files.size();
    }
244

I
Igor Canadi 已提交
245 246 247
    // a list of all files that we'll consider deleting
    // (every once in a while this is filled up with all files
    // in the DB directory)
248
    std::vector<CandidateFileInfo> candidate_files;
I
Igor Canadi 已提交
249 250

    // the list of all live sst files that cannot be deleted
251
    std::vector<FileDescriptor> sst_live;
I
Igor Canadi 已提交
252 253 254 255 256 257 258

    // a list of sst files that we need to delete
    std::vector<FileMetaData*> sst_delete_files;

    // a list of log files that we need to delete
    std::vector<uint64_t> log_delete_files;

259
    // a list of memtables to be free
K
Kai Liu 已提交
260
    autovector<MemTable*> memtables_to_free;
261

262
    autovector<SuperVersion*> superversions_to_free;
I
Igor Canadi 已提交
263

264
    SuperVersion* new_superversion;  // if nullptr no new superversion
I
Igor Canadi 已提交
265

I
Igor Canadi 已提交
266 267
    // the current manifest_file_number, log_number and prev_log_number
    // that corresponds to the set of files in 'live'.
268 269
    uint64_t manifest_file_number, pending_manifest_file_number, log_number,
        prev_log_number;
I
Igor Canadi 已提交
270

K
Kai Liu 已提交
271
    explicit DeletionState(bool create_superversion = false) {
I
Igor Canadi 已提交
272
      manifest_file_number = 0;
273
      pending_manifest_file_number = 0;
I
Igor Canadi 已提交
274 275
      log_number = 0;
      prev_log_number = 0;
276
      new_superversion = create_superversion ? new SuperVersion() : nullptr;
I
Igor Canadi 已提交
277 278 279 280 281 282 283
    }

    ~DeletionState() {
      // free pending memtables
      for (auto m : memtables_to_free) {
        delete m;
      }
284 285 286 287
      // free superversions
      for (auto s : superversions_to_free) {
        delete s;
      }
I
Igor Canadi 已提交
288 289 290
      // if new_superversion was not used, it will be non-nullptr and needs
      // to be freed here
      delete new_superversion;
I
Igor Canadi 已提交
291 292 293 294
    }
  };

  // Returns the list of live files in 'live' and the list
K
kailiu 已提交
295
  // of all files in the filesystem in 'candidate_files'.
I
Igor Canadi 已提交
296
  // If force == false and the last call was less than
297
  // db_options_.delete_obsolete_files_period_micros microseconds ago,
I
Igor Canadi 已提交
298 299 300 301 302 303 304 305 306 307 308
  // it will not fill up the deletion_state
  void FindObsoleteFiles(DeletionState& deletion_state,
                         bool force,
                         bool no_full_scan = false);

  // Diffs the files listed in filenames and those that do not
  // belong to live files are posibly removed. Also, removes all the
  // files in sst_delete_files and log_delete_files.
  // It is not necessary to hold the mutex when invoking this method.
  void PurgeObsoleteFiles(DeletionState& deletion_state);

309 310
  ColumnFamilyHandle* DefaultColumnFamily() const;

311
 protected:
H
heyongqiang 已提交
312 313
  Env* const env_;
  const std::string dbname_;
314
  unique_ptr<VersionSet> versions_;
315
  const DBOptions db_options_;
L
Lei Jin 已提交
316
  Statistics* stats_;
H
heyongqiang 已提交
317

318
  Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
319
                                SuperVersion* super_version, Arena* arena);
320

J
jorlow@chromium.org 已提交
321 322
 private:
  friend class DB;
323
  friend class InternalStats;
I
Igor Canadi 已提交
324
#ifndef ROCKSDB_LITE
L
Lei Jin 已提交
325
  friend class ForwardIterator;
I
Igor Canadi 已提交
326
#endif
327
  friend struct SuperVersion;
L
Lei Jin 已提交
328
  friend class CompactedDBImpl;
329
  struct CompactionState;
330

S
Stanislau Hlebik 已提交
331
  struct WriteContext;
J
jorlow@chromium.org 已提交
332 333 334 335 336 337

  Status NewDB();

  // Recover the descriptor from persistent storage.  May do a significant
  // amount of work to recover recently logged updates.  Any changes to
  // be made to the descriptor are added to *edit.
338 339
  Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
                 bool read_only = false, bool error_if_log_file_exist = false);
J
jorlow@chromium.org 已提交
340 341 342

  void MaybeIgnoreError(Status* s) const;

343 344
  const Status CreateArchivalDirectory();

J
jorlow@chromium.org 已提交
345 346 347
  // Delete any unneeded files and stale in-memory entries.
  void DeleteObsoleteFiles();

348
  // Flush the in-memory write buffer to storage.  Switches to a new
J
jorlow@chromium.org 已提交
349
  // log-file/memtable and writes a new descriptor iff successful.
350 351 352
  Status FlushMemTableToOutputFile(
      ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
      bool* madeProgress, DeletionState& deletion_state, LogBuffer* log_buffer);
J
jorlow@chromium.org 已提交
353

S
Stanislau Hlebik 已提交
354 355 356
  // REQUIRES: log_numbers are sorted in ascending order
  Status RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
                         SequenceNumber* max_sequence, bool read_only);
J
jorlow@chromium.org 已提交
357

358 359 360 361 362
  // The following two methods are used to flush a memtable to
  // storage. The first one is used atdatabase RecoveryTime (when the
  // database is opened) and is heavyweight because it holds the mutex
  // for the entire period. The second method WriteLevel0Table supports
  // concurrent flush memtables to storage.
363 364
  Status WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
                                     VersionEdit* edit);
365 366 367 368
  Status WriteLevel0Table(ColumnFamilyData* cfd,
      const MutableCFOptions& mutable_cf_options,
      const autovector<MemTable*>& mems,
      VersionEdit* edit, uint64_t* filenumber, LogBuffer* log_buffer);
S
Stanislau Hlebik 已提交
369

370 371
  void DelayWrite(uint64_t expiration_time);

I
Igor Canadi 已提交
372
  Status ScheduleFlushes(WriteContext* context);
373

S
Stanislau Hlebik 已提交
374 375 376
  Status SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
                                     WriteContext* context);

H
heyongqiang 已提交
377
  // Force current memtable contents to be flushed.
378
  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options);
H
heyongqiang 已提交
379

380
  // Wait for memtable flushed
381
  Status WaitForFlushMemTable(ColumnFamilyData* cfd);
H
heyongqiang 已提交
382

383 384 385
  void RecordFlushIOStats();
  void RecordCompactionIOStats();

386
  void MaybeScheduleFlushOrCompaction();
387 388 389 390
  static void BGWorkCompaction(void* db);
  static void BGWorkFlush(void* db);
  void BackgroundCallCompaction();
  void BackgroundCallFlush();
391 392
  Status BackgroundCompaction(bool* madeProgress, DeletionState& deletion_state,
                              LogBuffer* log_buffer);
H
Haobo Xu 已提交
393 394
  Status BackgroundFlush(bool* madeProgress, DeletionState& deletion_state,
                         LogBuffer* log_buffer);
395
  void CleanupCompaction(CompactionState* compact, Status status);
I
Igor Canadi 已提交
396
  Status DoCompactionWork(CompactionState* compact,
397
                          const MutableCFOptions& mutable_cf_options,
H
Haobo Xu 已提交
398 399
                          DeletionState& deletion_state,
                          LogBuffer* log_buffer);
J
jorlow@chromium.org 已提交
400

I
Igor Canadi 已提交
401 402 403 404
  // This function is called as part of compaction. It enables Flush process to
  // preempt compaction, since it's higher prioirty
  // Returns: micros spent executing
  uint64_t CallFlushDuringCompaction(ColumnFamilyData* cfd,
405 406
      const MutableCFOptions& mutable_cf_options, DeletionState& deletion_state,
      LogBuffer* log_buffer);
I
Igor Canadi 已提交
407

D
Danny Guo 已提交
408 409 410
  // Call compaction filter if is_compaction_v2 is not true. Then iterate
  // through input and compact the kv-pairs
  Status ProcessKeyValueCompaction(
411
    const MutableCFOptions& mutable_cf_options,
412
    bool is_snapshot_supported,
D
Danny Guo 已提交
413 414 415 416 417 418 419 420 421
    SequenceNumber visible_at_tip,
    SequenceNumber earliest_snapshot,
    SequenceNumber latest_snapshot,
    DeletionState& deletion_state,
    bool bottommost_level,
    int64_t& imm_micros,
    Iterator* input,
    CompactionState* compact,
    bool is_compaction_v2,
422
    int* num_output_records,
D
Danny Guo 已提交
423 424 425 426 427 428
    LogBuffer* log_buffer);

  // Call compaction_filter_v2->Filter() on kv-pairs in compact
  void CallCompactionFilterV2(CompactionState* compact,
    CompactionFilterV2* compaction_filter_v2);

429 430
  Status OpenCompactionOutputFile(CompactionState* compact,
      const MutableCFOptions& mutable_cf_options);
J
jorlow@chromium.org 已提交
431
  Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
432
  Status InstallCompactionResults(CompactionState* compact,
433
      const MutableCFOptions& mutable_cf_options, LogBuffer* log_buffer);
434 435
  void AllocateCompactionOutputFileNumbers(CompactionState* compact);
  void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
A
Abhishek Kona 已提交
436

I
Igor Canadi 已提交
437 438 439 440 441 442
#ifdef ROCKSDB_LITE
  void PurgeObsoleteWALFiles() {
    // this function is used for archiving WAL files. we don't need this in
    // ROCKSDB_LITE
  }
#else
443
  void PurgeObsoleteWALFiles();
444

445 446 447
  Status GetSortedWalsOfType(const std::string& path,
                             VectorLogPtr& log_files,
                             WalFileType type);
448

449 450 451 452 453
  // Requires: all_logs should be sorted with earliest log file first
  // Retains all log files in all_logs which contain updates with seq no.
  // Greater Than or Equal to the requested SequenceNumber.
  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
                                const SequenceNumber target);
454

455
  Status ReadFirstRecord(const WalFileType type, const uint64_t number,
I
Igor Canadi 已提交
456
                         SequenceNumber* sequence);
457

I
Igor Canadi 已提交
458
  Status ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
I
Igor Canadi 已提交
459
#endif  // ROCKSDB_LITE
460

461 462
  void PrintStatistics();

463
  // dump rocksdb.stats to LOG
464 465
  void MaybeDumpStats();

466 467 468 469 470 471 472
  // Return true if the current db supports snapshot.  If the current
  // DB does not support snapshot, then calling GetSnapshot() will always
  // return nullptr.
  //
  // @see GetSnapshot()
  virtual bool IsSnapshotSupported() const;

473 474
  // Return the minimum empty level that could hold the total data in the
  // input level. Return the input level, if such level could not be found.
475 476
  int FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
      const MutableCFOptions& mutable_cf_options, int level);
477

478 479 480
  // Move the files in the input level to the target level.
  // If target_level < 0, automatically calculate the minimum level that could
  // hold the data set.
I
Igor Canadi 已提交
481
  Status ReFitLevel(ColumnFamilyData* cfd, int level, int target_level = -1);
482

J
jorlow@chromium.org 已提交
483
  // table_cache_ provides its own synchronization
I
Igor Canadi 已提交
484
  std::shared_ptr<Cache> table_cache_;
J
jorlow@chromium.org 已提交
485

486
  // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
J
jorlow@chromium.org 已提交
487 488 489 490 491
  FileLock* db_lock_;

  // State below is protected by mutex_
  port::Mutex mutex_;
  port::AtomicPointer shutting_down_;
492 493 494 495 496 497 498 499 500
  // This condition variable is signaled on these conditions:
  // * whenever bg_compaction_scheduled_ goes down to 0
  // * if bg_manual_only_ > 0, whenever a compaction finishes, even if it hasn't
  // made any progress
  // * whenever a compaction made any progress
  // * whenever bg_flush_scheduled_ value decreases (i.e. whenever a flush is
  // done, even if it didn't make any progress)
  // * whenever there is an error in background flush or compaction
  port::CondVar bg_cv_;
501
  uint64_t logfile_number_;
502
  unique_ptr<log::Writer> log_;
I
Igor Canadi 已提交
503
  bool log_empty_;
504
  ColumnFamilyHandleImpl* default_cf_handle_;
505
  InternalStats* default_cf_internal_stats_;
506
  unique_ptr<ColumnFamilyMemTablesImpl> column_family_memtables_;
I
Igor Canadi 已提交
507 508 509 510 511 512 513 514 515 516 517 518 519
  struct LogFileNumberSize {
    explicit LogFileNumberSize(uint64_t _number)
        : number(_number), size(0), getting_flushed(false) {}
    void AddSize(uint64_t new_size) { size += new_size; }
    uint64_t number;
    uint64_t size;
    bool getting_flushed;
  };
  std::deque<LogFileNumberSize> alive_log_files_;
  uint64_t total_log_size_;
  // only used for dynamically adjusting max_total_wal_size. it is a sum of
  // [write_buffer_size * max_write_buffer_number] over all column families
  uint64_t max_total_in_memory_state_;
520 521 522
  // If true, we have only one (default) column family. We use this to optimize
  // some code-paths
  bool single_column_family_mode_;
I
Igor Canadi 已提交
523

524 525
  std::unique_ptr<Directory> db_directory_;

I
Igor Canadi 已提交
526 527
  WriteThread write_thread_;

528
  WriteBatch tmp_batch_;
529

530
  WriteController write_controller_;
I
Igor Canadi 已提交
531
  FlushScheduler flush_scheduler_;
532

J
jorlow@chromium.org 已提交
533 534
  SnapshotList snapshots_;

I
Igor Canadi 已提交
535 536 537 538
  // cache for ReadFirstRecord() calls
  std::unordered_map<uint64_t, SequenceNumber> read_first_record_cache_;
  port::Mutex read_first_record_cache_mutex_;

J
jorlow@chromium.org 已提交
539 540
  // Set of table files to protect from deletion because they are
  // part of ongoing compactions.
541 542
  // map from pending file number ID to their path IDs.
  FileNumToPathIdMap pending_outputs_;
J
jorlow@chromium.org 已提交
543

544 545 546 547
  // At least one compaction or flush job is pending but not yet scheduled
  // because of the max background thread limit.
  bool bg_schedule_needed_;

548
  // count how many background compactions are running or have been scheduled
549
  int bg_compaction_scheduled_;
J
jorlow@chromium.org 已提交
550

551 552 553 554 555
  // If non-zero, MaybeScheduleFlushOrCompaction() will only schedule manual
  // compactions (if manual_compaction_ is not null). This mechanism enables
  // manual compactions to wait until all other compactions are finished.
  int bg_manual_only_;

556 557 558
  // number of background memtable flush jobs, submitted to the HIGH pool
  int bg_flush_scheduled_;

H
hans@chromium.org 已提交
559 560
  // Information for a manual compaction
  struct ManualCompaction {
I
Igor Canadi 已提交
561
    ColumnFamilyData* cfd;
562 563
    int input_level;
    int output_level;
564
    uint32_t output_path_id;
G
Gabor Cselle 已提交
565
    bool done;
L
Lei Jin 已提交
566
    Status status;
567
    bool in_progress;           // compaction request being processed?
568 569
    const InternalKey* begin;   // nullptr means beginning of key range
    const InternalKey* end;     // nullptr means end of key range
G
Gabor Cselle 已提交
570
    InternalKey tmp_storage;    // Used to keep track of compaction progress
H
hans@chromium.org 已提交
571 572
  };
  ManualCompaction* manual_compaction_;
J
jorlow@chromium.org 已提交
573 574 575 576

  // Have we encountered a background error in paranoid mode?
  Status bg_error_;

577
  // shall we disable deletion of obsolete files
578 579 580 581 582 583
  // if 0 the deletion is enabled.
  // if non-zero, files will not be getting deleted
  // This enables two different threads to call
  // EnableFileDeletions() and DisableFileDeletions()
  // without any synchronization
  int disable_delete_obsolete_files_;
584

585 586 587
  // last time when DeleteObsoleteFiles was invoked
  uint64_t delete_obsolete_files_last_run_;

588 589 590
  // last time when PurgeObsoleteWALFiles ran.
  uint64_t purge_wal_files_last_run_;

591
  // last time stats were dumped to LOG
H
Haobo Xu 已提交
592
  std::atomic<uint64_t> last_stats_dump_time_microsec_;
593

594 595 596 597
  // obsolete files will be deleted every this seconds if ttl deletion is
  // enabled and archive size_limit is disabled.
  uint64_t default_interval_to_delete_obsolete_WAL_;

598 599
  bool flush_on_destroy_; // Used when disableWAL is true.

H
heyongqiang 已提交
600
  static const int KEEP_LOG_FILE_NUM = 1000;
H
heyongqiang 已提交
601
  std::string db_absolute_path_;
H
heyongqiang 已提交
602

603
  // The options to access storage files
L
Lei Jin 已提交
604
  const EnvOptions env_options_;
605

606 607 608 609 610 611
  // A value of true temporarily disables scheduling of background work
  bool bg_work_gate_closed_;

  // Guard against multiple concurrent refitting
  bool refitting_level_;

612 613 614
  // Indicate DB was opened successfully
  bool opened_successfully_;

J
jorlow@chromium.org 已提交
615 616 617 618
  // No copying allowed
  DBImpl(const DBImpl&);
  void operator=(const DBImpl&);

619 620 621 622 623 624
  // Return the earliest snapshot where seqno is visible.
  // Store the snapshot right before that, if any, in prev_snapshot
  inline SequenceNumber findEarliestVisibleSnapshot(
    SequenceNumber in,
    std::vector<SequenceNumber>& snapshots,
    SequenceNumber* prev_snapshot);
625

I
Igor Canadi 已提交
626
  // Background threads call this function, which is just a wrapper around
627
  // the cfd->InstallSuperVersion() function. Background threads carry
I
Igor Canadi 已提交
628
  // deletion_state which can have new_superversion already allocated.
629
  void InstallSuperVersion(ColumnFamilyData* cfd,
630 631
                           DeletionState& deletion_state,
                           const MutableCFOptions& mutable_cf_options);
I
Igor Canadi 已提交
632

633 634 635 636 637 638 639 640 641 642
  // Find Super version and reference it. Based on options, it might return
  // the thread local cached one.
  inline SuperVersion* GetAndRefSuperVersion(ColumnFamilyData* cfd);

  // Un-reference the super version and return it to thread local cache if
  // needed. If it is the last reference of the super version. Clean it up
  // after un-referencing it.
  inline void ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd,
                                           SuperVersion* sv);

I
Igor Canadi 已提交
643
#ifndef ROCKSDB_LITE
I
Igor Canadi 已提交
644 645 646
  using DB::GetPropertiesOfAllTables;
  virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
                                          TablePropertiesCollection* props)
647
      override;
I
Igor Canadi 已提交
648
#endif  // ROCKSDB_LITE
649

650 651
  // Function that Get and KeyMayExist call with no_io true or false
  // Note: 'value_found' from KeyMayExist propagates here
652 653 654
  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
                 const Slice& key, std::string* value,
                 bool* value_found = nullptr);
655 656 657 658

  bool GetIntPropertyInternal(ColumnFamilyHandle* column_family,
                              DBPropertyType property_type,
                              bool need_out_of_mutex, uint64_t* value);
J
jorlow@chromium.org 已提交
659 660 661 662 663 664 665
};

// Sanitize db options.  The caller should delete result.info_log if
// it is not equal to src.info_log.
extern Options SanitizeOptions(const std::string& db,
                               const InternalKeyComparator* icmp,
                               const Options& src);
666
extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
S
Siying Dong 已提交
667

M
miguelportilla 已提交
668 669 670 671 672 673 674
// Fix user-supplied options to be reasonable
template <class T, class V>
static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
  if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
  if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
}

F
Feng Zhu 已提交
675 676 677 678
// Dump db file summary, implemented in util/
extern void DumpDBFileSummary(const DBOptions& options,
                              const std::string& dbname);

679
}  // namespace rocksdb