disk_buffer_pool.cpp 20.8 KB
Newer Older
羽飞's avatar
羽飞 已提交
1 2 3 4 5 6 7 8 9 10 11
/* Copyright (c) 2021 Xie Meiyi(xiemeiyi@hust.edu.cn) and OceanBase and/or its affiliates. All rights reserved.
miniob is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:
         http://license.coscl.org.cn/MulanPSL2
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details. */

//
12
// Created by Meiyi & Longda on 2021/4/13.
羽飞's avatar
羽飞 已提交
13 14 15 16
//
#include <errno.h>
#include <string.h>

17
#include "storage/default/disk_buffer_pool.h"
L
Longda 已提交
18
#include "common/lang/mutex.h"
羽飞's avatar
羽飞 已提交
19
#include "common/log/log.h"
L
Longda 已提交
20
#include "common/os/os.h"
羽飞's avatar
羽飞 已提交
21
#include "common/io/io.h"
羽飞's avatar
羽飞 已提交
22 23 24

using namespace common;

羽飞's avatar
羽飞 已提交
25
static const int MEM_POOL_ITEM_NUM = 128;
L
Longda 已提交
26

27
////////////////////////////////////////////////////////////////////////////////
羽飞's avatar
羽飞 已提交
28

羽飞's avatar
羽飞 已提交
29
BPFrameManager::BPFrameManager(const char *name) : allocator_(name)
L
Longda 已提交
30 31
{}

羽飞's avatar
羽飞 已提交
32
RC BPFrameManager::init(int pool_num)
L
Longda 已提交
33
{
L
Longda Feng 已提交
34
  int ret = allocator_.init(false, pool_num);
羽飞's avatar
羽飞 已提交
35 36 37 38
  if (ret == 0) {
    return RC::SUCCESS;
  }
  return RC::GENERIC_ERROR;
L
Longda 已提交
39 40
}

羽飞's avatar
羽飞 已提交
41
RC BPFrameManager::cleanup()
L
Longda 已提交
42
{
羽飞's avatar
羽飞 已提交
43 44
  if (frames_.count() > 0) {
    return RC::GENERIC_ERROR;
L
Longda 已提交
45 46
  }

羽飞's avatar
羽飞 已提交
47 48 49 50
  frames_.destroy();
  return RC::SUCCESS;
}

51
int BPFrameManager::purge_frames(int count, std::function<RC(Frame *frame)> purger)
L
Longda 已提交
52
{
53 54 55 56 57 58 59 60 61
  std::lock_guard<std::mutex> lock_guard(lock_);

  std::vector<Frame *> frames_can_purge;
  if (count <= 0) {
    count = 1;
  }
  frames_can_purge.reserve(count);

  auto purge_finder = [&frames_can_purge, count](const FrameId &frame_id, Frame *const frame) {
羽飞's avatar
羽飞 已提交
62
    if (frame->can_purge()) {
63 64 65 66 67
      frame->pin();
      frames_can_purge.push_back(frame);
      if (frames_can_purge.size() >= static_cast<size_t>(count)) {
        return false;  // false to break the progress
      }
羽飞's avatar
羽飞 已提交
68
    }
L
Longda Feng 已提交
69
    return true;  // true continue to look up
羽飞's avatar
羽飞 已提交
70
  };
71

羽飞's avatar
羽飞 已提交
72
  frames_.foreach_reverse(purge_finder);
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
  LOG_INFO("purge frames find %ld pages total", frames_can_purge.size());

  /// 当前还在frameManager的锁内,而 purger 是一个非常耗时的操作
  /// 他需要把脏页数据刷新到磁盘上去,所以这里会极大地降低并发度
  int freed_count = 0;
  for (Frame *frame : frames_can_purge) {
    RC rc = purger(frame);
    if (RC::SUCCESS == rc) {
      free_internal(frame->frame_id(), frame);
      freed_count++;
    } else {
      frame->unpin();
      LOG_WARN("failed to purge frame. frame_id=%s, rc=%s", 
               to_string(frame->frame_id()).c_str(), strrc(rc));
    }
  }
  LOG_INFO("purge frame done. number=%d", freed_count);
  return freed_count;
羽飞's avatar
羽飞 已提交
91
}
L
Longda 已提交
92

羽飞's avatar
羽飞 已提交
93 94
Frame *BPFrameManager::get(int file_desc, PageNum page_num)
{
95
  FrameId frame_id(file_desc, page_num);
羽飞's avatar
羽飞 已提交
96
  std::lock_guard<std::mutex> lock_guard(lock_);
97 98 99 100 101
  return get_internal(frame_id);
}

Frame *BPFrameManager::get_internal(const FrameId &frame_id)
{
羽飞's avatar
羽飞 已提交
102 103
  Frame *frame = nullptr;
  (void)frames_.get(frame_id, frame);
104 105 106
  if (frame != nullptr) {
    frame->pin();
  }
羽飞's avatar
羽飞 已提交
107
  return frame;
L
Longda 已提交
108 109
}

羽飞's avatar
羽飞 已提交
110
Frame *BPFrameManager::alloc(int file_desc, PageNum page_num)
L
Longda 已提交
111
{
112
  FrameId frame_id(file_desc, page_num);
羽飞's avatar
羽飞 已提交
113 114

  std::lock_guard<std::mutex> lock_guard(lock_);
115 116 117
  Frame *frame = get_internal(frame_id);
  if (frame != nullptr) {
    return frame;
羽飞's avatar
羽飞 已提交
118 119 120 121
  }

  frame = allocator_.alloc();
  if (frame != nullptr) {
122 123 124
    ASSERT(frame->pin_count() == 0, "got an invalid frame that pin count is not 0. frame=%s", 
           to_string(*frame).c_str());
    frame->pin();
羽飞's avatar
羽飞 已提交
125 126 127
    frames_.put(frame_id, frame);
  }
  return frame;
L
Longda 已提交
128 129
}

羽飞's avatar
羽飞 已提交
130
RC BPFrameManager::free(int file_desc, PageNum page_num, Frame *frame)
L
Longda 已提交
131
{
132
  FrameId frame_id(file_desc, page_num);
L
Longda 已提交
133

羽飞's avatar
羽飞 已提交
134
  std::lock_guard<std::mutex> lock_guard(lock_);
135 136 137 138 139
  return free_internal(frame_id, frame);
}

RC BPFrameManager::free_internal(const FrameId &frame_id, Frame *frame)
{
羽飞's avatar
羽飞 已提交
140 141
  Frame *frame_source = nullptr;
  bool found = frames_.get(frame_id, frame_source);
142 143 144
  ASSERT(found && frame == frame_source && frame->pin_count() == 1,
         "failed to free frame. found=%d, frameId=%s, frame_source=%p, frame=%p, pinCount=%d, lbt=%s",
         found, to_string(frame_id).c_str(), frame_source, frame, frame->pin_count(), lbt());
L
Longda 已提交
145

146
  frame->unpin();
羽飞's avatar
羽飞 已提交
147 148 149
  frames_.remove(frame_id);
  allocator_.free(frame);
  return RC::SUCCESS;
L
Longda 已提交
150 151
}

羽飞's avatar
羽飞 已提交
152
std::list<Frame *> BPFrameManager::find_list(int file_desc)
L
Longda 已提交
153
{
羽飞's avatar
羽飞 已提交
154 155 156
  std::lock_guard<std::mutex> lock_guard(lock_);

  std::list<Frame *> frames;
157
  auto fetcher = [&frames, file_desc](const FrameId &frame_id, Frame *const frame) -> bool {
羽飞's avatar
羽飞 已提交
158
    if (file_desc == frame_id.file_desc()) {
159
      frame->pin();
羽飞's avatar
羽飞 已提交
160 161 162 163
      frames.push_back(frame);
    }
    return true;
  };
L
Longda Feng 已提交
164
  frames_.foreach (fetcher);
羽飞's avatar
羽飞 已提交
165
  return frames;
L
Longda 已提交
166 167
}

羽飞's avatar
羽飞 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
////////////////////////////////////////////////////////////////////////////////
BufferPoolIterator::BufferPoolIterator()
{}
BufferPoolIterator::~BufferPoolIterator()
{}
RC BufferPoolIterator::init(DiskBufferPool &bp, PageNum start_page /* = 0 */)
{
  bitmap_.init(bp.file_header_->bitmap, bp.file_header_->page_count);
  if (start_page <= 0) {
    current_page_num_ = 0;
  } else {
    current_page_num_ = start_page;
  }
  return RC::SUCCESS;
}

bool BufferPoolIterator::has_next()
{
羽飞's avatar
羽飞 已提交
186
  return bitmap_.next_setted_bit(current_page_num_ + 1) != -1;
羽飞's avatar
羽飞 已提交
187 188 189 190
}

PageNum BufferPoolIterator::next()
{
羽飞's avatar
羽飞 已提交
191
  PageNum next_page = bitmap_.next_setted_bit(current_page_num_ + 1);
羽飞's avatar
羽飞 已提交
192 193 194 195 196 197 198 199 200 201 202 203 204
  if (next_page != -1) {
    current_page_num_ = next_page;
  }
  return next_page;
}

RC BufferPoolIterator::reset()
{
  current_page_num_ = 0;
  return RC::SUCCESS;
}

////////////////////////////////////////////////////////////////////////////////
羽飞's avatar
羽飞 已提交
205
DiskBufferPool::DiskBufferPool(BufferPoolManager &bp_manager, BPFrameManager &frame_manager)
L
Longda Feng 已提交
206 207
    : bp_manager_(bp_manager), frame_manager_(frame_manager)
{}
羽飞's avatar
羽飞 已提交
208

L
Longda 已提交
209 210
DiskBufferPool::~DiskBufferPool()
{
羽飞's avatar
羽飞 已提交
211
  close_file();
212
  LOG_INFO("disk buffer pool exit");
L
Longda 已提交
213 214
}

羽飞's avatar
羽飞 已提交
215
RC DiskBufferPool::open_file(const char *file_name)
羽飞's avatar
羽飞 已提交
216
{
217 218
  int fd = open(file_name, O_RDWR);
  if (fd < 0) {
羽飞's avatar
羽飞 已提交
219 220 221
    LOG_ERROR("Failed to open file %s, because %s.", file_name, strerror(errno));
    return RC::IOERR_ACCESS;
  }
222
  LOG_INFO("Successfully open buffer pool file %s.", file_name);
羽飞's avatar
羽飞 已提交
223

羽飞's avatar
羽飞 已提交
224 225
  file_name_ = file_name;
  file_desc_ = fd;
羽飞's avatar
羽飞 已提交
226

羽飞's avatar
羽飞 已提交
227
  RC rc = RC::SUCCESS;
羽飞's avatar
羽飞 已提交
228
  rc = allocate_frame(BP_HEADER_PAGE, &hdr_frame_);
羽飞's avatar
羽飞 已提交
229 230
  if (rc != RC::SUCCESS) {
    LOG_ERROR("failed to allocate frame for header. file name %s", file_name_.c_str());
羽飞's avatar
羽飞 已提交
231
    close(fd);
羽飞's avatar
羽飞 已提交
232 233
    file_desc_ = -1;
    return rc;
羽飞's avatar
羽飞 已提交
234
  }
羽飞's avatar
羽飞 已提交
235

236 237 238
  hdr_frame_->set_file_desc(fd);
  hdr_frame_->access();

羽飞's avatar
羽飞 已提交
239
  if ((rc = load_page(BP_HEADER_PAGE, hdr_frame_)) != RC::SUCCESS) {
L
Longda 已提交
240
    LOG_ERROR("Failed to load first page of %s, due to %s.", file_name, strerror(errno));
羽飞's avatar
羽飞 已提交
241
    purge_frame(BP_HEADER_PAGE, hdr_frame_);
羽飞's avatar
羽飞 已提交
242
    close(fd);
羽飞's avatar
羽飞 已提交
243 244
    file_desc_ = -1;
    return rc;
羽飞's avatar
羽飞 已提交
245 246
  }

羽飞's avatar
羽飞 已提交
247
  file_header_ = (BPFileHeader *)hdr_frame_->data();
L
Longda 已提交
248

羽飞's avatar
羽飞 已提交
249
  LOG_INFO("Successfully open %s. file_desc=%d, hdr_frame=%p", file_name, file_desc_, hdr_frame_);
羽飞's avatar
羽飞 已提交
250 251 252
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
253
RC DiskBufferPool::close_file()
羽飞's avatar
羽飞 已提交
254
{
羽飞's avatar
羽飞 已提交
255 256 257
  RC rc = RC::SUCCESS;
  if (file_desc_ < 0) {
    return rc;
羽飞's avatar
羽飞 已提交
258 259
  }

260 261
  hdr_frame_->unpin();

羽飞's avatar
羽飞 已提交
262
  // TODO: 理论上是在回放时回滚未提交事务,但目前没有undo log,因此不下刷数据page,只通过redo log回放
263 264 265
  rc = purge_all_pages();
  if (rc != RC::SUCCESS) {
    LOG_ERROR("failed to close %s, due to failed to purge pages. rc=%s", file_name_.c_str(), strrc(rc));
羽飞's avatar
羽飞 已提交
266
    return rc;
羽飞's avatar
羽飞 已提交
267 268
  }

269
  disposed_pages_.clear();
L
Longda 已提交
270

羽飞's avatar
羽飞 已提交
271 272
  if (close(file_desc_) < 0) {
    LOG_ERROR("Failed to close fileId:%d, fileName:%s, error:%s", file_desc_, file_name_.c_str(), strerror(errno));
羽飞's avatar
羽飞 已提交
273 274
    return RC::IOERR_CLOSE;
  }
羽飞's avatar
羽飞 已提交
275 276 277 278
  LOG_INFO("Successfully close file %d:%s.", file_desc_, file_name_.c_str());
  file_desc_ = -1;

  bp_manager_.close_file(file_name_.c_str());
羽飞's avatar
羽飞 已提交
279 280 281
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
282
RC DiskBufferPool::get_this_page(PageNum page_num, Frame **frame)
羽飞's avatar
羽飞 已提交
283
{
羽飞's avatar
羽飞 已提交
284
  RC rc = RC::SUCCESS;
285
  *frame = nullptr;
羽飞's avatar
羽飞 已提交
286

羽飞's avatar
羽飞 已提交
287
  Frame *used_match_frame = frame_manager_.get(file_desc_, page_num);
L
Longda 已提交
288
  if (used_match_frame != nullptr) {
289
    used_match_frame->access();
羽飞's avatar
羽飞 已提交
290
    *frame = used_match_frame;
L
Longda 已提交
291
    return RC::SUCCESS;
羽飞's avatar
羽飞 已提交
292 293
  }

294 295
  std::scoped_lock lock_guard(lock_); // 直接加了一把大锁,其实可以根据访问的页面来细化提高并行度

羽飞's avatar
羽飞 已提交
296
  // Allocate one page and load the data into this page
羽飞's avatar
羽飞 已提交
297
  Frame *allocated_frame = nullptr;
298 299
  rc = allocate_frame(page_num, &allocated_frame);
  if (rc != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
300 301 302 303
    LOG_ERROR("Failed to alloc frame %s:%d, due to failed to alloc page.", file_name_.c_str(), page_num);
    return rc;
  }

304 305 306 307
  allocated_frame->set_file_desc(file_desc_);
  // allocated_frame->pin(); // pined in manager::get
  allocated_frame->access();

羽飞's avatar
羽飞 已提交
308 309
  if ((rc = load_page(page_num, allocated_frame)) != RC::SUCCESS) {
    LOG_ERROR("Failed to load page %s:%d", file_name_.c_str(), page_num);
羽飞's avatar
羽飞 已提交
310
    purge_frame(page_num, allocated_frame);
羽飞's avatar
羽飞 已提交
311 312 313 314
    return rc;
  }

  *frame = allocated_frame;
羽飞's avatar
羽飞 已提交
315 316 317
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
318
RC DiskBufferPool::allocate_page(Frame **frame)
羽飞's avatar
羽飞 已提交
319
{
羽飞's avatar
羽飞 已提交
320
  RC rc = RC::SUCCESS;
羽飞's avatar
羽飞 已提交
321

322 323
  lock_.lock();
  
羽飞's avatar
羽飞 已提交
324
  int byte = 0, bit = 0;
羽飞's avatar
羽飞 已提交
325
  if ((file_header_->allocated_pages) < (file_header_->page_count)) {
羽飞's avatar
羽飞 已提交
326
    // There is one free page
羽飞's avatar
羽飞 已提交
327
    for (int i = 0; i < file_header_->page_count; i++) {
羽飞's avatar
羽飞 已提交
328 329
      byte = i / 8;
      bit = i % 8;
羽飞's avatar
羽飞 已提交
330 331 332
      if (((file_header_->bitmap[byte]) & (1 << bit)) == 0) {
        (file_header_->allocated_pages)++;
        file_header_->bitmap[byte] |= (1 << bit);
L
Longda 已提交
333
        // TODO,  do we need clean the loaded page's data?
L
Longda Feng 已提交
334
        hdr_frame_->mark_dirty();
335 336

        lock_.unlock();
羽飞's avatar
羽飞 已提交
337
        return get_this_page(i, frame);
羽飞's avatar
羽飞 已提交
338 339 340 341
      }
    }
  }

羽飞's avatar
羽飞 已提交
342 343
  if (file_header_->page_count >= BPFileHeader::MAX_PAGE_NUM) {
    LOG_WARN("file buffer pool is full. page count %d, max page count %d",
344 345 346
        file_header_->page_count, BPFileHeader::MAX_PAGE_NUM);
    lock_.unlock();
    return RC::BUFFERPOOL_NOBUF;
羽飞's avatar
羽飞 已提交
347 348 349
  }

  PageNum page_num = file_header_->page_count;
羽飞's avatar
羽飞 已提交
350
  Frame *allocated_frame = nullptr;
羽飞's avatar
羽飞 已提交
351
  if ((rc = allocate_frame(page_num, &allocated_frame)) != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
352
    LOG_ERROR("Failed to allocate frame %s, due to no free page.", file_name_.c_str());
353
    lock_.unlock();
羽飞's avatar
羽飞 已提交
354
    return rc;
羽飞's avatar
羽飞 已提交
355 356
  }

羽飞's avatar
羽飞 已提交
357 358
  file_header_->allocated_pages++;
  file_header_->page_count++;
羽飞's avatar
羽飞 已提交
359 360 361

  byte = page_num / 8;
  bit = page_num % 8;
羽飞's avatar
羽飞 已提交
362 363
  file_header_->bitmap[byte] |= (1 << bit);
  hdr_frame_->mark_dirty();
羽飞's avatar
羽飞 已提交
364

365 366
  allocated_frame->set_file_desc(file_desc_);
  allocated_frame->access();
羽飞's avatar
羽飞 已提交
367
  allocated_frame->clear_page();
368
  allocated_frame->set_page_num(file_header_->page_count - 1);
羽飞's avatar
羽飞 已提交
369

L
Longda 已提交
370
  // Use flush operation to extension file
371
  if ((rc = flush_page_internal(*allocated_frame)) != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
372
    LOG_WARN("Failed to alloc page %s , due to failed to extend one page.", file_name_.c_str());
L
Longda 已提交
373 374
    // skip return false, delay flush the extended page
    // return tmp;
羽飞's avatar
羽飞 已提交
375 376
  }

377
  lock_.unlock();
L
Longda 已提交
378

379
  *frame = allocated_frame;
羽飞's avatar
羽飞 已提交
380 381 382
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
383
RC DiskBufferPool::dispose_page(PageNum page_num)
羽飞's avatar
羽飞 已提交
384
{
385 386 387 388 389 390 391 392
  std::scoped_lock lock_guard(lock_);
  Frame *used_frame = frame_manager_.get(file_desc_, page_num);
  if (used_frame != nullptr) {
    ASSERT("the page try to dispose is in use. frame:%s", to_string(*used_frame).c_str());
    frame_manager_.free(file_desc_, page_num, used_frame);
  } else {
    LOG_WARN("failed to fetch the page while disposing it. pageNum=%d", page_num);
    return RC::NOTFOUND;
羽飞's avatar
羽飞 已提交
393 394
  }

395
  hdr_frame_->mark_dirty();
羽飞's avatar
羽飞 已提交
396
  file_header_->allocated_pages--;
羽飞's avatar
羽飞 已提交
397
  char tmp = 1 << (page_num % 8);
羽飞's avatar
羽飞 已提交
398
  file_header_->bitmap[page_num / 8] &= ~tmp;
羽飞's avatar
羽飞 已提交
399 400 401
  return RC::SUCCESS;
}

402 403 404 405 406 407
RC DiskBufferPool::unpin_page(Frame *frame)
{
  frame->unpin();
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
408
RC DiskBufferPool::purge_frame(PageNum page_num, Frame *buf)
羽飞's avatar
羽飞 已提交
409
{
410 411 412
  if (buf->pin_count() != 1) {
    LOG_INFO("Begin to free page %d of %d(file id), but it's pin count > 1:%d.",
        buf->page_num(), buf->file_desc(), buf->pin_count());
L
Longda 已提交
413 414 415
    return RC::LOCKED_UNLOCK;
  }

416 417
  if (buf->dirty()) {
    RC rc = flush_page_internal(*buf);
L
Longda 已提交
418
    if (rc != RC::SUCCESS) {
419
      LOG_WARN("Failed to flush page %d of %d(file desc) during purge page.", buf->page_num(), buf->file_desc());
L
Longda 已提交
420 421 422 423
      return rc;
    }
  }

424
  LOG_DEBUG("Successfully purge frame =%p, page %d of %d(file desc)", buf, buf->page_num(), buf->file_desc());
羽飞's avatar
羽飞 已提交
425
  frame_manager_.free(file_desc_, page_num, buf);
L
Longda 已提交
426
  return RC::SUCCESS;
羽飞's avatar
羽飞 已提交
427
}
L
Longda 已提交
428

羽飞's avatar
羽飞 已提交
429
RC DiskBufferPool::purge_page(PageNum page_num)
羽飞's avatar
羽飞 已提交
430
{
431
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
432
  Frame *used_frame = frame_manager_.get(file_desc_, page_num);
L
Longda 已提交
433
  if (used_frame != nullptr) {
羽飞's avatar
羽飞 已提交
434
    return purge_frame(page_num, used_frame);
羽飞's avatar
羽飞 已提交
435
  }
L
Longda 已提交
436

羽飞's avatar
羽飞 已提交
437 438 439
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
440
RC DiskBufferPool::purge_all_pages()
羽飞's avatar
羽飞 已提交
441
{
羽飞's avatar
羽飞 已提交
442
  std::list<Frame *> used = frame_manager_.find_list(file_desc_);
443 444

  std::scoped_lock lock_guard(lock_);
L
Longda 已提交
445 446
  for (std::list<Frame *>::iterator it = used.begin(); it != used.end(); ++it) {
    Frame *frame = *it;
447 448

    purge_frame(frame->page_num(), frame);
羽飞's avatar
羽飞 已提交
449 450 451 452
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
453
RC DiskBufferPool::check_all_pages_unpinned()
羽飞's avatar
羽飞 已提交
454
{
羽飞's avatar
羽飞 已提交
455
  std::list<Frame *> frames = frame_manager_.find_list(file_desc_);
456 457 458 459 460 461 462 463 464 465

  std::scoped_lock lock_guard(lock_);
  for (Frame *frame : frames) {
    frame->unpin();
    if (frame->page_num() == BP_HEADER_PAGE && frame->pin_count() > 1) {
      LOG_WARN("This page has been pinned. file desc=%d, pageNum:%d, pin count=%d",
          file_desc_, frame->page_num(), frame->pin_count());
    } else if (frame->page_num() != BP_HEADER_PAGE && frame->pin_count() > 0) {
      LOG_WARN("This page has been pinned. file desc=%d, pageNum:%d, pin count=%d",
          file_desc_, frame->page_num(), frame->pin_count());
羽飞's avatar
羽飞 已提交
466 467
    }
  }
羽飞's avatar
羽飞 已提交
468
  LOG_INFO("all pages have been checked of file desc %d", file_desc_);
羽飞's avatar
羽飞 已提交
469 470 471
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
472
RC DiskBufferPool::flush_page(Frame &frame)
473 474 475 476 477 478
{
  std::scoped_lock lock_guard(lock_);
  return flush_page_internal(frame);
}

RC DiskBufferPool::flush_page_internal(Frame &frame)
羽飞's avatar
羽飞 已提交
479 480 481 482
{
  // The better way is use mmap the block into memory,
  // so it is easier to flush data to file.

483
  Page &page = frame.page();
羽飞's avatar
羽飞 已提交
484 485 486
  s64_t offset = ((s64_t)page.page_num) * sizeof(Page);
  if (lseek(file_desc_, offset, SEEK_SET) == offset - 1) {
    LOG_ERROR("Failed to flush page %lld of %d due to failed to seek %s.", offset, file_desc_, strerror(errno));
羽飞's avatar
羽飞 已提交
487 488 489
    return RC::IOERR_SEEK;
  }

羽飞's avatar
羽飞 已提交
490
  if (writen(file_desc_, &page, sizeof(Page)) != 0) {
羽飞's avatar
羽飞 已提交
491
    LOG_ERROR("Failed to flush page %lld of %d due to %s.", offset, file_desc_, strerror(errno));
羽飞's avatar
羽飞 已提交
492 493
    return RC::IOERR_WRITE;
  }
494 495
  frame.clear_dirty();
  LOG_DEBUG("Flush block. file desc=%d, pageNum=%d", file_desc_, page.page_num);
羽飞's avatar
羽飞 已提交
496 497 498 499

  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512
RC DiskBufferPool::flush_all_pages()
{
  std::list<Frame *> used = frame_manager_.find_list(file_desc_);
  for (Frame *frame : used) {
    RC rc = flush_page(*frame);
    if (rc != RC::SUCCESS) {
      LOG_WARN("failed to flush all pages");
      return rc;
    }
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
513 514 515 516 517 518
RC DiskBufferPool::recover_page(PageNum page_num)
{
  int byte = 0, bit = 0;
  byte = page_num / 8;
  bit = page_num % 8;

519
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
520 521 522 523 524 525 526 527 528
  if (!(file_header_->bitmap[byte] & (1 << bit))) {
    file_header_->bitmap[byte] |= (1 << bit);
    file_header_->allocated_pages++;
    file_header_->page_count++;
    hdr_frame_->mark_dirty();
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
529
RC DiskBufferPool::allocate_frame(PageNum page_num, Frame **buffer)
羽飞's avatar
羽飞 已提交
530
{
531 532
  auto purger = [this](Frame *frame) {
    if (!frame->dirty()) {
羽飞's avatar
羽飞 已提交
533 534
      return RC::SUCCESS;
    }
羽飞's avatar
羽飞 已提交
535

536 537 538 539 540
    RC rc = RC::SUCCESS;
    if (frame->file_desc() == file_desc_) {
      rc = this->flush_page_internal(*frame);
    } else {
      rc = bp_manager_.flush_page(*frame);
羽飞's avatar
羽飞 已提交
541 542
    }

543 544
    if (rc != RC::SUCCESS) {
      LOG_ERROR("Failed to aclloc block due to failed to flush old block. rc=%s", strrc(rc));
羽飞's avatar
羽飞 已提交
545
    }
546 547
    return rc;
  };
L
Longda 已提交
548

549 550 551 552 553 554 555 556
  while (true) {
    Frame *frame = frame_manager_.alloc(file_desc_, page_num);
    if (frame != nullptr) {
      *buffer = frame;
      return RC::SUCCESS;
    }

    (void)frame_manager_.purge_frames(1/*count*/, purger);
羽飞's avatar
羽飞 已提交
557
  }
558
  return RC::BUFFERPOOL_NOBUF;
羽飞's avatar
羽飞 已提交
559 560
}

羽飞's avatar
羽飞 已提交
561
RC DiskBufferPool::check_page_num(PageNum page_num)
羽飞's avatar
羽飞 已提交
562
{
羽飞's avatar
羽飞 已提交
563 564 565
  if (page_num >= file_header_->page_count) {
    LOG_ERROR("Invalid pageNum:%d, file's name:%s", page_num, file_name_.c_str());
    return RC::BUFFERPOOL_INVALID_PAGE_NUM;
羽飞's avatar
羽飞 已提交
566
  }
羽飞's avatar
羽飞 已提交
567 568 569
  if ((file_header_->bitmap[page_num / 8] & (1 << (page_num % 8))) == 0) {
    LOG_ERROR("Invalid pageNum:%d, file's name:%s", page_num, file_name_.c_str());
    return RC::BUFFERPOOL_INVALID_PAGE_NUM;
羽飞's avatar
羽飞 已提交
570 571 572 573
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
574
RC DiskBufferPool::load_page(PageNum page_num, Frame *frame)
羽飞's avatar
羽飞 已提交
575
{
576
  s64_t offset = ((s64_t)page_num) * BP_PAGE_SIZE;
羽飞's avatar
羽飞 已提交
577
  if (lseek(file_desc_, offset, SEEK_SET) == -1) {
L
Longda Feng 已提交
578
    LOG_ERROR("Failed to load page %s:%d, due to failed to lseek:%s.", file_name_.c_str(), page_num, strerror(errno));
羽飞's avatar
羽飞 已提交
579 580 581

    return RC::IOERR_SEEK;
  }
羽飞's avatar
羽飞 已提交
582

583 584
  Page &page = frame->page();
  int ret = readn(file_desc_, &page, BP_PAGE_SIZE);
羽飞's avatar
羽飞 已提交
585 586
  if (ret != 0) {
    LOG_ERROR("Failed to load page %s:%d, due to failed to read data:%s, ret=%d, page count=%d",
587
        file_name_.c_str(), page_num, strerror(errno), ret, file_header_->allocated_pages);
羽飞's avatar
羽飞 已提交
588
    return RC::IOERR_READ;
羽飞's avatar
羽飞 已提交
589 590 591 592
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
593
RC DiskBufferPool::get_page_count(int *page_count)
羽飞's avatar
羽飞 已提交
594
{
羽飞's avatar
羽飞 已提交
595 596 597 598 599 600 601 602
  *page_count = file_header_->allocated_pages;
  return RC::SUCCESS;
}
int DiskBufferPool::file_desc() const
{
  return file_desc_;
}
////////////////////////////////////////////////////////////////////////////////
603
BufferPoolManager::BufferPoolManager(int page_num /* = 0 */)
羽飞's avatar
羽飞 已提交
604
{
605 606 607 608 609
  if (page_num <= 0) {
    page_num = MEM_POOL_ITEM_NUM * DEFAULT_ITEM_NUM_PER_POOL;
  }
  const int pool_num = std::max(page_num / DEFAULT_ITEM_NUM_PER_POOL, 1);
  frame_manager_.init(pool_num);
羽飞's avatar
羽飞 已提交
610 611 612 613 614 615
}

BufferPoolManager::~BufferPoolManager()
{
  std::unordered_map<std::string, DiskBufferPool *> tmp_bps;
  tmp_bps.swap(buffer_pools_);
L
Longda Feng 已提交
616

羽飞's avatar
羽飞 已提交
617 618
  for (auto &iter : tmp_bps) {
    delete iter.second;
羽飞's avatar
羽飞 已提交
619
  }
羽飞's avatar
羽飞 已提交
620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
}

RC BufferPoolManager::create_file(const char *file_name)
{
  int fd = open(file_name, O_RDWR | O_CREAT | O_EXCL, S_IREAD | S_IWRITE);
  if (fd < 0) {
    LOG_ERROR("Failed to create %s, due to %s.", file_name, strerror(errno));
    return RC::SCHEMA_DB_EXIST;
  }

  close(fd);

  /**
   * Here don't care about the failure
   */
  fd = open(file_name, O_RDWR);
  if (fd < 0) {
    LOG_ERROR("Failed to open for readwrite %s, due to %s.", file_name, strerror(errno));
    return RC::IOERR_ACCESS;
  }

  Page page;
642
  memset(&page, 0, BP_PAGE_SIZE);
羽飞's avatar
羽飞 已提交
643 644 645 646 647 648 649 650 651 652 653

  BPFileHeader *file_header = (BPFileHeader *)page.data;
  file_header->allocated_pages = 1;
  file_header->page_count = 1;

  char *bitmap = file_header->bitmap;
  bitmap[0] |= 0x01;
  if (lseek(fd, 0, SEEK_SET) == -1) {
    LOG_ERROR("Failed to seek file %s to position 0, due to %s .", file_name, strerror(errno));
    close(fd);
    return RC::IOERR_SEEK;
羽飞's avatar
羽飞 已提交
654
  }
羽飞's avatar
羽飞 已提交
655

656
  if (writen(fd, (char *)&page, BP_PAGE_SIZE) != 0) {
羽飞's avatar
羽飞 已提交
657 658 659 660 661 662 663
    LOG_ERROR("Failed to write header to file %s, due to %s.", file_name, strerror(errno));
    close(fd);
    return RC::IOERR_WRITE;
  }

  close(fd);
  LOG_INFO("Successfully create %s.", file_name);
羽飞's avatar
羽飞 已提交
664 665 666
  return RC::SUCCESS;
}

L
Longda Feng 已提交
667
RC BufferPoolManager::open_file(const char *_file_name, DiskBufferPool *&_bp)
羽飞's avatar
羽飞 已提交
668
{
羽飞's avatar
羽飞 已提交
669
  std::string file_name(_file_name);
L
Longda Feng 已提交
670

671
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
672 673 674 675
  if (buffer_pools_.find(file_name) != buffer_pools_.end()) {
    LOG_WARN("file already opened. file name=%s", _file_name);
    return RC::BUFFERPOOL_OPEN;
  }
羽飞's avatar
羽飞 已提交
676

羽飞's avatar
羽飞 已提交
677 678 679 680 681 682
  DiskBufferPool *bp = new DiskBufferPool(*this, frame_manager_);
  RC rc = bp->open_file(_file_name);
  if (rc != RC::SUCCESS) {
    LOG_WARN("failed to open file name");
    delete bp;
    return rc;
羽飞's avatar
羽飞 已提交
683
  }
羽飞's avatar
羽飞 已提交
684 685 686

  buffer_pools_.insert(std::pair<std::string, DiskBufferPool *>(file_name, bp));
  fd_buffer_pools_.insert(std::pair<int, DiskBufferPool *>(bp->file_desc(), bp));
687
  LOG_DEBUG("insert buffer pool into fd buffer pools. fd=%d, bp=%p, lbt=%s", bp->file_desc(), bp, lbt());
羽飞's avatar
羽飞 已提交
688 689 690 691 692 693 694
  _bp = bp;
  return RC::SUCCESS;
}

RC BufferPoolManager::close_file(const char *_file_name)
{
  std::string file_name(_file_name);
695 696 697

  std::scoped_lock lock_guard(lock_);

羽飞's avatar
羽飞 已提交
698 699 700 701
  auto iter = buffer_pools_.find(file_name);
  if (iter == buffer_pools_.end()) {
    LOG_WARN("file has not opened: %s", _file_name);
    return RC::INTERNAL;
羽飞's avatar
羽飞 已提交
702
  }
羽飞's avatar
羽飞 已提交
703 704

  int fd = iter->second->file_desc();
705 706 707 708 709 710 711 712 713 714 715
  if (0 == fd_buffer_pools_.erase(fd)) {
    int count = 0;
    for (auto fd_iter = fd_buffer_pools_.begin(); fd_iter != fd_buffer_pools_.end(); ++fd_iter) {
      if (fd_iter->second == iter->second) {
        fd_buffer_pools_.erase(fd_iter);
        count = 1;
        break;
      }
    }
    ASSERT(count == 1, "the buffer pool was not erased from fd buffer pools.");
  }
羽飞's avatar
羽飞 已提交
716 717 718 719

  DiskBufferPool *bp = iter->second;
  buffer_pools_.erase(iter);
  delete bp;
羽飞's avatar
羽飞 已提交
720 721
  return RC::SUCCESS;
}
羽飞's avatar
羽飞 已提交
722 723 724 725

RC BufferPoolManager::flush_page(Frame &frame)
{
  int fd = frame.file_desc();
726 727

  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
  auto iter = fd_buffer_pools_.find(fd);
  if (iter == fd_buffer_pools_.end()) {
    LOG_WARN("unknown buffer pool of fd %d", fd);
    return RC::INTERNAL;
  }

  DiskBufferPool *bp = iter->second;
  return bp->flush_page(frame);
}

static BufferPoolManager *default_bpm = nullptr;
void BufferPoolManager::set_instance(BufferPoolManager *bpm)
{
  if (default_bpm != nullptr && bpm != nullptr) {
    LOG_ERROR("default buffer pool manager has been setted");
    abort();
  }
  default_bpm = bpm;
}
BufferPoolManager &BufferPoolManager::instance()
{
  return *default_bpm;
}