disk_buffer_pool.cpp 21.3 KB
Newer Older
羽飞's avatar
羽飞 已提交
1 2 3 4 5 6 7 8 9 10 11
/* Copyright (c) 2021 Xie Meiyi(xiemeiyi@hust.edu.cn) and OceanBase and/or its affiliates. All rights reserved.
miniob is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:
         http://license.coscl.org.cn/MulanPSL2
THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details. */

//
12
// Created by Meiyi & Longda on 2021/4/13.
羽飞's avatar
羽飞 已提交
13 14 15 16
//
#include <errno.h>
#include <string.h>

17
#include "storage/default/disk_buffer_pool.h"
L
Longda 已提交
18
#include "common/lang/mutex.h"
羽飞's avatar
羽飞 已提交
19
#include "common/log/log.h"
L
Longda 已提交
20
#include "common/os/os.h"
羽飞's avatar
羽飞 已提交
21
#include "common/io/io.h"
羽飞's avatar
羽飞 已提交
22 23

using namespace common;
羽飞's avatar
羽飞 已提交
24
using namespace std;
羽飞's avatar
羽飞 已提交
25

羽飞's avatar
羽飞 已提交
26 27 28 29 30 31 32 33 34 35 36
static const int MEM_POOL_ITEM_NUM = 20;

////////////////////////////////////////////////////////////////////////////////

string BPFileHeader::to_string() const
{
  stringstream ss;
  ss << "pageCount:" << page_count
     << ", allocatedCount:" << allocated_pages;
  return ss.str();
}
L
Longda 已提交
37

38
////////////////////////////////////////////////////////////////////////////////
羽飞's avatar
羽飞 已提交
39

羽飞's avatar
羽飞 已提交
40
BPFrameManager::BPFrameManager(const char *name) : allocator_(name)
L
Longda 已提交
41 42
{}

羽飞's avatar
羽飞 已提交
43
RC BPFrameManager::init(int pool_num)
L
Longda 已提交
44
{
L
Longda Feng 已提交
45
  int ret = allocator_.init(false, pool_num);
羽飞's avatar
羽飞 已提交
46 47 48
  if (ret == 0) {
    return RC::SUCCESS;
  }
羽飞's avatar
羽飞 已提交
49
  return RC::NOMEM;
L
Longda 已提交
50 51
}

羽飞's avatar
羽飞 已提交
52
RC BPFrameManager::cleanup()
L
Longda 已提交
53
{
羽飞's avatar
羽飞 已提交
54
  if (frames_.count() > 0) {
羽飞's avatar
羽飞 已提交
55
    return RC::INTERNAL;
L
Longda 已提交
56 57
  }

羽飞's avatar
羽飞 已提交
58 59 60 61
  frames_.destroy();
  return RC::SUCCESS;
}

62
int BPFrameManager::purge_frames(int count, std::function<RC(Frame *frame)> purger)
L
Longda 已提交
63
{
64 65 66 67 68 69 70 71 72
  std::lock_guard<std::mutex> lock_guard(lock_);

  std::vector<Frame *> frames_can_purge;
  if (count <= 0) {
    count = 1;
  }
  frames_can_purge.reserve(count);

  auto purge_finder = [&frames_can_purge, count](const FrameId &frame_id, Frame *const frame) {
羽飞's avatar
羽飞 已提交
73
    if (frame->can_purge()) {
74 75 76 77 78
      frame->pin();
      frames_can_purge.push_back(frame);
      if (frames_can_purge.size() >= static_cast<size_t>(count)) {
        return false;  // false to break the progress
      }
羽飞's avatar
羽飞 已提交
79
    }
L
Longda Feng 已提交
80
    return true;  // true continue to look up
羽飞's avatar
羽飞 已提交
81
  };
82

羽飞's avatar
羽飞 已提交
83
  frames_.foreach_reverse(purge_finder);
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
  LOG_INFO("purge frames find %ld pages total", frames_can_purge.size());

  /// 当前还在frameManager的锁内,而 purger 是一个非常耗时的操作
  /// 他需要把脏页数据刷新到磁盘上去,所以这里会极大地降低并发度
  int freed_count = 0;
  for (Frame *frame : frames_can_purge) {
    RC rc = purger(frame);
    if (RC::SUCCESS == rc) {
      free_internal(frame->frame_id(), frame);
      freed_count++;
    } else {
      frame->unpin();
      LOG_WARN("failed to purge frame. frame_id=%s, rc=%s", 
               to_string(frame->frame_id()).c_str(), strrc(rc));
    }
  }
  LOG_INFO("purge frame done. number=%d", freed_count);
  return freed_count;
羽飞's avatar
羽飞 已提交
102
}
L
Longda 已提交
103

羽飞's avatar
羽飞 已提交
104 105
Frame *BPFrameManager::get(int file_desc, PageNum page_num)
{
106
  FrameId frame_id(file_desc, page_num);
羽飞's avatar
羽飞 已提交
107
  std::lock_guard<std::mutex> lock_guard(lock_);
108 109 110 111 112
  return get_internal(frame_id);
}

Frame *BPFrameManager::get_internal(const FrameId &frame_id)
{
羽飞's avatar
羽飞 已提交
113 114
  Frame *frame = nullptr;
  (void)frames_.get(frame_id, frame);
115 116 117
  if (frame != nullptr) {
    frame->pin();
  }
羽飞's avatar
羽飞 已提交
118
  return frame;
L
Longda 已提交
119 120
}

羽飞's avatar
羽飞 已提交
121
Frame *BPFrameManager::alloc(int file_desc, PageNum page_num)
L
Longda 已提交
122
{
123
  FrameId frame_id(file_desc, page_num);
羽飞's avatar
羽飞 已提交
124 125

  std::lock_guard<std::mutex> lock_guard(lock_);
126 127 128
  Frame *frame = get_internal(frame_id);
  if (frame != nullptr) {
    return frame;
羽飞's avatar
羽飞 已提交
129 130 131 132
  }

  frame = allocator_.alloc();
  if (frame != nullptr) {
133 134 135
    ASSERT(frame->pin_count() == 0, "got an invalid frame that pin count is not 0. frame=%s", 
           to_string(*frame).c_str());
    frame->pin();
羽飞's avatar
羽飞 已提交
136 137 138
    frames_.put(frame_id, frame);
  }
  return frame;
L
Longda 已提交
139 140
}

羽飞's avatar
羽飞 已提交
141
RC BPFrameManager::free(int file_desc, PageNum page_num, Frame *frame)
L
Longda 已提交
142
{
143
  FrameId frame_id(file_desc, page_num);
L
Longda 已提交
144

羽飞's avatar
羽飞 已提交
145
  std::lock_guard<std::mutex> lock_guard(lock_);
146 147 148 149 150
  return free_internal(frame_id, frame);
}

RC BPFrameManager::free_internal(const FrameId &frame_id, Frame *frame)
{
羽飞's avatar
羽飞 已提交
151
  Frame *frame_source = nullptr;
羽飞's avatar
羽飞 已提交
152
  [[maybe_unused]] bool found = frames_.get(frame_id, frame_source);
153 154 155
  ASSERT(found && frame == frame_source && frame->pin_count() == 1,
         "failed to free frame. found=%d, frameId=%s, frame_source=%p, frame=%p, pinCount=%d, lbt=%s",
         found, to_string(frame_id).c_str(), frame_source, frame, frame->pin_count(), lbt());
L
Longda 已提交
156

157
  frame->unpin();
羽飞's avatar
羽飞 已提交
158 159 160
  frames_.remove(frame_id);
  allocator_.free(frame);
  return RC::SUCCESS;
L
Longda 已提交
161 162
}

羽飞's avatar
羽飞 已提交
163
std::list<Frame *> BPFrameManager::find_list(int file_desc)
L
Longda 已提交
164
{
羽飞's avatar
羽飞 已提交
165 166 167
  std::lock_guard<std::mutex> lock_guard(lock_);

  std::list<Frame *> frames;
168
  auto fetcher = [&frames, file_desc](const FrameId &frame_id, Frame *const frame) -> bool {
羽飞's avatar
羽飞 已提交
169
    if (file_desc == frame_id.file_desc()) {
170
      frame->pin();
羽飞's avatar
羽飞 已提交
171 172 173 174
      frames.push_back(frame);
    }
    return true;
  };
L
Longda Feng 已提交
175
  frames_.foreach (fetcher);
羽飞's avatar
羽飞 已提交
176
  return frames;
L
Longda 已提交
177 178
}

羽飞's avatar
羽飞 已提交
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
////////////////////////////////////////////////////////////////////////////////
BufferPoolIterator::BufferPoolIterator()
{}
BufferPoolIterator::~BufferPoolIterator()
{}
RC BufferPoolIterator::init(DiskBufferPool &bp, PageNum start_page /* = 0 */)
{
  bitmap_.init(bp.file_header_->bitmap, bp.file_header_->page_count);
  if (start_page <= 0) {
    current_page_num_ = 0;
  } else {
    current_page_num_ = start_page;
  }
  return RC::SUCCESS;
}

bool BufferPoolIterator::has_next()
{
羽飞's avatar
羽飞 已提交
197
  return bitmap_.next_setted_bit(current_page_num_ + 1) != -1;
羽飞's avatar
羽飞 已提交
198 199 200 201
}

PageNum BufferPoolIterator::next()
{
羽飞's avatar
羽飞 已提交
202
  PageNum next_page = bitmap_.next_setted_bit(current_page_num_ + 1);
羽飞's avatar
羽飞 已提交
203 204 205 206 207 208 209 210 211 212 213 214 215
  if (next_page != -1) {
    current_page_num_ = next_page;
  }
  return next_page;
}

RC BufferPoolIterator::reset()
{
  current_page_num_ = 0;
  return RC::SUCCESS;
}

////////////////////////////////////////////////////////////////////////////////
羽飞's avatar
羽飞 已提交
216
DiskBufferPool::DiskBufferPool(BufferPoolManager &bp_manager, BPFrameManager &frame_manager)
L
Longda Feng 已提交
217 218
    : bp_manager_(bp_manager), frame_manager_(frame_manager)
{}
羽飞's avatar
羽飞 已提交
219

L
Longda 已提交
220 221
DiskBufferPool::~DiskBufferPool()
{
羽飞's avatar
羽飞 已提交
222
  close_file();
223
  LOG_INFO("disk buffer pool exit");
L
Longda 已提交
224 225
}

羽飞's avatar
羽飞 已提交
226
RC DiskBufferPool::open_file(const char *file_name)
羽飞's avatar
羽飞 已提交
227
{
228 229
  int fd = open(file_name, O_RDWR);
  if (fd < 0) {
羽飞's avatar
羽飞 已提交
230 231 232
    LOG_ERROR("Failed to open file %s, because %s.", file_name, strerror(errno));
    return RC::IOERR_ACCESS;
  }
233
  LOG_INFO("Successfully open buffer pool file %s.", file_name);
羽飞's avatar
羽飞 已提交
234

羽飞's avatar
羽飞 已提交
235 236
  file_name_ = file_name;
  file_desc_ = fd;
羽飞's avatar
羽飞 已提交
237

羽飞's avatar
羽飞 已提交
238
  RC rc = RC::SUCCESS;
羽飞's avatar
羽飞 已提交
239
  rc = allocate_frame(BP_HEADER_PAGE, &hdr_frame_);
羽飞's avatar
羽飞 已提交
240 241
  if (rc != RC::SUCCESS) {
    LOG_ERROR("failed to allocate frame for header. file name %s", file_name_.c_str());
羽飞's avatar
羽飞 已提交
242
    close(fd);
羽飞's avatar
羽飞 已提交
243 244
    file_desc_ = -1;
    return rc;
羽飞's avatar
羽飞 已提交
245
  }
羽飞's avatar
羽飞 已提交
246

247 248 249
  hdr_frame_->set_file_desc(fd);
  hdr_frame_->access();

羽飞's avatar
羽飞 已提交
250
  if ((rc = load_page(BP_HEADER_PAGE, hdr_frame_)) != RC::SUCCESS) {
L
Longda 已提交
251
    LOG_ERROR("Failed to load first page of %s, due to %s.", file_name, strerror(errno));
羽飞's avatar
羽飞 已提交
252
    purge_frame(BP_HEADER_PAGE, hdr_frame_);
羽飞's avatar
羽飞 已提交
253
    close(fd);
羽飞's avatar
羽飞 已提交
254 255
    file_desc_ = -1;
    return rc;
羽飞's avatar
羽飞 已提交
256 257
  }

羽飞's avatar
羽飞 已提交
258
  file_header_ = (BPFileHeader *)hdr_frame_->data();
L
Longda 已提交
259

羽飞's avatar
羽飞 已提交
260 261
  LOG_INFO("Successfully open %s. file_desc=%d, hdr_frame=%p, file header=%s",
           file_name, file_desc_, hdr_frame_, file_header_->to_string().c_str());
羽飞's avatar
羽飞 已提交
262 263 264
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
265
RC DiskBufferPool::close_file()
羽飞's avatar
羽飞 已提交
266
{
羽飞's avatar
羽飞 已提交
267 268 269
  RC rc = RC::SUCCESS;
  if (file_desc_ < 0) {
    return rc;
羽飞's avatar
羽飞 已提交
270 271
  }

272 273
  hdr_frame_->unpin();

羽飞's avatar
羽飞 已提交
274
  // TODO: 理论上是在回放时回滚未提交事务,但目前没有undo log,因此不下刷数据page,只通过redo log回放
275 276 277
  rc = purge_all_pages();
  if (rc != RC::SUCCESS) {
    LOG_ERROR("failed to close %s, due to failed to purge pages. rc=%s", file_name_.c_str(), strrc(rc));
羽飞's avatar
羽飞 已提交
278
    return rc;
羽飞's avatar
羽飞 已提交
279 280
  }

281
  disposed_pages_.clear();
L
Longda 已提交
282

羽飞's avatar
羽飞 已提交
283 284
  if (close(file_desc_) < 0) {
    LOG_ERROR("Failed to close fileId:%d, fileName:%s, error:%s", file_desc_, file_name_.c_str(), strerror(errno));
羽飞's avatar
羽飞 已提交
285 286
    return RC::IOERR_CLOSE;
  }
羽飞's avatar
羽飞 已提交
287 288 289 290
  LOG_INFO("Successfully close file %d:%s.", file_desc_, file_name_.c_str());
  file_desc_ = -1;

  bp_manager_.close_file(file_name_.c_str());
羽飞's avatar
羽飞 已提交
291 292 293
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
294
RC DiskBufferPool::get_this_page(PageNum page_num, Frame **frame)
羽飞's avatar
羽飞 已提交
295
{
羽飞's avatar
羽飞 已提交
296
  RC rc = RC::SUCCESS;
297
  *frame = nullptr;
羽飞's avatar
羽飞 已提交
298

羽飞's avatar
羽飞 已提交
299
  Frame *used_match_frame = frame_manager_.get(file_desc_, page_num);
L
Longda 已提交
300
  if (used_match_frame != nullptr) {
301
    used_match_frame->access();
羽飞's avatar
羽飞 已提交
302
    *frame = used_match_frame;
L
Longda 已提交
303
    return RC::SUCCESS;
羽飞's avatar
羽飞 已提交
304 305
  }

306 307
  std::scoped_lock lock_guard(lock_); // 直接加了一把大锁,其实可以根据访问的页面来细化提高并行度

羽飞's avatar
羽飞 已提交
308
  // Allocate one page and load the data into this page
羽飞's avatar
羽飞 已提交
309
  Frame *allocated_frame = nullptr;
310 311
  rc = allocate_frame(page_num, &allocated_frame);
  if (rc != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
312 313 314 315
    LOG_ERROR("Failed to alloc frame %s:%d, due to failed to alloc page.", file_name_.c_str(), page_num);
    return rc;
  }

316 317 318 319
  allocated_frame->set_file_desc(file_desc_);
  // allocated_frame->pin(); // pined in manager::get
  allocated_frame->access();

羽飞's avatar
羽飞 已提交
320 321
  if ((rc = load_page(page_num, allocated_frame)) != RC::SUCCESS) {
    LOG_ERROR("Failed to load page %s:%d", file_name_.c_str(), page_num);
羽飞's avatar
羽飞 已提交
322
    purge_frame(page_num, allocated_frame);
羽飞's avatar
羽飞 已提交
323 324 325 326
    return rc;
  }

  *frame = allocated_frame;
羽飞's avatar
羽飞 已提交
327 328 329
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
330
RC DiskBufferPool::allocate_page(Frame **frame)
羽飞's avatar
羽飞 已提交
331
{
羽飞's avatar
羽飞 已提交
332
  RC rc = RC::SUCCESS;
羽飞's avatar
羽飞 已提交
333

334 335
  lock_.lock();
  
羽飞's avatar
羽飞 已提交
336
  int byte = 0, bit = 0;
羽飞's avatar
羽飞 已提交
337
  if ((file_header_->allocated_pages) < (file_header_->page_count)) {
羽飞's avatar
羽飞 已提交
338
    // There is one free page
羽飞's avatar
羽飞 已提交
339
    for (int i = 0; i < file_header_->page_count; i++) {
羽飞's avatar
羽飞 已提交
340 341
      byte = i / 8;
      bit = i % 8;
羽飞's avatar
羽飞 已提交
342 343 344
      if (((file_header_->bitmap[byte]) & (1 << bit)) == 0) {
        (file_header_->allocated_pages)++;
        file_header_->bitmap[byte] |= (1 << bit);
L
Longda 已提交
345
        // TODO,  do we need clean the loaded page's data?
L
Longda Feng 已提交
346
        hdr_frame_->mark_dirty();
347 348

        lock_.unlock();
羽飞's avatar
羽飞 已提交
349
        return get_this_page(i, frame);
羽飞's avatar
羽飞 已提交
350 351 352 353
      }
    }
  }

羽飞's avatar
羽飞 已提交
354 355
  if (file_header_->page_count >= BPFileHeader::MAX_PAGE_NUM) {
    LOG_WARN("file buffer pool is full. page count %d, max page count %d",
356 357 358
        file_header_->page_count, BPFileHeader::MAX_PAGE_NUM);
    lock_.unlock();
    return RC::BUFFERPOOL_NOBUF;
羽飞's avatar
羽飞 已提交
359 360 361
  }

  PageNum page_num = file_header_->page_count;
羽飞's avatar
羽飞 已提交
362
  Frame *allocated_frame = nullptr;
羽飞's avatar
羽飞 已提交
363
  if ((rc = allocate_frame(page_num, &allocated_frame)) != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
364
    LOG_ERROR("Failed to allocate frame %s, due to no free page.", file_name_.c_str());
365
    lock_.unlock();
羽飞's avatar
羽飞 已提交
366
    return rc;
羽飞's avatar
羽飞 已提交
367 368
  }

羽飞's avatar
羽飞 已提交
369 370 371
  LOG_INFO("allocate new page. file=%s, pageNum=%d, pin=%d",
           file_name_.c_str(), page_num, allocated_frame->pin_count());

羽飞's avatar
羽飞 已提交
372 373
  file_header_->allocated_pages++;
  file_header_->page_count++;
羽飞's avatar
羽飞 已提交
374 375 376

  byte = page_num / 8;
  bit = page_num % 8;
羽飞's avatar
羽飞 已提交
377 378
  file_header_->bitmap[byte] |= (1 << bit);
  hdr_frame_->mark_dirty();
羽飞's avatar
羽飞 已提交
379

380 381
  allocated_frame->set_file_desc(file_desc_);
  allocated_frame->access();
羽飞's avatar
羽飞 已提交
382
  allocated_frame->clear_page();
383
  allocated_frame->set_page_num(file_header_->page_count - 1);
羽飞's avatar
羽飞 已提交
384

L
Longda 已提交
385
  // Use flush operation to extension file
386
  if ((rc = flush_page_internal(*allocated_frame)) != RC::SUCCESS) {
羽飞's avatar
羽飞 已提交
387
    LOG_WARN("Failed to alloc page %s , due to failed to extend one page.", file_name_.c_str());
L
Longda 已提交
388 389
    // skip return false, delay flush the extended page
    // return tmp;
羽飞's avatar
羽飞 已提交
390 391
  }

392
  lock_.unlock();
L
Longda 已提交
393

394
  *frame = allocated_frame;
羽飞's avatar
羽飞 已提交
395 396 397
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
398
RC DiskBufferPool::dispose_page(PageNum page_num)
羽飞's avatar
羽飞 已提交
399
{
400 401 402 403 404 405 406 407
  std::scoped_lock lock_guard(lock_);
  Frame *used_frame = frame_manager_.get(file_desc_, page_num);
  if (used_frame != nullptr) {
    ASSERT("the page try to dispose is in use. frame:%s", to_string(*used_frame).c_str());
    frame_manager_.free(file_desc_, page_num, used_frame);
  } else {
    LOG_WARN("failed to fetch the page while disposing it. pageNum=%d", page_num);
    return RC::NOTFOUND;
羽飞's avatar
羽飞 已提交
408 409
  }

410
  hdr_frame_->mark_dirty();
羽飞's avatar
羽飞 已提交
411
  file_header_->allocated_pages--;
羽飞's avatar
羽飞 已提交
412
  char tmp = 1 << (page_num % 8);
羽飞's avatar
羽飞 已提交
413
  file_header_->bitmap[page_num / 8] &= ~tmp;
羽飞's avatar
羽飞 已提交
414 415 416
  return RC::SUCCESS;
}

417 418 419 420 421 422
RC DiskBufferPool::unpin_page(Frame *frame)
{
  frame->unpin();
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
423
RC DiskBufferPool::purge_frame(PageNum page_num, Frame *buf)
羽飞's avatar
羽飞 已提交
424
{
425 426 427
  if (buf->pin_count() != 1) {
    LOG_INFO("Begin to free page %d of %d(file id), but it's pin count > 1:%d.",
        buf->page_num(), buf->file_desc(), buf->pin_count());
L
Longda 已提交
428 429 430
    return RC::LOCKED_UNLOCK;
  }

431 432
  if (buf->dirty()) {
    RC rc = flush_page_internal(*buf);
L
Longda 已提交
433
    if (rc != RC::SUCCESS) {
434
      LOG_WARN("Failed to flush page %d of %d(file desc) during purge page.", buf->page_num(), buf->file_desc());
L
Longda 已提交
435 436 437 438
      return rc;
    }
  }

439
  LOG_DEBUG("Successfully purge frame =%p, page %d of %d(file desc)", buf, buf->page_num(), buf->file_desc());
羽飞's avatar
羽飞 已提交
440
  frame_manager_.free(file_desc_, page_num, buf);
L
Longda 已提交
441
  return RC::SUCCESS;
羽飞's avatar
羽飞 已提交
442
}
L
Longda 已提交
443

羽飞's avatar
羽飞 已提交
444
RC DiskBufferPool::purge_page(PageNum page_num)
羽飞's avatar
羽飞 已提交
445
{
446
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
447
  Frame *used_frame = frame_manager_.get(file_desc_, page_num);
L
Longda 已提交
448
  if (used_frame != nullptr) {
羽飞's avatar
羽飞 已提交
449
    return purge_frame(page_num, used_frame);
羽飞's avatar
羽飞 已提交
450
  }
L
Longda 已提交
451

羽飞's avatar
羽飞 已提交
452 453 454
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
455
RC DiskBufferPool::purge_all_pages()
羽飞's avatar
羽飞 已提交
456
{
羽飞's avatar
羽飞 已提交
457
  std::list<Frame *> used = frame_manager_.find_list(file_desc_);
458 459

  std::scoped_lock lock_guard(lock_);
L
Longda 已提交
460 461
  for (std::list<Frame *>::iterator it = used.begin(); it != used.end(); ++it) {
    Frame *frame = *it;
462 463

    purge_frame(frame->page_num(), frame);
羽飞's avatar
羽飞 已提交
464 465 466 467
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
468
RC DiskBufferPool::check_all_pages_unpinned()
羽飞's avatar
羽飞 已提交
469
{
羽飞's avatar
羽飞 已提交
470
  std::list<Frame *> frames = frame_manager_.find_list(file_desc_);
471 472 473 474 475 476 477 478 479 480

  std::scoped_lock lock_guard(lock_);
  for (Frame *frame : frames) {
    frame->unpin();
    if (frame->page_num() == BP_HEADER_PAGE && frame->pin_count() > 1) {
      LOG_WARN("This page has been pinned. file desc=%d, pageNum:%d, pin count=%d",
          file_desc_, frame->page_num(), frame->pin_count());
    } else if (frame->page_num() != BP_HEADER_PAGE && frame->pin_count() > 0) {
      LOG_WARN("This page has been pinned. file desc=%d, pageNum:%d, pin count=%d",
          file_desc_, frame->page_num(), frame->pin_count());
羽飞's avatar
羽飞 已提交
481 482
    }
  }
羽飞's avatar
羽飞 已提交
483
  LOG_INFO("all pages have been checked of file desc %d", file_desc_);
羽飞's avatar
羽飞 已提交
484 485 486
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
487
RC DiskBufferPool::flush_page(Frame &frame)
488 489 490 491 492 493
{
  std::scoped_lock lock_guard(lock_);
  return flush_page_internal(frame);
}

RC DiskBufferPool::flush_page_internal(Frame &frame)
羽飞's avatar
羽飞 已提交
494 495 496 497
{
  // The better way is use mmap the block into memory,
  // so it is easier to flush data to file.

498
  Page &page = frame.page();
羽飞's avatar
羽飞 已提交
499 500 501
  s64_t offset = ((s64_t)page.page_num) * sizeof(Page);
  if (lseek(file_desc_, offset, SEEK_SET) == offset - 1) {
    LOG_ERROR("Failed to flush page %lld of %d due to failed to seek %s.", offset, file_desc_, strerror(errno));
羽飞's avatar
羽飞 已提交
502 503 504
    return RC::IOERR_SEEK;
  }

羽飞's avatar
羽飞 已提交
505
  if (writen(file_desc_, &page, sizeof(Page)) != 0) {
羽飞's avatar
羽飞 已提交
506
    LOG_ERROR("Failed to flush page %lld of %d due to %s.", offset, file_desc_, strerror(errno));
羽飞's avatar
羽飞 已提交
507 508
    return RC::IOERR_WRITE;
  }
509
  frame.clear_dirty();
羽飞's avatar
羽飞 已提交
510
  LOG_DEBUG("Flush block. file desc=%d, pageNum=%d, pin count=%d", file_desc_, page.page_num, frame.pin_count());
羽飞's avatar
羽飞 已提交
511 512 513 514

  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
515 516 517 518 519 520 521 522 523 524 525 526 527
RC DiskBufferPool::flush_all_pages()
{
  std::list<Frame *> used = frame_manager_.find_list(file_desc_);
  for (Frame *frame : used) {
    RC rc = flush_page(*frame);
    if (rc != RC::SUCCESS) {
      LOG_WARN("failed to flush all pages");
      return rc;
    }
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
528 529 530 531 532 533
RC DiskBufferPool::recover_page(PageNum page_num)
{
  int byte = 0, bit = 0;
  byte = page_num / 8;
  bit = page_num % 8;

534
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
535 536 537 538 539 540 541 542 543
  if (!(file_header_->bitmap[byte] & (1 << bit))) {
    file_header_->bitmap[byte] |= (1 << bit);
    file_header_->allocated_pages++;
    file_header_->page_count++;
    hdr_frame_->mark_dirty();
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
544
RC DiskBufferPool::allocate_frame(PageNum page_num, Frame **buffer)
羽飞's avatar
羽飞 已提交
545
{
546 547
  auto purger = [this](Frame *frame) {
    if (!frame->dirty()) {
羽飞's avatar
羽飞 已提交
548 549
      return RC::SUCCESS;
    }
羽飞's avatar
羽飞 已提交
550

551 552 553 554 555
    RC rc = RC::SUCCESS;
    if (frame->file_desc() == file_desc_) {
      rc = this->flush_page_internal(*frame);
    } else {
      rc = bp_manager_.flush_page(*frame);
羽飞's avatar
羽飞 已提交
556 557
    }

558 559
    if (rc != RC::SUCCESS) {
      LOG_ERROR("Failed to aclloc block due to failed to flush old block. rc=%s", strrc(rc));
羽飞's avatar
羽飞 已提交
560
    }
561 562
    return rc;
  };
L
Longda 已提交
563

564 565 566 567 568 569 570 571
  while (true) {
    Frame *frame = frame_manager_.alloc(file_desc_, page_num);
    if (frame != nullptr) {
      *buffer = frame;
      return RC::SUCCESS;
    }

    (void)frame_manager_.purge_frames(1/*count*/, purger);
羽飞's avatar
羽飞 已提交
572
  }
573
  return RC::BUFFERPOOL_NOBUF;
羽飞's avatar
羽飞 已提交
574 575
}

羽飞's avatar
羽飞 已提交
576
RC DiskBufferPool::check_page_num(PageNum page_num)
羽飞's avatar
羽飞 已提交
577
{
羽飞's avatar
羽飞 已提交
578 579 580
  if (page_num >= file_header_->page_count) {
    LOG_ERROR("Invalid pageNum:%d, file's name:%s", page_num, file_name_.c_str());
    return RC::BUFFERPOOL_INVALID_PAGE_NUM;
羽飞's avatar
羽飞 已提交
581
  }
羽飞's avatar
羽飞 已提交
582 583 584
  if ((file_header_->bitmap[page_num / 8] & (1 << (page_num % 8))) == 0) {
    LOG_ERROR("Invalid pageNum:%d, file's name:%s", page_num, file_name_.c_str());
    return RC::BUFFERPOOL_INVALID_PAGE_NUM;
羽飞's avatar
羽飞 已提交
585 586 587 588
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
589
RC DiskBufferPool::load_page(PageNum page_num, Frame *frame)
羽飞's avatar
羽飞 已提交
590
{
591
  s64_t offset = ((s64_t)page_num) * BP_PAGE_SIZE;
羽飞's avatar
羽飞 已提交
592
  if (lseek(file_desc_, offset, SEEK_SET) == -1) {
L
Longda Feng 已提交
593
    LOG_ERROR("Failed to load page %s:%d, due to failed to lseek:%s.", file_name_.c_str(), page_num, strerror(errno));
羽飞's avatar
羽飞 已提交
594 595 596

    return RC::IOERR_SEEK;
  }
羽飞's avatar
羽飞 已提交
597

598 599
  Page &page = frame->page();
  int ret = readn(file_desc_, &page, BP_PAGE_SIZE);
羽飞's avatar
羽飞 已提交
600
  if (ret != 0) {
羽飞's avatar
羽飞 已提交
601 602
    LOG_ERROR("Failed to load page %s, file_desc:%d, page num:%d, due to failed to read data:%s, ret=%d, page count=%d",
              file_name_.c_str(), file_desc_, page_num, strerror(errno), ret, file_header_->allocated_pages);
羽飞's avatar
羽飞 已提交
603
    return RC::IOERR_READ;
羽飞's avatar
羽飞 已提交
604 605 606 607
  }
  return RC::SUCCESS;
}

羽飞's avatar
羽飞 已提交
608
RC DiskBufferPool::get_page_count(int *page_count)
羽飞's avatar
羽飞 已提交
609
{
羽飞's avatar
羽飞 已提交
610 611 612 613 614 615 616 617
  *page_count = file_header_->allocated_pages;
  return RC::SUCCESS;
}
int DiskBufferPool::file_desc() const
{
  return file_desc_;
}
////////////////////////////////////////////////////////////////////////////////
618
BufferPoolManager::BufferPoolManager(int page_num /* = 0 */)
羽飞's avatar
羽飞 已提交
619
{
620 621 622 623 624
  if (page_num <= 0) {
    page_num = MEM_POOL_ITEM_NUM * DEFAULT_ITEM_NUM_PER_POOL;
  }
  const int pool_num = std::max(page_num / DEFAULT_ITEM_NUM_PER_POOL, 1);
  frame_manager_.init(pool_num);
羽飞's avatar
羽飞 已提交
625 626 627 628 629 630
}

BufferPoolManager::~BufferPoolManager()
{
  std::unordered_map<std::string, DiskBufferPool *> tmp_bps;
  tmp_bps.swap(buffer_pools_);
L
Longda Feng 已提交
631

羽飞's avatar
羽飞 已提交
632 633
  for (auto &iter : tmp_bps) {
    delete iter.second;
羽飞's avatar
羽飞 已提交
634
  }
羽飞's avatar
羽飞 已提交
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
}

RC BufferPoolManager::create_file(const char *file_name)
{
  int fd = open(file_name, O_RDWR | O_CREAT | O_EXCL, S_IREAD | S_IWRITE);
  if (fd < 0) {
    LOG_ERROR("Failed to create %s, due to %s.", file_name, strerror(errno));
    return RC::SCHEMA_DB_EXIST;
  }

  close(fd);

  /**
   * Here don't care about the failure
   */
  fd = open(file_name, O_RDWR);
  if (fd < 0) {
    LOG_ERROR("Failed to open for readwrite %s, due to %s.", file_name, strerror(errno));
    return RC::IOERR_ACCESS;
  }

  Page page;
657
  memset(&page, 0, BP_PAGE_SIZE);
羽飞's avatar
羽飞 已提交
658 659 660 661 662 663 664 665 666 667 668

  BPFileHeader *file_header = (BPFileHeader *)page.data;
  file_header->allocated_pages = 1;
  file_header->page_count = 1;

  char *bitmap = file_header->bitmap;
  bitmap[0] |= 0x01;
  if (lseek(fd, 0, SEEK_SET) == -1) {
    LOG_ERROR("Failed to seek file %s to position 0, due to %s .", file_name, strerror(errno));
    close(fd);
    return RC::IOERR_SEEK;
羽飞's avatar
羽飞 已提交
669
  }
羽飞's avatar
羽飞 已提交
670

671
  if (writen(fd, (char *)&page, BP_PAGE_SIZE) != 0) {
羽飞's avatar
羽飞 已提交
672 673 674 675 676 677 678
    LOG_ERROR("Failed to write header to file %s, due to %s.", file_name, strerror(errno));
    close(fd);
    return RC::IOERR_WRITE;
  }

  close(fd);
  LOG_INFO("Successfully create %s.", file_name);
羽飞's avatar
羽飞 已提交
679 680 681
  return RC::SUCCESS;
}

L
Longda Feng 已提交
682
RC BufferPoolManager::open_file(const char *_file_name, DiskBufferPool *&_bp)
羽飞's avatar
羽飞 已提交
683
{
羽飞's avatar
羽飞 已提交
684
  std::string file_name(_file_name);
L
Longda Feng 已提交
685

686
  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
687 688 689 690
  if (buffer_pools_.find(file_name) != buffer_pools_.end()) {
    LOG_WARN("file already opened. file name=%s", _file_name);
    return RC::BUFFERPOOL_OPEN;
  }
羽飞's avatar
羽飞 已提交
691

羽飞's avatar
羽飞 已提交
692 693 694 695 696 697
  DiskBufferPool *bp = new DiskBufferPool(*this, frame_manager_);
  RC rc = bp->open_file(_file_name);
  if (rc != RC::SUCCESS) {
    LOG_WARN("failed to open file name");
    delete bp;
    return rc;
羽飞's avatar
羽飞 已提交
698
  }
羽飞's avatar
羽飞 已提交
699 700 701

  buffer_pools_.insert(std::pair<std::string, DiskBufferPool *>(file_name, bp));
  fd_buffer_pools_.insert(std::pair<int, DiskBufferPool *>(bp->file_desc(), bp));
702
  LOG_DEBUG("insert buffer pool into fd buffer pools. fd=%d, bp=%p, lbt=%s", bp->file_desc(), bp, lbt());
羽飞's avatar
羽飞 已提交
703 704 705 706 707 708 709
  _bp = bp;
  return RC::SUCCESS;
}

RC BufferPoolManager::close_file(const char *_file_name)
{
  std::string file_name(_file_name);
710

羽飞's avatar
羽飞 已提交
711
  lock_.lock();
712

羽飞's avatar
羽飞 已提交
713 714
  auto iter = buffer_pools_.find(file_name);
  if (iter == buffer_pools_.end()) {
羽飞's avatar
羽飞 已提交
715 716
    LOG_TRACE("file has not opened: %s", _file_name);
    lock_.unlock();
羽飞's avatar
羽飞 已提交
717
    return RC::INTERNAL;
羽飞's avatar
羽飞 已提交
718
  }
羽飞's avatar
羽飞 已提交
719 720

  int fd = iter->second->file_desc();
721 722 723 724 725 726 727 728 729 730 731
  if (0 == fd_buffer_pools_.erase(fd)) {
    int count = 0;
    for (auto fd_iter = fd_buffer_pools_.begin(); fd_iter != fd_buffer_pools_.end(); ++fd_iter) {
      if (fd_iter->second == iter->second) {
        fd_buffer_pools_.erase(fd_iter);
        count = 1;
        break;
      }
    }
    ASSERT(count == 1, "the buffer pool was not erased from fd buffer pools.");
  }
羽飞's avatar
羽飞 已提交
732 733 734

  DiskBufferPool *bp = iter->second;
  buffer_pools_.erase(iter);
羽飞's avatar
羽飞 已提交
735 736
  lock_.unlock();
  
羽飞's avatar
羽飞 已提交
737
  delete bp;
羽飞's avatar
羽飞 已提交
738 739
  return RC::SUCCESS;
}
羽飞's avatar
羽飞 已提交
740 741 742 743

RC BufferPoolManager::flush_page(Frame &frame)
{
  int fd = frame.file_desc();
744 745

  std::scoped_lock lock_guard(lock_);
羽飞's avatar
羽飞 已提交
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
  auto iter = fd_buffer_pools_.find(fd);
  if (iter == fd_buffer_pools_.end()) {
    LOG_WARN("unknown buffer pool of fd %d", fd);
    return RC::INTERNAL;
  }

  DiskBufferPool *bp = iter->second;
  return bp->flush_page(frame);
}

static BufferPoolManager *default_bpm = nullptr;
void BufferPoolManager::set_instance(BufferPoolManager *bpm)
{
  if (default_bpm != nullptr && bpm != nullptr) {
    LOG_ERROR("default buffer pool manager has been setted");
    abort();
  }
  default_bpm = bpm;
}
BufferPoolManager &BufferPoolManager::instance()
{
  return *default_bpm;
}