Skip to content

M
miniob

oceanbase / miniob
大约 1 年 前同步成功

通知 74
Star 1521
Fork 537
M
miniob
未验证 提交 93b79cc6 编写于 作者: 羽飞's avatar 羽飞 提交者： GitHub
浏览文件
操作

Refactor (#195) 

### What problem were solved in this pull request?

Issue Number: close #173 close #136 

ref #174 
ref #165 

Problem:
这个PR修改了多个问题，可以参考各个issue。包括observer编译两次、代码目录规划不合理、command executor未全部实现等

### What is changed and how it works?
做一些重构优化，包括CMakelist、一些代码目录、command executor
上级 aea25b25
展开全部 隐藏空白更改
内联 并排
Showing with 1208 addition and 1056 deletion
benchmark/bplus_tree_concurrency_test.cpp
浏览文件 @ 93b79cc6
......@@ -17,7 +17,6 @@ See the Mulan PSL v2 for more details. */
#include "storage/index/bplus_tree.h"
#include "storage/default/disk_buffer_pool.h"
#include "rc.h"
#include "common/log/log.h"
#include "integer_generator.h"
......
benchmark/record_manager_concurrency_test.cpp
浏览文件 @ 93b79cc6
......@@ -21,7 +21,6 @@ See the Mulan PSL v2 for more details. */
#include "storage/default/disk_buffer_pool.h"
#include "storage/common/condition_filter.h"
#include "storage/trx/vacuous_trx.h"
#include "rc.h"
#include "common/log/log.h"
#include "integer_generator.h"
......
build.sh
浏览文件 @ 93b79cc6
......@@ -78,7 +78,7 @@ function do_init
git checkout release-2.1.12-stable && \
mkdir -p build && \
cd build && \
cmake .. -DEVENT__DISABLE_OPENSSL=ON && \
cmake .. -DEVENT__DISABLE_OPENSSL=ON -DEVENT__LIBRARY_TYPE=BOTH && \
make -j4 && \
make install
......
deps/common/CMakeLists.txt
浏览文件 @ 93b79cc6
PROJECT(common)
MESSAGE("Begin to build " ${PROJECT_NAME})
MESSAGE(STATUS "This is PROJECT_BINARY_DIR dir " ${PROJECT_BINARY_DIR})
MESSAGE(STATUS "This is PROJECT_SOURCE_DIR dir " ${PROJECT_SOURCE_DIR})
INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/..)
#find_file() 只会找一个文件
#AUX_SOURCE_DIRECTORY(. SRC_LIST)
#FOREACH(F ${SRC_LIST})
# MESSAGE(${F})
#ENDFOREACH(F)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/deps)
FILE(GLOB_RECURSE ALL_SRC *.cpp)
FOREACH(F ${ALL_SRC})
......@@ -17,7 +9,6 @@ FOREACH(F ${ALL_SRC})
MESSAGE("Use " ${F})
ENDFOREACH(F)
#SHARED，动态库
#STATIC，静态库
ADD_LIBRARY(common STATIC ${ALL_SRC} )
......@@ -25,17 +16,7 @@ ADD_LIBRARY(common STATIC ${ALL_SRC} )
# 编译静态库时，自动会把同名的动态库给删除， 因此需要临时设置一下
SET_TARGET_PROPERTIES(common PROPERTIES CLEAN_DIRECT_OUTPUT 1)
# 设置版本号 VERSION指代动态库版本，SOVERSION指代API版本
SET(MAIJOR_VER 1)
SET(MINOR_VER 0)
SET(PATCH_VER 0)
SET(OTHER_VER 1)
ADD_DEFINITIONS(-DMAIJOR_VER=${MAIJOR_VER} -DMINOR_VER=${MINOR_VER} -DPATCH_VER=${PATCH_VER} -DOTHER_VER=${OTHER_VER})
SET_TARGET_PROPERTIES(common PROPERTIES VERSION ${MAIJOR_VER}.${MINOR_VER}.${PATCH_VER} SOVERSION ${MAIJOR_VER})
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/../../lib)
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
# Target 必须在定义 ADD_EXECUTABLE 之后， programs 不受这个限制
# TARGETS和PROGRAMS 的默认权限是OWNER_EXECUTE, GROUP_EXECUTE, 和WORLD_EXECUTE，即755权限， programs 都是处理脚步类
# 类型分为RUNTIME／LIBRARY／ARCHIVE, prog
......@@ -50,6 +31,6 @@ FILE(GLOB_RECURSE ALL_HEADER *.h)
FOREACH(F ${ALL_HEADER})
file(RELATIVE_PATH RELAPATH_HEADER ${PROJECT_SOURCE_DIR} ${F}) # 获取相对路径
get_filename_component(headDir ${RELAPATH_HEADER} DIRECTORY)
MESSAGE("Install " ${RELAPATH_HEADER} " to " ${CMAKE_INSTALL_PREFIX} "/" ${PROJECT_NAME} "/include/" ${headDir})
MESSAGE("Install " ${RELAPATH_HEADER} " to " ${CMAKE_INSTALL_PREFIX} "/" common "/include/" ${headDir})
INSTALL(FILES ${RELAPATH_HEADER} DESTINATION include/${headDir})
ENDFOREACH(F)
deps/common/defs.h
浏览文件 @ 93b79cc6
......@@ -23,7 +23,6 @@ See the Mulan PSL v2 for more details. */
#include <sys/types.h>
#include <unistd.h>
#include "common/version.h"
namespace common {
#ifndef gettid
......@@ -35,14 +34,8 @@ namespace common {
#endif
inline const std::string &theSwVersion()
enum
{
static const std::string swVersion(VERSION_STR);
return swVersion;
}
enum {
// General Error Codes
STATUS_SUCCESS = 0, //!< Success status should be zero,
STATUS_INVALID_PARAM, //!< Invalid parameter
......@@ -55,32 +48,11 @@ enum {
STATUS_UNKNOW_ERROR,
STATUS_LAST_ERR //!< last error code
};
const unsigned int ONE_KILO = 1024;
const unsigned int ONE_MILLION = ONE_KILO * ONE_KILO;
const unsigned int ONE_GIGA = ONE_MILLION * ONE_KILO;
const unsigned int FILENAME_LENGTH_MAX = 256; // the max filename length
static const char FILE_PATH_SPLIT = '/';
static const char FILE_PATH_SPLIT_STR[] = "/";
/*
* Define types
*
*/
typedef unsigned char u8_t;
typedef unsigned short u16_t;
typedef unsigned int u32_t;
typedef unsigned long long u64_t;
typedef char s8_t;
typedef short s16_t;
typedef int s32_t;
typedef long long s64_t;
#define LOCAL_HOST "localhost"
#define EPSILON (1E-6)
} // namespace common
deps/common/io/io.cpp
浏览文件 @ 93b79cc6
......@@ -79,7 +79,7 @@ int readFromFile(const std::string &fileName, char *&outputData, size_t &fileSiz
return 0;
}
int writeToFile(const std::string &fileName, const char *data, u32_t dataSize, const char *openMode)
int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize, const char *openMode)
{
FILE *file = fopen(fileName.c_str(), openMode);
if (file == NULL) {
......@@ -87,7 +87,7 @@ int writeToFile(const std::string &fileName, const char *data, u32_t dataSize, c
return -1;
}
u32_t leftSize = dataSize;
uint32_t leftSize = dataSize;
const char *buffer = data;
while (leftSize > 0) {
int writeCount = fwrite(buffer, 1, leftSize, file);
......@@ -106,7 +106,7 @@ int writeToFile(const std::string &fileName, const char *data, u32_t dataSize, c
return 0;
}
int getFileLines(const std::string &fileName, u64_t &lineNum)
int getFileLines(const std::string &fileName, uint64_t &lineNum)
{
lineNum = 0;
......@@ -130,7 +130,7 @@ int getFileLines(const std::string &fileName, u64_t &lineNum)
return 0;
}
int getFileNum(u64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive)
int getFileNum(int64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive)
{
try {
DIR *dirp = NULL;
......@@ -327,7 +327,7 @@ int touch(const std::string &path)
return 0;
}
int getFileSize(const char *filePath, u64_t &fileLen)
int getFileSize(const char *filePath, int64_t &fileLen)
{
if (filePath == NULL || *filePath == '\0') {
std::cerr << "invalid filepath" << std::endl;
......
deps/common/io/io.h
浏览文件 @ 93b79cc6
......@@ -28,12 +28,12 @@ namespace common {
*/
int readFromFile(const std::string &fileName, char *&data, size_t &fileSize);
int writeToFile(const std::string &fileName, const char *data, u32_t dataSize, const char *openMode);
int writeToFile(const std::string &fileName, const char *data, uint32_t dataSize, const char *openMode);
/**
* return the line number which line.strip() isn't empty
*/
int getFileLines(const std::string &fileName, u64_t &lineNum);
int getFileLines(const std::string &fileName, uint64_t &lineNum);
/** Get file list from the dir
* don't care ".", "..", ".****" hidden files
......@@ -47,7 +47,7 @@ int getFileLines(const std::string &fileName, u64_t &lineNum);
*/
int getFileList(
std::vector<std::string> &fileList, const std::string &path, const std::string &pattern, bool recursive);
int getFileNum(u64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive);
int getFileNum(uint64_t &fileNum, const std::string &path, const std::string &pattern, bool recursive);
int getDirList(std::vector<std::string> &dirList, const std::string &path, const std::string &pattern);
int touch(const std::string &fileName);
......@@ -55,7 +55,7 @@ int touch(const std::string &fileName);
/**
* get file size
*/
int getFileSize(const char *filePath, u64_t &fileLen);
int getFileSize(const char *filePath, uint64_t &fileLen);
/**
* @brief 一次性写入所有指定数据
......
deps/common/io/roll_select_dir.h
浏览文件 @ 93b79cc6
......@@ -46,7 +46,7 @@ public:
std::string mBaseDir;
std::vector<std::string> mSubdirs;
pthread_mutex_t mMutex;
u32_t mPos;
uint32_t mPos;
};
} // namespace common
......
deps/common/log/log.h
浏览文件 @ 93b79cc6
......@@ -31,6 +31,11 @@ See the Mulan PSL v2 for more details. */
namespace common {
const unsigned int ONE_KILO = 1024;
const unsigned int ONE_MILLION = ONE_KILO * ONE_KILO;
const unsigned int ONE_GIGA = ONE_MILLION * ONE_KILO;
const unsigned int FILENAME_LENGTH_MAX = 256; // the max filename length
const int LOG_STATUS_OK = 0;
const int LOG_STATUS_ERR = 1;
const int LOG_MAX_LINE = 100000;
......@@ -182,7 +187,7 @@ extern Log *g_log;
p->tm_min, \
p->tm_sec, \
usec, \
(u32_t)getpid(), \
(int32_t)getpid(), \
gettid(), \
common::g_log->context_id()); \
common::g_log->rotate(p->tm_year + 1900, p->tm_mon + 1, p->tm_mday); \
......@@ -194,7 +199,7 @@ extern Log *g_log;
(common::g_log)->prefix_msg(level), \
__FUNCTION__, \
__FILE_NAME__, \
(u32_t)__LINE__ \
(int32_t)__LINE__ \
); \
}
......
deps/common/mm/mem.cpp.skip 已删除 100644 → 0
浏览文件 @ aea25b25
// __CR__
// Copyright (c) 2021 LongdaFeng All Rights Reserved
//
// This software contains the intellectual property of LongdaFeng
// or is licensed to LongdaFeng from third parties. Use of this
// software and the intellectual property contained therein is
// expressly limited to the terms and conditions of the License Agreement
// under which it is provided by or on behalf of LongdaFeng.
// __CR__
//
// Created by Longda on 2010
//
#ifdef MEM_DEBUG
#include <cstddef>
#include <stdlib.h>
#include <string.h>
#include "lang/mutex.h"
#include "mm/lmem.h"
#define MEM_ID_HASH(p) (((unsigned long)(p) >> 8) % MEM_HASHTABLE_SIZE)
pthread_mutex_t CLMemTrace::mMutex = PTHREAD_MUTEX_INITIALIZER;
u64_t CLMemTrace::mUsedSize = 0;
MemID *CLMemTrace::mMemIDs[MEM_HASHTABLE_SIZE] = {0};
bool CLMemTrace::mVerbose = false;
;
void *CLMemTrace::malloc(size_t size, const char *file, const int line, bool retry) throw(std::bad_alloc)
{
size_t allocSize = size + sizeof(MemID);
void *usedPointer = NULL;
do {
MemID *ptr = (MemID *)::malloc(allocSize);
if (ptr) {
// successfully alloc memory
// set the MemID
strncpy(ptr->mFile, file, MemID::MEM_FILENAME_LEN - 1);
ptr->mFile[MemID::MEM_FILENAME_LEN - 1] = '\0';
ptr->mLine = line;
ptr->mSize = size;
usedPointer = (char *)ptr + sizeof(MemID);
u64_t hashIndex = (u64_t)MEM_ID_HASH(usedPointer);
MUTEX_LOCK(&mMutex);
ptr->mNext = mMemIDs[hashIndex];
mMemIDs[hashIndex] = ptr;
mUsedSize += size;
MUTEX_UNLOCK(&mMutex);
if (mVerbose) {
LOG_INFO("%s:%d alloc %llu memory %p", file, line, size, usedPointer);
}
return usedPointer;
}
if (retry == false) {
throw std::bad_alloc();
break;
}
std::new_handler allocHandler = getNewHandler();
if (allocHandler) {
(*allocHandler)();
} else {
throw std::bad_alloc();
break;
}
} while (retry);
return NULL;
}
void *CLMemTrace::realloc(void *pointer, size_t size, const char *file, const int line)
{
if (pointer == NULL && size == 0) {
return NULL;
} else if (pointer == NULL && size != 0) {
try {
return malloc(size, file, line);
} catch (std::bad_alloc &e) {
LOG_WARN("NO memory to alloc for %llu", size);
return NULL;
}
} else if (pointer && size == 0) {
free(pointer);
return NULL;
}
// the left case, ptr && size
MemID *pMemID = NULL;
MemID *pLast = NULL;
MemID *pFreeMemID = NULL;
MemID *pNewMemID = NULL;
MemID oldMemID;
bool foundOld = false;
// use big lock
MUTEX_LOCK(&mMutex);
u64_t hashIndex = MEM_ID_HASH(pointer);
pMemID = mMemIDs[hashIndex];
while (pMemID) {
if ((char *)pMemID + sizeof(MemID) != pointer) {
// not found
pLast = pMemID;
pMemID = pMemID->mNext;
continue;
}
// find
foundOld = true;
// backup old MemID firstly
memcpy(&oldMemID, pMemID, sizeof(MemID));
u64_t allocSize = size + sizeof(MemID);
pNewMemID = (MemID *)::realloc(pMemID, allocSize);
if (pNewMemID == NULL) {
// case 1:no memory to alloc, free the old one
if (pLast == NULL) {
mMemIDs[hashIndex] = pMemID->mNext;
} else {
pLast->mNext = pMemID->mNext;
}
pFreeMemID = pMemID;
mUsedSize -= oldMemID.mSize;
break;
}
// set the new pNewMemID
strncpy(pNewMemID->mFile, file, MemID::MEM_FILENAME_LEN - 1);
pNewMemID->mFile[MemID::MEM_FILENAME_LEN - 1] = '\0';
pNewMemID->mLine = line;
pNewMemID->mSize = size;
mUsedSize -= oldMemID.mSize;
mUsedSize += size;
if (pNewMemID == pMemID) {
// case 2: just extension the old memory
pFreeMemID = NULL;
break;
} else {
// case 3: the old memory can't meet the requirement, alloc new
/**
* Firstly, remove the old one from table
* don't add new before remove the old one
*/
if (pLast == NULL) {
mMemIDs[hashIndex] = pMemID->mNext;
} else {
pLast->mNext = pMemID->mNext;
}
pFreeMemID = pMemID;
/**
* Secondly, add the new one to table
*/
u64_t newHashIndex = (u64_t)MEM_ID_HASH((char *)pNewMemID + sizeof(MemID));
pNewMemID->mNext = mMemIDs[newHashIndex];
mMemIDs[newHashIndex] = pNewMemID;
/**
* Third, do memory copy
* to simplify the old logic, copy memory here
*/
memcpy((char *)pNewMemID + sizeof(MemID), (char *)pFreeMemID + sizeof(MemID), pFreeMemID->mSize);
break;
}
}
MUTEX_UNLOCK(&mMutex);
if (foundOld == false) {
LOG_WARN("Something is wrong, the old pointer %p isn't found, so alloc new one", pointer);
try {
return malloc(size, file, line, false);
} catch (std::bad_alloc &e) {
LOG_WARN("NO memory to alloc for %llu", size);
return NULL;
};
}
if (mVerbose) {
LOG_INFO("Delete %p, file:%s, line:%u, size:%llu", pointer, oldMemID.mFile, oldMemID.mLine, oldMemID.mSize);
}
if (pFreeMemID) {
::free(pFreeMemID);
}
if (pNewMemID) {
if (mVerbose) {
LOG_INFO("Alloc %p, file:%s, line:%u, size:%llu",
(char *)pNewMemID + sizeof(MemID),
pNewMemID->mFile,
pNewMemID->mLine,
pNewMemID->mSize);
}
return pNewMemID;
}
return NULL;
}
void CLMemTrace::free(void *pointer)
{
if (pointer == NULL) {
LOG_WARN("Free one empty pointer");
return;
}
u64_t hashIndex = MEM_ID_HASH(pointer);
MemID *pMemID = NULL;
MemID *pLast = NULL;
// use big lock
MUTEX_LOCK(&mMutex);
pMemID = mMemIDs[hashIndex];
while (pMemID) {
if ((char *)pMemID + sizeof(MemID) == pointer) {
// find
if (pLast == NULL) {
mMemIDs[hashIndex] = pMemID->mNext;
} else {
pLast->mNext = pMemID->mNext;
}
mUsedSize -= pMemID->mSize;
break;
} else {
pLast = pMemID;
pMemID = pMemID->mNext;
}
}
MUTEX_UNLOCK(&mMutex);
if (pMemID) {
if (mVerbose) {
LOG_INFO("Delete %p, file:%s, line:%u, size:%llu", pointer, pMemID->mFile, pMemID->mLine, pMemID->mSize);
}
::free(pMemID);
return;
} else {
// not found
LOG_ERROR("Double free for pointer :%p", pointer);
}
return;
}
std::new_handler CLMemTrace::getNewHandler()
{
std::new_handler newHandler = NULL;
MUTEX_LOCK(&mMutex);
newHandler = std::set_new_handler(0);
std::set_new_handler(newHandler);
MUTEX_UNLOCK(&mMutex);
return newHandler;
}
void CLMemTrace::output()
{
for (int i = 0; i < MEM_HASHTABLE_SIZE; ++i) {
// Don't lock outside of the loop
// 1. avoid output too long to alloc/free memory
// 2. if LOG_INFO alloc memory, it will leading to dead loop
MUTEX_LOCK(&mMutex);
MemID *ptr = mMemIDs[i];
if (ptr == NULL) {
MUTEX_UNLOCK(&mMutex);
continue;
}
while (ptr) {
// if LOG_INFO alloc memory, it will easy leading to dead lock
LOG_INFO(
"Exist %p, file:%s, line:%u, size:%llu", (char *)ptr + sizeof(MemID), ptr->mFile, ptr->mLine, ptr->mSize);
ptr = ptr->mNext;
}
MUTEX_UNLOCK(&mMutex);
}
}
void *operator new(std::size_t size, const char *file, int line)
{
return CLMemTrace::malloc(size, file, line, true);
}
void *operator new[](std::size_t size, const char *file, int line)
{
return operator new(size, file, line);
}
void *operator new(std::size_t size) throw(std::bad_alloc)
{
return operator new(size, "<Unknown>", 0);
}
void *operator new[](std::size_t size) throw(std::bad_alloc)
{
return operator new(size);
}
void *operator new(std::size_t size, const std::nothrow_t &) throw()
{
void *pointer = NULL;
try {
pointer = operator new(size);
} catch (std::bad_alloc &e) {
LOG_WARN("Failed to alloc memory");
return NULL;
}
return pointer;
}
void *operator new[](std::size_t size, const std::nothrow_t &) throw()
{
void *pointer = NULL;
try {
pointer = operator[] new(size);
} catch (std::bad_alloc &e) {
LOG_WARN("Failed to alloc memory");
return NULL;
}
return pointer;
}
void operator delete(void *pointer)
{
CLMemTrace::free(pointer);
}
void operator delete[](void *pointer)
{
operator delete(pointer);
}