Skip to content

O
oceanbase

soremi / oceanbase
与 Fork 源项目一致

Fork自 oceanbase / oceanbase

通知 1
Star 0
Fork 0
O
oceanbase

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 71d86596 编写于 作者: O obdev 提交者： wangzelin.wzl
浏览文件
操作

add task hash map to das ref

上级 3eef7b05
隐藏空白更改
内联 并排
Showing with 426 addition and 89 deletion
src/sql/das/ob_das_context.cpp
浏览文件 @ 71d86596
......@@ -135,10 +135,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc,
ObDASTabletLoc *&tablet_loc)
{
int ret = OB_SUCCESS;
FOREACH(tmp_node, table_loc.tablet_locs_) {
if ((*tmp_node)->tablet_id_ == tablet_id) {
tablet_loc = *tmp_node;
}
if (OB_FAIL(table_loc.get_tablet_loc_by_id(tablet_id, tablet_loc))) {
LOG_WARN("get tablet loc failed", KR(ret));
}
if (OB_SUCC(ret) && tablet_loc == nullptr) {
LOG_DEBUG("tablet location is not exists, begin to construct it", K(table_loc), K(tablet_id));
......@@ -152,7 +150,7 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc,
tablet_id,
*tablet_loc))) {
LOG_WARN("nonblock get tablet location failed", K(ret), KPC(table_loc.loc_meta_), K(tablet_id));
} else if (OB_FAIL(table_loc.tablet_locs_.push_back(tablet_loc))) {
} else if (OB_FAIL(table_loc.add_tablet_loc(tablet_loc))) {
LOG_WARN("store tablet location info failed", K(ret));
} else {
tablet_loc->loc_meta_ = table_loc.loc_meta_;
......@@ -178,7 +176,9 @@ int ObDASCtx::check_same_server(const ObDASTabletLoc *tablet_loc)
ObDASTabletLoc *first_tablet = NULL;
FOREACH_X(table_node, table_locs_, NULL == first_tablet) {
ObDASTableLoc *cur_table_loc = *table_node;
FOREACH_X(tablet_node, cur_table_loc->tablet_locs_, NULL == first_tablet) {
for (DASTabletLocListIter tablet_node = cur_table_loc->tablet_locs_begin();
NULL == first_tablet && tablet_node != cur_table_loc->tablet_locs_end();
++tablet_node) {
first_tablet = *tablet_node;
}
}
......@@ -198,10 +198,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc,
{
int ret = OB_SUCCESS;
const ObOptTabletLoc &opt_tablet_loc = candi_tablet_loc.get_partition_location();
FOREACH(tmp_node, table_loc.tablet_locs_) {
if ((*tmp_node)->tablet_id_ == opt_tablet_loc.get_tablet_id()) {
tablet_loc = *tmp_node;
}
if (OB_FAIL(table_loc.get_tablet_loc_by_id(opt_tablet_loc.get_tablet_id(), tablet_loc))) {
LOG_WARN("get tablet loc failed", KR(ret), K(opt_tablet_loc.get_tablet_id()));
}
if (OB_SUCC(ret) && tablet_loc == nullptr) {
ObLSReplicaLocation replica_loc;
......@@ -217,7 +215,7 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc,
tablet_loc->tablet_id_ = opt_tablet_loc.get_tablet_id();
tablet_loc->ls_id_ = opt_tablet_loc.get_ls_id();
tablet_loc->loc_meta_ = table_loc.loc_meta_;
if (OB_FAIL(table_loc.tablet_locs_.push_back(tablet_loc))) {
if (OB_FAIL(table_loc.add_tablet_loc(tablet_loc))) {
LOG_WARN("store tablet loc failed", K(ret), K(tablet_loc));
}
}
......@@ -260,7 +258,7 @@ OB_INLINE int ObDASCtx::build_related_tablet_loc(ObDASTabletLoc &tablet_loc)
related_tablet_loc->loc_meta_ = related_table_loc->loc_meta_;
related_tablet_loc->next_ = tablet_loc.next_;
tablet_loc.next_ = related_tablet_loc;
if (OB_FAIL(related_table_loc->tablet_locs_.push_back(related_tablet_loc))) {
if (OB_FAIL(related_table_loc->add_tablet_loc(related_tablet_loc))) {
LOG_WARN("add related tablet location failed", K(ret));
}
}
......@@ -274,7 +272,8 @@ OB_INLINE int ObDASCtx::build_related_table_loc(ObDASTableLoc &table_loc)
{
int ret = OB_SUCCESS;
if (!table_loc.loc_meta_->related_table_ids_.empty()) {
FOREACH_X(node, table_loc.tablet_locs_, OB_SUCC(ret)) {
for (DASTabletLocListIter node = table_loc.tablet_locs_begin();
OB_SUCC(ret) && node != table_loc.tablet_locs_end(); ++node) {
ObDASTabletLoc *tablet_loc = *node;
if (OB_FAIL(build_related_tablet_loc(*tablet_loc))) {
LOG_WARN("build related tablet loc failed", K(ret));
......@@ -353,7 +352,7 @@ int ObDASCtx::add_candi_table_loc(const ObDASTableLocMeta &loc_meta,
LOG_WARN("extended tablet loc failed", K(ret));
}
}
LOG_TRACE("das table loc assign finish", K(candi_table_loc), K(loc_meta), K(table_loc->tablet_locs_));
LOG_TRACE("das table loc assign finish", K(candi_table_loc), K(loc_meta), K(table_loc->get_tablet_locs()));
return ret;
}
......@@ -363,7 +362,8 @@ bool ObDASCtx::has_same_lsid(ObLSID *lsid)
ObLSID first_lsid;
FOREACH_X(table_node, table_locs_, bret) {
ObDASTableLoc *table_loc = *table_node;
FOREACH_X(tablet_node, table_loc->tablet_locs_, bret) {
for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin();
bret && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) {
ObDASTabletLoc *tablet_loc = *tablet_node;
if (!first_lsid.is_valid()) {
first_lsid = tablet_loc->ls_id_;
......@@ -386,7 +386,7 @@ int64_t ObDASCtx::get_related_tablet_cnt() const
int64_t total_cnt = 0;
FOREACH(table_node, table_locs_) {
ObDASTableLoc *table_loc = *table_node;
total_cnt += table_loc->tablet_locs_.size();
total_cnt += table_loc->get_tablet_locs().size();
}
return total_cnt;
......@@ -408,14 +408,14 @@ int ObDASCtx::rebuild_tablet_loc_reference()
ObTableID related_table_id = table_loc->loc_meta_->related_table_ids_.at(i);
ObDASTableLoc *related_table_loc = get_table_loc_by_id(table_loc_id, related_table_id);
related_table_loc->rebuild_reference_ = 1;
if (table_loc->tablet_locs_.size() != related_table_loc->tablet_locs_.size()) {
if (table_loc->get_tablet_locs().size() != related_table_loc->get_tablet_locs().size()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet location count not matched", K(ret),
KPC(table_loc), KPC(related_table_loc));
}
DASTabletLocList::iterator tablet_iter = table_loc->tablet_locs_.begin();
DASTabletLocList::iterator related_tablet_iter = related_table_loc->tablet_locs_.begin();
for (; OB_SUCC(ret) && tablet_iter != table_loc->tablet_locs_.end();
DASTabletLocList::iterator tablet_iter = table_loc->tablet_locs_begin();
DASTabletLocList::iterator related_tablet_iter = related_table_loc->tablet_locs_begin();
for (; OB_SUCC(ret) && tablet_iter != table_loc->tablet_locs_end();
++tablet_iter, ++related_tablet_iter) {
ObDASTabletLoc *tablet_loc = *tablet_iter;
ObDASTabletLoc *related_tablet_loc = *related_tablet_iter;
......@@ -460,8 +460,8 @@ bool ObDASCtx::is_partition_hit()
{
bool bret = true;
if (same_server_) {
if (!table_locs_.empty() && !table_locs_.get_first()->tablet_locs_.empty()) {
if (MYADDR == table_locs_.get_first()->tablet_locs_.get_first()->server_) {
if (!table_locs_.empty() && !table_locs_.get_first()->get_tablet_locs().empty()) {
if (MYADDR == table_locs_.get_first()->get_first_tablet_loc()->server_) {
// all local partitions
bret = true;
} else {
......@@ -476,7 +476,7 @@ bool ObDASCtx::is_partition_hit()
// For background, please see comments for ObDASCtx::is_partition_hit().
void ObDASCtx::unmark_need_check_server()
{
if (!table_locs_.empty() && !table_locs_.get_first()->tablet_locs_.empty()) {
if (!table_locs_.empty() && !table_locs_.get_first()->get_tablet_locs().empty()) {
need_check_server_ = false;
}
}
......
src/sql/das/ob_das_define.cpp
浏览文件 @ 71d86596
......@@ -164,5 +164,176 @@ int ObDASTableLoc::assign(const ObCandiTableLoc &candi_table_loc)
LOG_DEBUG("das table loc assign", K(candi_table_loc), KPC_(loc_meta), K(tablet_locs_));
return ret;
}
int ObDASTableLoc::get_tablet_loc_by_id(const ObTabletID &tablet_id,
ObDASTabletLoc *&tablet_loc)
{
int ret = OB_SUCCESS;
tablet_loc = NULL;
lookup_cnt_++;
if (tablet_locs_map_.created()) {
if (OB_FAIL(tablet_locs_map_.get(tablet_id, tablet_loc))) {
if (OB_HASH_NOT_EXIST != ret) {
LOG_WARN("look up from hash map failed", KR(ret), K(tablet_id));
}
}
}
if (OB_SUCC(ret) && NULL != tablet_loc) {
// found in hash map
} else if (OB_HASH_NOT_EXIST == ret) {
// key not found
ret = OB_SUCCESS;
} else {
FOREACH(tmp_node, tablet_locs_) {
ObDASTabletLoc *tmp_tablet_loc = *tmp_node;
if (tmp_tablet_loc->tablet_id_ == tablet_id) {
tablet_loc = tmp_tablet_loc;
break;
}
}
}
if (OB_FAIL(ret) || tablet_locs_map_.created()) {
// do nothing
} else if (lookup_cnt_ > DAS_TABLET_LOC_LOOKUP_THRESHOLD
&& tablet_locs_.size() > DAS_TABLET_LOC_SIZE_THRESHOLD
&& OB_FAIL(create_tablet_locs_map())) {
LOG_WARN("create tablet locs hash map failed", KR(ret));
}
return ret;
}
int ObDASTableLoc::add_tablet_loc(ObDASTabletLoc *tablet_loc)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(tablet_loc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet loc is null", KR(ret), KP(tablet_loc));
} else if (OB_FAIL(tablet_locs_.push_back(tablet_loc))) {
LOG_WARN("push into tablet locs list failed", KR(ret), KPC(tablet_loc));
} else if (tablet_locs_map_.created()) {
if (OB_FAIL(tablet_locs_map_.set(tablet_loc->tablet_id_, tablet_loc))) {
LOG_WARN("insert into tablet locs map failed", KR(ret), KPC(tablet_loc));
}
}
return ret;
}
int TabletHashMap::create(int64_t bucket_num)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(0 >= bucket_num)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid bucket number", KR(ret), K(bucket_num));
} else if (created()) {
ret = OB_INIT_TWICE;
LOG_WARN("hash map was already created", KR(ret));
} else if (FALSE_IT(bucket_num = hash::cal_next_prime(bucket_num))) {
} else if (OB_ISNULL(buckets_ = static_cast<TabletHashNode **>(
allocator_.alloc(bucket_num * sizeof(TabletHashNode *))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc buckets failed", KR(ret));
} else {
MEMSET(buckets_, 0, bucket_num * sizeof(TabletHashNode *));
bucket_num_ = bucket_num;
is_inited_ = true;
}
return ret;
}
int TabletHashMap::find_node(const ObTabletID key,
TabletHashNode *head,
TabletHashNode *&node) const
{
int ret = OB_SUCCESS;
node = NULL;
if (!created()) {
ret = OB_NOT_INIT;
LOG_WARN("hash map was not created", KR(ret));
} else if (NULL == head) {
// do nothing
} else {
TabletHashNode *cur = head;
while (NULL != cur && NULL == node) {
if (cur->key_ == key) {
node = cur;
} else {
cur = cur->next_;
}
}
}
return ret;
}
int TabletHashMap::set(const ObTabletID key, ObDASTabletLoc *value)
{
int ret = OB_SUCCESS;
if (!created()) {
ret = OB_NOT_INIT;
LOG_WARN("hash map was not created", KR(ret));
} else {
TabletHashNode *&bucket = buckets_[key.hash() % bucket_num_];
TabletHashNode *dst_node = NULL;
if (OB_FAIL(find_node(key, bucket, dst_node))) {
LOG_WARN("find node failed", KR(ret));
} else if (NULL != dst_node) {
ret = OB_HASH_EXIST;
LOG_WARN("key already exists", KR(ret), K(key), KP(value));
} else {
TabletHashNode *new_node = static_cast<TabletHashNode *>(
allocator_.alloc(sizeof(TabletHashNode)));
if (OB_ISNULL(new_node)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc new node failed", KR(ret));
} else {
new_node->key_ = key;
new_node->value_ = value;
new_node->next_ = bucket;
bucket = new_node;
}
}
}
return ret;
}
int TabletHashMap::get(const ObTabletID key, ObDASTabletLoc *&value)
{
int ret = OB_SUCCESS;
if (!created()) {
ret = OB_NOT_INIT;
LOG_WARN("hash map was not created", KR(ret));
} else {
TabletHashNode *&bucket = buckets_[key.hash() % bucket_num_];
TabletHashNode *dst_node = NULL;
if (OB_FAIL(find_node(key, bucket, dst_node))) {
LOG_WARN("find node failed", KR(ret));
} else if (NULL == dst_node) {
ret = OB_HASH_NOT_EXIST;
LOG_WARN("key dost not exist", KR(ret), K(key));
} else {
value = dst_node->value_;
}
}
return ret;
}
int ObDASTableLoc::create_tablet_locs_map()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(tablet_locs_map_.created())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet locs map was already created", KR(ret), K(tablet_locs_map_.created()));
} else if (OB_FAIL(tablet_locs_map_.create(DAS_TABLET_LOC_MAP_BUCKET_SIZE))) {
LOG_WARN("create tablet locs map failed", KR(ret));
} else {
for (DASTabletLocListIter tablet_node = tablet_locs_begin();
OB_SUCC(ret) && tablet_node != tablet_locs_end(); ++tablet_node) {
ObDASTabletLoc *tablet_loc = *tablet_node;
if (OB_FAIL(tablet_locs_map_.set(tablet_loc->tablet_id_, tablet_loc))) {
LOG_WARN("insert into tablet locs map failed", KR(ret), KPC(tablet_loc));
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase
src/sql/das/ob_das_define.h
浏览文件 @ 71d86596
......@@ -172,10 +172,42 @@ private:
int assign(const ObDASTabletLoc &other);
};
static const int64_t DAS_TABLET_LOC_LOOKUP_THRESHOLD = 1000;
static const int64_t DAS_TABLET_LOC_SIZE_THRESHOLD = 10;
static const int64_t DAS_TABLET_LOC_MAP_BUCKET_SIZE = 5000;
typedef common::ObList<ObDASTabletLoc*, common::ObIAllocator> DASTabletLocList;
typedef common::ObList<ObDASTabletLoc*, common::ObIAllocator>::iterator DASTabletLocListIter;
typedef common::ObIArray<ObDASTabletLoc*> DASTabletLocIArray;
typedef common::ObSEArray<ObDASTabletLoc*, 1> DASTabletLocSEArray;
typedef common::ObArray<ObDASTabletLoc*> DASTabletLocArray;
class TabletHashMap
{
struct TabletHashNode
{
ObTabletID key_;
ObDASTabletLoc *value_;
struct TabletHashNode *next_;
};
public:
TabletHashMap(common::ObIAllocator &allocator)
: allocator_(allocator), bucket_num_(0), buckets_(NULL), is_inited_(false)
{}
virtual ~TabletHashMap() {} // does not free memory
int create(int64_t bucket_num);
bool created() const { return is_inited_; }
int set(const ObTabletID key, ObDASTabletLoc *value);
int get(const ObTabletID key, ObDASTabletLoc *&value);
private:
int find_node(const ObTabletID key, TabletHashNode *head, TabletHashNode *&node) const;
common::ObIAllocator &allocator_;
int64_t bucket_num_;
TabletHashNode **buckets_;
bool is_inited_;
};
/**
* store the location information of which tables are accessed in this plan
* generate this table location when this plan is chosen in the plan cache or generated this plan by CG
......@@ -188,8 +220,10 @@ public:
ObDASTableLoc(common::ObIAllocator &allocator)
: allocator_(allocator),
loc_meta_(nullptr),
flags_(0),
tablet_locs_(allocator),
flags_(0)
tablet_locs_map_(allocator),
lookup_cnt_(0)
{ }
~ObDASTableLoc() = default;
......@@ -197,6 +231,12 @@ public:
int64_t get_table_location_key() const { return loc_meta_->table_loc_id_; }
int64_t get_ref_table_id() const { return loc_meta_->ref_table_id_; }
bool empty() const { return tablet_locs_.size() == 0; }
const DASTabletLocList &get_tablet_locs() const { return tablet_locs_; }
ObDASTabletLoc *get_first_tablet_loc() { return tablet_locs_.get_first(); }
DASTabletLocListIter tablet_locs_begin() { return tablet_locs_.begin(); }
DASTabletLocListIter tablet_locs_end() { return tablet_locs_.end(); }
int get_tablet_loc_by_id(const ObTabletID &tablet_id, ObDASTabletLoc *&tablet_loc);
int add_tablet_loc(ObDASTabletLoc *table_loc);
TO_STRING_KV(KPC_(loc_meta),
K_(tablet_locs),
......@@ -213,21 +253,6 @@ public:
*/
common::ObIAllocator &allocator_;
const ObDASTableLocMeta *loc_meta_;
/**
* The reason for using ObList to store ObTabletLoc objects is that
* during the execution process,
* some operators will dynamically extend some new partitions,
* so tablet_locs_ needs to be able to expand dynamically.
* ObArray/ObSEArray will cause dynamic extension or release of memory when expanding members.
* At the same time, in partition task retry,
* it is very convenient to clear the invalid TabletLoc object,
* so ObList is more suitable for its needs.
* Why not use ObDList?
* Considering that ObTabletLoc may be referenced by list of other modules, such as PX,
* and elements in ObDList can only be referenced by one list at the same time,
* ObDList has more restrictions and is not suitable as a container for ObTabletLoc
**/
DASTabletLocList tablet_locs_;
union {
/**
* used to mark some status related to table access,
......@@ -242,9 +267,29 @@ public:
uint64_t reserved_ : 60;
};
};
private:
DISALLOW_COPY_AND_ASSIGN(ObDASTableLoc);
int assign(const ObDASTableLoc &other);
int create_tablet_locs_map();
/**
* The reason for using ObList to store ObTabletLoc objects is that
* during the execution process,
* some operators will dynamically extend some new partitions,
* so tablet_locs_ needs to be able to expand dynamically.
* ObArray/ObSEArray will cause dynamic extension or release of memory when expanding members.
* At the same time, in partition task retry,
* it is very convenient to clear the invalid TabletLoc object,
* so ObList is more suitable for its needs.
* Why not use ObDList?
* Considering that ObTabletLoc may be referenced by list of other modules, such as PX,
* and elements in ObDList can only be referenced by one list at the same time,
* ObDList has more restrictions and is not suitable as a container for ObTabletLoc
**/
DASTabletLocList tablet_locs_;
TabletHashMap tablet_locs_map_;
int64_t lookup_cnt_;
};
typedef common::ObList<ObDASTableLoc*, common::ObIAllocator> DASTableLocList;
typedef common::ObFixedArray<uint64_t, common::ObIAllocator> UIntFixedArray;
......
src/sql/das/ob_das_ref.cpp
浏览文件 @ 71d86596
......@@ -24,6 +24,19 @@ namespace oceanbase
using namespace common;
namespace sql
{
bool DasRefKey::operator==(const DasRefKey &other) const
{
return (tablet_loc_ == other.tablet_loc_ && op_type_ == other.op_type_);
}
uint64_t DasRefKey::hash() const
{
uint64_t hash = 0;
hash = murmurhash(&tablet_loc_, sizeof(tablet_loc_), hash);
hash = murmurhash(&op_type_, sizeof(op_type_), hash);
return hash;
}
ObDASRef::ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx)
: das_alloc_(exec_ctx.get_allocator()),
reuse_alloc_(nullptr),
......@@ -34,6 +47,9 @@ ObDASRef::ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx)
frozen_op_node_(nullptr),
expr_frame_info_(nullptr),
wild_datum_info_(eval_ctx),
lookup_cnt_(0),
task_cnt_(0),
task_map_(),
flags_(0)
{
}
......@@ -45,22 +61,91 @@ DASOpResultIter ObDASRef::begin_result_iter()
ObIDASTaskOp* ObDASRef::find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type)
{
int ret = OB_SUCCESS;
ObIDASTaskOp *das_task = nullptr;
if (nullptr == frozen_op_node_) {
frozen_op_node_ = batched_tasks_.get_header_node();
}
DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node());
for (; nullptr == das_task && !task_iter.is_end(); ++task_iter) {
ObIDASTaskOp *tmp_task = *task_iter;
if (tmp_task != nullptr &&
tmp_task->get_tablet_loc() == tablet_loc &&
tmp_task->get_type() == op_type) {
das_task = tmp_task;
lookup_cnt_++;
if (task_map_.created()) {
DasRefKey key(tablet_loc, op_type);
if (OB_FAIL(task_map_.get_refactored(key, das_task))) {
if (OB_HASH_NOT_EXIST != ret) {
LOG_WARN("look up from hash map failed", KR(ret), KP(tablet_loc), K(op_type));
}
}
}
if (OB_SUCC(ret) && NULL != das_task) {
// found in hash map
} else if (OB_HASH_NOT_EXIST == ret) {
// key not found
} else {
DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node());
for (; nullptr == das_task && !task_iter.is_end(); ++task_iter) {
ObIDASTaskOp *tmp_task = *task_iter;
if (tmp_task != nullptr &&
tmp_task->get_tablet_loc() == tablet_loc &&
tmp_task->get_type() == op_type) {
das_task = tmp_task;
}
}
}
if (OB_FAIL(ret) || task_map_.created()) {
// do nothing
} else if (lookup_cnt_ > DAS_REF_TASK_LOOKUP_THRESHOLD
&& task_cnt_ > DAS_REF_TASK_SIZE_THRESHOLD
&& OB_FAIL(create_task_map())) {
LOG_WARN("create task hash map failed", KR(ret));
}
return das_task;
}
int ObDASRef::create_task_map()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(task_map_.created())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task map was already created", KR(ret), K(task_map_.created()));
} else if (OB_FAIL(task_map_.create(DAS_REF_MAP_BUCKET_SIZE, ObModIds::OB_HASH_BUCKET))) {
LOG_WARN("create task map failed", KR(ret));
} else {
DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node());
for (; OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) {
ObIDASTaskOp *task = *task_iter;
if (OB_ISNULL(task)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("task is null", KR(ret), KP(task));
} else {
DasRefKey key(task->get_tablet_loc(), task->get_type());
if (OB_FAIL(task_map_.set_refactored(key, task))) {
LOG_WARN("insert into task map failed", KR(ret), K(key), KP(task));
}
}
}
if (OB_FAIL(ret)) {
task_map_.destroy();
}
}
return ret;
}
int ObDASRef::add_batched_task(ObIDASTaskOp *das_task)
{
int ret = OB_SUCCESS;
if (OB_FAIL(batched_tasks_.store_obj(das_task))) {
LOG_WARN("store das task failed", KR(ret));
} else if (task_map_.created()) {
DasRefKey key(das_task->get_tablet_loc(), das_task->get_type());
if (OB_FAIL(task_map_.set_refactored(key, das_task))) {
LOG_WARN("insert into task map failed", KR(ret), K(key), KP(das_task));
}
}
if (OB_SUCC(ret)) {
task_cnt_++;
}
return ret;
}
void ObDASRef::print_all_das_task()
{
DASTaskIter task_iter(batched_tasks_.get_header_node()->get_next(), batched_tasks_.get_header_node());
......@@ -148,6 +233,11 @@ int ObDASRef::execute_all_task()
void ObDASRef::set_frozen_node()
{
frozen_op_node_ = batched_tasks_.get_last_node();
lookup_cnt_ = 0;
task_cnt_ = 0;
if (task_map_.created()) {
task_map_.clear();
}
}
int ObDASRef::close_all_task()
......@@ -187,6 +277,9 @@ int ObDASRef::close_all_task()
session->get_trans_result().set_incomplete();
}
batched_tasks_.destroy();
if (task_map_.created()) {
task_map_.destroy();
}
}
return ret;
}
......@@ -203,8 +296,6 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc,
LOG_WARN("get das task id failed", KR(ret));
} else if (OB_FAIL(das_factory.create_das_task_op(op_type, task_op))) {
LOG_WARN("create das task op failed", K(ret), KPC(task_op));
} else if (OB_FAIL(add_batched_task(task_op))) {
LOG_WARN("add batched task failed", K(ret), KPC(task_op));
} else {
task_op->set_trans_desc(session->get_tx_desc());
task_op->set_snapshot(&get_exec_ctx().get_das_ctx().get_snapshot());
......@@ -218,6 +309,9 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc,
LOG_WARN("init task info failed", K(ret));
}
}
if (OB_SUCC(ret) && OB_FAIL(add_batched_task(task_op))) {
LOG_WARN("add batched task failed", KR(ret), KPC(task_op));
}
return ret;
}
......@@ -225,6 +319,11 @@ void ObDASRef::reset()
{
das_factory_.cleanup();
batched_tasks_.destroy();
lookup_cnt_ = 0;
task_cnt_ = 0;
if (task_map_.created()) {
task_map_.destroy();
}
flags_ = false;
frozen_op_node_ = nullptr;
expr_frame_info_ = nullptr;
......@@ -238,6 +337,11 @@ void ObDASRef::reuse()
{
das_factory_.cleanup();
batched_tasks_.destroy();
lookup_cnt_ = 0;
task_cnt_ = 0;
if (task_map_.created()) {
task_map_.destroy();
}
frozen_op_node_ = nullptr;
if (reuse_alloc_ != nullptr) {
reuse_alloc_->reset_remain_one_page();
......
src/sql/das/ob_das_ref.h
浏览文件 @ 71d86596
......@@ -23,6 +23,32 @@ namespace sql
{
class ObDASScanOp;
class ObDASInsertOp;
struct DasRefKey
{
public:
DasRefKey()
: tablet_loc_(NULL),
op_type_(ObDASOpType::DAS_OP_INVALID)
{}
DasRefKey(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type)
: tablet_loc_(tablet_loc),
op_type_(op_type)
{}
~DasRefKey() {}
bool operator==(const DasRefKey &other) const;
uint64_t hash() const;
TO_STRING_KV(KP_(tablet_loc), K_(op_type));
public:
const ObDASTabletLoc *tablet_loc_;
ObDASOpType op_type_;
};
static const int64_t DAS_REF_TASK_LOOKUP_THRESHOLD = 1000;
static const int64_t DAS_REF_TASK_SIZE_THRESHOLD = 1000;
static const int64_t DAS_REF_MAP_BUCKET_SIZE = 5000;
typedef common::hash::ObHashMap<DasRefKey, ObIDASTaskOp *, common::hash::NoPthreadDefendMode> ObDASRefMap;
class ObDASRef
{
public:
......@@ -37,7 +63,7 @@ public:
template <typename DASOp>
bool has_das_op(const ObDASTabletLoc *tablet_loc, DASOp *&das_op);
ObIDASTaskOp* find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type);
int add_batched_task(ObIDASTaskOp *das_task) { return batched_tasks_.store_obj(das_task); }
int add_batched_task(ObIDASTaskOp *das_task);
//创建一个DAS Task，并由das_ref持有
template <typename DASOp>
int prepare_das_task(const ObDASTabletLoc *tablet_loc, DASOp *&task_op);
......@@ -67,6 +93,7 @@ public:
void set_lookup_iter(DASOpResultIter *lookup_iter) { wild_datum_info_.lookup_iter_ = lookup_iter; }
private:
DISABLE_COPY_ASSIGN(ObDASRef);
int create_task_map();
private:
typedef common::ObObjNode<ObIDASTaskOp*> DasOpNode;
//declare das allocator
......@@ -84,6 +111,9 @@ private:
DasOpNode *frozen_op_node_; // 初始为链表的head节点，冻结一次之后为链表的最后一个节点
const ObExprFrameInfo *expr_frame_info_;
DASOpResultIter::WildDatumPtrInfo wild_datum_info_;
int64_t lookup_cnt_;
int64_t task_cnt_;
ObDASRefMap task_map_;
public:
//all flags
union {
......
src/sql/das/ob_das_utils.cpp
浏览文件 @ 71d86596
......@@ -59,22 +59,6 @@ int ObDASUtils::store_warning_msg(const ObWarningBuffer &wb, obrpc::ObRpcResultC
return ret;
}
int ObDASUtils::get_tablet_loc_by_id(const ObTabletID &tablet_id,
ObDASTableLoc &table_loc,
ObDASTabletLoc *&tablet_loc)
{
int ret = OB_SUCCESS;
tablet_loc = nullptr;
FOREACH(tmp_node, table_loc.tablet_locs_) {
ObDASTabletLoc *tmp_tablet_loc = *tmp_node;
if (tmp_tablet_loc->tablet_id_ == tablet_id) {
tablet_loc = tmp_tablet_loc;
break;
}
}
return ret;
}
int ObDASUtils::check_nested_sql_mutating(ObTableID ref_table_id, ObExecContext &exec_ctx)
{
int ret = OB_SUCCESS;
......
src/sql/engine/dml/ob_table_lock_op.cpp
浏览文件 @ 71d86596
......@@ -325,7 +325,7 @@ int ObTableLockOp::calc_tablet_loc(const ObLockCtDef &lock_ctdef,
} else {
//direct lock to storage
tablet_loc = (MY_INPUT.get_tablet_loc() != nullptr ?
MY_INPUT.get_tablet_loc() : MY_INPUT.get_table_loc()->tablet_locs_.get_first());
MY_INPUT.get_tablet_loc() : MY_INPUT.get_table_loc()->get_first_tablet_loc());
}
return ret;
}
......
src/sql/engine/dml/ob_table_modify_op.cpp
浏览文件 @ 71d86596
......@@ -775,7 +775,7 @@ int ObTableModifyOp::calc_single_table_loc()
K(table_loc_id), K(ref_table_id), K(das_ctx.get_table_loc_list()));
} else {
get_input()->table_loc_ = table_loc;
get_input()->tablet_loc_ = table_loc->tablet_locs_.get_first();
get_input()->tablet_loc_ = table_loc->get_first_tablet_loc();
}
}
return ret;
......
src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp
浏览文件 @ 71d86596
......@@ -170,11 +170,11 @@ int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx,
if (NO_PARTITION_ID_FLAG == part_id_idx) {
// 如果row中没有partition id expr对应的cell，默认partition id为0
ObDASTableLoc *table_loc = del_rtdef_.das_rtdef_.table_loc_;
if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) {
if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc));
} else {
tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_;
tablet_id = table_loc->get_first_tablet_loc()->tablet_id_;
}
} else if (child_->get_spec().output_.count() > part_id_idx) {
ObExpr *expr = child_->get_spec().output_.at(part_id_idx);
......
src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp
浏览文件 @ 71d86596
......@@ -139,11 +139,11 @@ int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx,
row = &child_->get_spec().output_;
if (NO_PARTITION_ID_FLAG == part_id_idx) {
ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_;
if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) {
if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc));
} else {
tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_;
tablet_id = table_loc->get_first_tablet_loc()->tablet_id_;
}
} else if (child_->get_spec().output_.count() > part_id_idx) {
ObExpr *expr = child_->get_spec().output_.at(part_id_idx);
......
src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp
浏览文件 @ 71d86596
......@@ -141,11 +141,11 @@ int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx,
row = &child_->get_spec().output_;
if (NO_PARTITION_ID_FLAG == part_id_idx) {
ObDASTableLoc *table_loc = upd_rtdef_.dupd_rtdef_.table_loc_;
if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) {
if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc));
} else {
tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_;
tablet_id = table_loc->get_first_tablet_loc()->tablet_id_;
}
} else if (child_->get_spec().output_.count() > part_id_idx) {
ObExpr *expr = child_->get_spec().output_.at(part_id_idx);
......
src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp
浏览文件 @ 71d86596
......@@ -54,11 +54,11 @@ int ObPxMultiPartSSTableInsertOp::get_tablet_id_from_row(const ObExprPtrIArray &
tablet_id.reset();
if (NO_PARTITION_ID_FLAG == part_id_idx) {
ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_;
if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) {
if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc));
} else {
tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_;
tablet_id = table_loc->get_first_tablet_loc()->tablet_id_;
}
} else if (part_id_idx < 0) {
ret = OB_ERR_UNEXPECTED;
......
src/sql/engine/px/ob_px_sub_coord.cpp
浏览文件 @ 71d86596
......@@ -885,7 +885,8 @@ int ObPxSubCoord::rebuild_sqc_access_table_locations()
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected table loc", K(ret));
} else {
FOREACH(tmp_node, table_loc->tablet_locs_) {
for (DASTabletLocListIter tmp_node = table_loc->tablet_locs_begin();
tmp_node != table_loc->tablet_locs_end(); ++tmp_node) {
ObDASTabletLoc *tablet_loc = *tmp_node;
if (tablet_loc->tablet_id_ == location_keys.at(i).tablet_id_) {
if (OB_FAIL(access_locations.push_back(tablet_loc))) {
......
src/sql/engine/px/ob_px_util.cpp
浏览文件 @ 71d86596
......@@ -189,7 +189,7 @@ int ObPXServerAddrUtil::alloc_by_data_distribution_inner(
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get phy table location", K(ret));
} else {
const DASTabletLocList &locations = table_loc->tablet_locs_;
const DASTabletLocList &locations = table_loc->get_tablet_locs();
if (locations.size() <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the location array is empty", K(locations.size()), K(ret));
......@@ -511,7 +511,8 @@ int ObPXServerAddrUtil::alloc_by_random_distribution(ObExecContext &exec_ctx,
DASTabletLocArray locations;
FOREACH_X(tmp_node, table_locs, OB_SUCC(ret)) {
ObDASTableLoc *table_loc = *tmp_node;
FOREACH_X(tablet_node, table_loc->tablet_locs_, OB_SUCC(ret)) {
for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin();
OB_SUCC(ret) && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) {
OZ(locations.push_back(*tablet_node));
}
}
......@@ -792,7 +793,7 @@ int ObPXServerAddrUtil::set_sqcs_accessed_location(ObExecContext &ctx,
int ret = OB_SUCCESS;
common::ObArray<ObPxSqcMeta *> sqcs;
int n_locations = 0;
const DASTabletLocList &locations = table_loc->tablet_locs_;
const DASTabletLocList &locations = table_loc->get_tablet_locs();
DASTabletLocSEArray temp_locations;
if (OB_ISNULL(table_loc) || OB_ISNULL(phy_op)) {
ret = OB_ERR_UNEXPECTED;
......
src/sql/engine/table/ob_table_scan_op.cpp
浏览文件 @ 71d86596
......@@ -690,7 +690,8 @@ int ObTableScanOp::prepare_das_task()
}
} else if (OB_LIKELY(nullptr == MY_CTDEF.das_dppr_tbl_)) {
ObDASTableLoc *table_loc = tsc_rtdef_.scan_rtdef_.table_loc_;
FOREACH_X(node, table_loc->tablet_locs_, OB_SUCC(ret)) {
for (DASTabletLocListIter node = table_loc->tablet_locs_begin();
OB_SUCC(ret) && node != table_loc->tablet_locs_end(); ++node) {
ObDASTabletLoc *tablet_loc = *node;
if (OB_FAIL(create_one_das_task(tablet_loc))) {
LOG_WARN("create one das task failed", K(ret));
......@@ -807,7 +808,7 @@ int ObTableScanOp::init_table_scan_rtdef()
if (OB_FAIL(init_das_scan_rtdef(scan_ctdef, scan_rtdef, loc_meta))) {
LOG_WARN("init das scan rtdef failed", K(ret));
} else if (!MY_SPEC.use_dist_das_ && !MY_SPEC.gi_above_ && !scan_rtdef.table_loc_->empty()) {
MY_INPUT.tablet_loc_ = scan_rtdef.table_loc_->tablet_locs_.get_first();
MY_INPUT.tablet_loc_ = scan_rtdef.table_loc_->get_first_tablet_loc();
}
}
if (OB_SUCC(ret) && MY_CTDEF.lookup_ctdef_ != nullptr) {
......@@ -2062,9 +2063,8 @@ int ObTableScanOp::get_access_tablet_loc(ObGranuleTaskInfo &info)
iter_end_ = true;
ret = OB_SUCCESS;
}
} else if (OB_FAIL(ObDASUtils::get_tablet_loc_by_id(info.tablet_loc_->tablet_id_,
*tsc_rtdef_.scan_rtdef_.table_loc_,
MY_INPUT.tablet_loc_))) {
} else if (OB_FAIL(tsc_rtdef_.scan_rtdef_.table_loc_->get_tablet_loc_by_id(info.tablet_loc_->tablet_id_,
MY_INPUT.tablet_loc_))) {
//need use `get_tablet_loc_by_id` to find my px work thread's tablet_loc,
//because the tablet_loc in SQC maybe shared with other px work thread,
//the tablet loc maybe modify in das partition retry
......
src/sql/executor/ob_remote_identity_task_spliter.cpp
浏览文件 @ 71d86596
......@@ -62,7 +62,7 @@ int ObRemoteIdentityTaskSpliter::get_next_task(ObTaskInfo *&task)
// t1 union t1这种情况， t1(p0) union t2(p0)这种情况，等等，
// 都是remote模式，但table_loc_list的count可能大于1
// 优化器必须保证：remote模式下，所有表的location都是一致的，并且都是单分区。
ObDASTabletLoc *first_tablet_loc = first_table_loc->tablet_locs_.get_first();
ObDASTabletLoc *first_tablet_loc = first_table_loc->get_first_tablet_loc();
if (OB_ISNULL(ptr = allocator_->alloc(sizeof(ObTaskInfo)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to alloc ObTaskInfo", K(ret));
......
src/sql/executor/ob_remote_scheduler.cpp
浏览文件 @ 71d86596
......@@ -161,7 +161,7 @@ int ObRemoteScheduler::build_remote_task(ObExecContext &ctx,
remote_task.set_query_schema_version(task_exec_ctx.get_query_tenant_begin_schema_version(),
task_exec_ctx.get_query_sys_begin_schema_version());
remote_task.set_remote_sql_info(&plan_ctx->get_remote_sql_info());
ObDASTabletLoc *first_tablet_loc = DAS_CTX(ctx).get_table_loc_list().get_first()->tablet_locs_.get_first();
ObDASTabletLoc *first_tablet_loc = DAS_CTX(ctx).get_table_loc_list().get_first()->get_first_tablet_loc();
if (OB_ISNULL(session = ctx.get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is null", K(ret));
......
src/sql/executor/ob_task_executor_ctx.cpp
浏览文件 @ 71d86596
......@@ -326,7 +326,8 @@ int ObTaskExecutorCtxUtil::refresh_location_cache(ObTaskExecutorCtx &task_exec_c
DASTableLocList &table_locs = DAS_CTX(*task_exec_ctx.get_exec_context()).get_table_loc_list();
FOREACH_X(tmp_node, table_locs, OB_SUCC(ret)) {
ObDASTableLoc *table_loc = *tmp_node;
FOREACH_X(tablet_node, table_loc->tablet_locs_, OB_SUCC(ret)) {
for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin();
OB_SUCC(ret) && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) {
const ObDASTabletLoc *tablet_loc = *tablet_node;
if (is_nonblock) {
const int64_t expire_renew_time = 0; //表示在刷location cache之前不清空现有的location cache
......
src/sql/ob_sql.cpp
浏览文件 @ 71d86596
......@@ -4218,7 +4218,7 @@ int ObSql::check_need_reroute(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *plan, bool
} else {
const ObTableSchema *table_schema = NULL;
ObDASTableLoc *first_table_loc = DAS_CTX(pc_ctx.exec_ctx_).get_table_loc_list().get_first();
ObDASTabletLoc *first_tablet_loc = first_table_loc->tablet_locs_.get_first();
ObDASTabletLoc *first_tablet_loc = first_table_loc->get_first_tablet_loc();
ObLSReplicaLocation ls_replica_loc;
ObDASLocationRouter &loc_router = DAS_CTX(pc_ctx.exec_ctx_).get_location_router();
if (OB_FAIL(pc_ctx.sql_ctx_.schema_guard_->get_table_schema(
......
src/sql/optimizer/ob_table_location.cpp
浏览文件 @ 71d86596
......@@ -5166,7 +5166,7 @@ int ObTableLocation::get_full_leader_table_loc(ObIAllocator &allocator,
OX(tablet_loc = new(tablet_buf) ObDASTabletLoc());
OX(tablet_loc->loc_meta_ = loc_meta);
OZ(ObDASLocationRouter::get_leader(tenant_id, tablet_ids.at(i), *tablet_loc, expire_renew_time));
OZ(table_loc->tablet_locs_.push_back(tablet_loc));
OZ(table_loc->add_tablet_loc(tablet_loc));
}
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或