diff --git a/src/sql/optimizer/ob_log_group_by.cpp b/src/sql/optimizer/ob_log_group_by.cpp index b01872ab7f3628fc2a4536877bb27d3bd2110f5d..4619f440e6ca6c471fe13d4081413605691f92be 100644 --- a/src/sql/optimizer/ob_log_group_by.cpp +++ b/src/sql/optimizer/ob_log_group_by.cpp @@ -476,75 +476,53 @@ int ObLogGroupBy::inner_replace_op_exprs( int ObLogGroupBy::print_outline_data(PlanText &plan_text) { int ret = OB_SUCCESS; - char *buf = plan_text.buf_; - int64_t &buf_len = plan_text.buf_len_; - int64_t &pos = plan_text.pos_; const ObDMLStmt *stmt = NULL; ObString qb_name; - const ObLogicalOperator *child = NULL; - const ObLogicalOperator *op = NULL; - const ObLogGroupBy *group_by_op = NULL; if (is_push_down()) { /* print outline in top group by */ - } else if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt()) - || OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { + } else if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL", K(ret), K(get_plan()), K(stmt), K(child)); - } else if (OB_FAIL(child->get_pushdown_op(log_op_def::LOG_GROUP_BY, op))) { - LOG_WARN("failed to get push down group by", K(ret)); + LOG_WARN("unexpected NULL", K(ret), K(get_plan()), K(stmt)); } else if (OB_FAIL(stmt->get_qb_name(qb_name))) { LOG_WARN("fail to get qb_name", K(ret), K(stmt->get_stmt_id())); - } else if (NULL != op && - OB_FAIL(BUF_PRINTF("%s%s(@\"%.*s\")", - ObQueryHint::get_outline_indent(plan_text.is_oneline_), - ObHint::get_hint_name(T_GBY_PUSHDOWN), - qb_name.length(), qb_name.ptr()))) { - LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos)); - } else if (OB_FALSE_IT(group_by_op = static_cast(NULL == op ? this : op))) { - } else if (HASH_AGGREGATE == group_by_op->get_algo() && - OB_FAIL(BUF_PRINTF("%s%s(@\"%.*s\")", - ObQueryHint::get_outline_indent(plan_text.is_oneline_), - ObHint::get_hint_name(T_USE_HASH_AGGREGATE), - qb_name.length(), qb_name.ptr()))) { - LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos)); - } else {/*do nothing*/} + } else { + if (OB_SUCC(ret) && has_push_down_) { + ObOptHint hint(T_GBY_PUSHDOWN); + hint.set_qb_name(qb_name); + if (OB_FAIL(hint.print_hint(plan_text))) { + LOG_WARN("failed to print hint", K(ret), K(hint)); + } + } + if (OB_SUCC(ret) && (use_hash_aggr_ || use_part_sort_)) { + ObAggHint hint(use_hash_aggr_ ? T_USE_HASH_AGGREGATE : T_NO_USE_HASH_AGGREGATE); + hint.set_qb_name(qb_name); + hint.set_use_partition_sort(use_part_sort_); + if (OB_FAIL(hint.print_hint(plan_text))) { + LOG_WARN("failed to print hint", K(ret), K(hint)); + } + } + } return ret; } int ObLogGroupBy::print_used_hint(PlanText &plan_text) { int ret = OB_SUCCESS; - const ObLogicalOperator *child = NULL; - const ObLogicalOperator *op = NULL; + const ObHint *hint = NULL; if (is_push_down()) { /* print outline in top group by */ } else if (OB_ISNULL(get_plan())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", K(ret), K(get_plan())); - } else if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL", K(ret), K(child)); - } else if (OB_FAIL(child->get_pushdown_op(log_op_def::LOG_GROUP_BY, op))) { - LOG_WARN("failed to get push down group by", K(ret)); - } else { - const ObHint *use_hash = get_plan()->get_log_plan_hint().get_normal_hint(T_USE_HASH_AGGREGATE); - const ObHint *pushdown = get_plan()->get_log_plan_hint().get_normal_hint(T_GBY_PUSHDOWN); - if (NULL != use_hash) { - // for pushdown group by, need get pushdown algo - const ObLogGroupBy *group_by_op = static_cast(NULL == op ? this : op); - bool match_hint = (HASH_AGGREGATE == group_by_op->get_algo() && use_hash->is_enable_hint()) - || (MERGE_AGGREGATE == group_by_op->get_algo() && use_hash->is_disable_hint()); - if (match_hint && OB_FAIL(use_hash->print_hint(plan_text))) { - LOG_WARN("failed to print used hint for group by", K(ret), K(*use_hash)); - } - } - if (OB_SUCC(ret) && NULL != pushdown) { - bool match_hint = NULL == op ? pushdown->is_disable_hint() - : pushdown->is_enable_hint(); - if (match_hint && OB_FAIL(pushdown->print_hint(plan_text))) { - LOG_WARN("failed to print used hint for group by", K(ret), K(*pushdown)); - } - } + } else if (NULL != (hint = get_plan()->get_log_plan_hint().get_normal_hint(T_GBY_PUSHDOWN)) + && hint->is_enable_hint() == has_push_down_ + && OB_FAIL(hint->print_hint(plan_text))) { + LOG_WARN("failed to print used hint for group by", K(ret), K(*hint)); + } else if (NULL != (hint = get_plan()->get_log_plan_hint().get_normal_hint(T_USE_HASH_AGGREGATE)) + && hint->is_enable_hint() == use_hash_aggr_ + && static_cast(hint)->force_partition_sort() == use_part_sort_ + && OB_FAIL(hint->print_hint(plan_text))) { + LOG_WARN("failed to print used hint for group by", K(ret), K(*hint)); } return ret; } diff --git a/src/sql/optimizer/ob_log_group_by.h b/src/sql/optimizer/ob_log_group_by.h index 20d3ebced64dc64b2d0c594681c1ccd392bfd716..622c8282915f20f201f550b713d5600162337473 100644 --- a/src/sql/optimizer/ob_log_group_by.h +++ b/src/sql/optimizer/ob_log_group_by.h @@ -76,7 +76,10 @@ public: aggr_stage_(ObThreeStageAggrStage::NONE_STAGE), three_stage_info_(), rollup_adaptive_info_(), - force_push_down_(false) + force_push_down_(false), + use_hash_aggr_(false), + has_push_down_(false), + use_part_sort_(false) {} virtual ~ObLogGroupBy() {} @@ -197,6 +200,8 @@ public: { return ObRollupStatus::ROLLUP_COLLECTOR == rollup_adaptive_info_.rollup_status_; } inline void set_force_push_down(bool force_push_down) { force_push_down_ = force_push_down; } + void set_group_by_outline_info(bool use_hash_aggr, bool has_push_down, bool use_part_sort = false) + { use_hash_aggr_ = use_hash_aggr; has_push_down_ = has_push_down; use_part_sort_ = use_part_sort; } virtual int get_plan_item_info(PlanText &plan_text, ObSqlPlanItem &plan_item) override; @@ -230,6 +235,10 @@ private: // for rollup distributor and collector ObRollupAdaptiveInfo rollup_adaptive_info_; bool force_push_down_; // control by _aggregation_optimization_settings + // use print outline + bool use_hash_aggr_; + bool has_push_down_; + bool use_part_sort_; }; } // end of namespace sql } // end of namespace oceanbase diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 5e9aa9212072248ef0c8313b896e742f3d528154..c021424a4e39f421e8b787f298112c55f7522418 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -6321,6 +6321,8 @@ int ObLogPlan::create_three_stage_group_plan(const ObIArray &group_b } else if (OB_FAIL(third_group_by->set_rollup_info(third_rollup_status, helper.rollup_id_expr_))) { LOG_WARN("failed to set rollup parallel info", K(ret)); + } else { + third_group_by->set_group_by_outline_info(HASH_AGGREGATE == second_aggr_algo, true); } } return ret; @@ -6527,7 +6529,9 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it is_from_povit, origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); - } else { /*do nothing*/ } + } else { + static_cast(top)->set_group_by_outline_info(false, false); + } } else if (!groupby_helper.distinct_exprs_.empty() && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(groupby_helper.distinct_exprs_, is_partition_wise))) { @@ -6566,7 +6570,9 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it is_from_povit, origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); - } else { /*do nothing*/ } + } else { + static_cast(top)->set_group_by_outline_info(false, groupby_helper.can_basic_pushdown_ || is_partition_wise); //zzydebug + } } return ret; @@ -6645,8 +6651,6 @@ int ObLogPlan::init_groupby_helper(const ObIArray &group_exprs, ObSEArray group_rollup_exprs; bool push_group = false; groupby_helper.is_scalar_group_by_ = true; - groupby_helper.force_use_hash_ = get_log_plan_hint().use_hash_aggregate(); - groupby_helper.force_use_merge_ = get_log_plan_hint().use_merge_aggregate(); if (OB_FAIL(candidates_.get_best_plan(best_plan))) { LOG_WARN("failed to get best plan", K(ret)); } else if (OB_ISNULL(best_plan) || @@ -6659,6 +6663,11 @@ int ObLogPlan::init_groupby_helper(const ObIArray &group_exprs, } else if (OB_FAIL(append(group_rollup_exprs, group_exprs)) || OB_FAIL(append(group_rollup_exprs, rollup_exprs))) { LOG_WARN("failed to append group rollup exprs", K(ret)); + } else if (OB_FAIL(get_log_plan_hint().get_aggregation_info(groupby_helper.force_use_hash_, + groupby_helper.force_use_merge_, + groupby_helper.force_part_sort_, + groupby_helper.force_normal_sort_))) { + LOG_WARN("failed to get aggregation info from hint", K(ret)); } else if (OB_FAIL(check_scalar_groupby_pushdown(aggr_items, groupby_helper.can_scalar_pushdown_))) { LOG_WARN("failed to check scalar group by pushdown", K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 56943e74c6ea2ab6f1f661b0bd768fb2417d1f08..0e48054086c0f3ba54831f9fb13b1ec76dc6aa2f 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -487,6 +487,8 @@ public: can_rollup_pushdown_(false), force_use_hash_(false), force_use_merge_(false), + force_part_sort_(false), + force_normal_sort_(false), is_scalar_group_by_(false), distinct_exprs_(), aggr_code_expr_(NULL), @@ -506,6 +508,8 @@ public: bool can_rollup_pushdown_; bool force_use_hash_; // has use_hash_aggregation/use_hash_distinct hint bool force_use_merge_; // has no_use_hash_aggregation/no_use_hash_distinct hint + bool force_part_sort_; // force use partition sort for merge group by + bool force_normal_sort_; // disable use partition sort for merge group by bool is_scalar_group_by_; ObSEArray distinct_exprs_; diff --git a/src/sql/optimizer/ob_log_set.cpp b/src/sql/optimizer/ob_log_set.cpp index 7737bab08013c7a59971f00a6541ec65615bffc8..9f37853edae764d4ae851ed725c321d018ff83ae 100644 --- a/src/sql/optimizer/ob_log_set.cpp +++ b/src/sql/optimizer/ob_log_set.cpp @@ -809,7 +809,7 @@ int ObLogSet::print_outline_data(PlanText &plan_text) LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos)); } else if (OB_FAIL(construct_pq_set_hint(hint))) { LOG_WARN("fail to construct pq set hint", K(ret)); - } else if (hint.get_dist_methods().empty()) { + } else if (hint.get_dist_methods().empty() && hint.get_left_branch().empty()) { /*do nothing*/ } else if (OB_FALSE_IT(hint.set_qb_name(qb_name))) { } else if (hint.print_hint(plan_text)) { diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 8456060301cf88b1bb8d9280cd741bc19acc2f92..c24f4cb41dba70e6794709ffc8acc0a27e7ba81f 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -1760,6 +1760,10 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text) } else if (NULL != table_hint->parallel_hint_ && table_hint->parallel_hint_->get_parallel() > 1 && OB_FAIL(table_hint->parallel_hint_->print_hint(plan_text))) { LOG_WARN("failed to print table parallel hint", K(ret)); + } else if (NULL != table_hint->use_das_hint_ + && use_das() == table_hint->use_das_hint_->is_enable_hint() + && OB_FAIL(table_hint->use_das_hint_->print_hint(plan_text))) { + LOG_WARN("failed to print use das hint", K(ret)); } else if (table_hint->index_list_.empty()) { /*do nothing*/ } else if (OB_UNLIKELY(table_hint->index_list_.count() != table_hint->index_hints_.count())) { diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index 5918694ff76e76e64b1318f7b49ce8c681906052..1786965764469c341b9ed96828b0affca79cc9d5 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -332,19 +332,25 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArrayis_distributed() || is_partition_wise) { + bool part_sort_valid = !groupby_helper.force_normal_sort_; + bool normal_sort_valid = !groupby_helper.force_part_sort_; bool can_ignore_merge_plan = !(groupby_plans.empty() || groupby_helper.force_use_merge_); - if (OB_FAIL(create_merge_group_plan(reduce_exprs, - group_by_exprs, - group_directions, - rollup_exprs, - rollup_directions, - aggr_items, - having_exprs, - is_from_povit, - groupby_helper, - candidate_plan, - groupby_plans, - can_ignore_merge_plan))) { + if (OB_FAIL(update_part_sort_method(part_sort_valid, normal_sort_valid))) { + LOG_WARN("fail to update part sort method", K(ret)); + } else if (OB_FAIL(create_merge_group_plan(reduce_exprs, + group_by_exprs, + group_directions, + rollup_exprs, + rollup_directions, + aggr_items, + having_exprs, + is_from_povit, + groupby_helper, + candidate_plan, + groupby_plans, + part_sort_valid, + normal_sort_valid, + can_ignore_merge_plan))) { LOG_WARN("failed to create merge group by plan", K(ret)); } } else { @@ -370,16 +376,25 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray &group_by_ex const GroupingOpHelper &groupby_helper, const bool ignore_hint, bool &use_hash_valid, - bool &use_merge_valid) + bool &use_merge_valid, + bool &part_sort_valid, + bool &normal_sort_valid) { int ret = OB_SUCCESS; - use_hash_valid = ignore_hint ? true : !groupby_helper.force_use_merge_; - use_merge_valid = ignore_hint ? true : !groupby_helper.force_use_hash_; - ObSQLSessionInfo *session = get_optimizer_context().get_session_info(); - ObObj val; - if (OB_ISNULL(get_stmt()) || OB_ISNULL(session)) { + if (ignore_hint) { + use_hash_valid = true; + use_merge_valid = true; + part_sort_valid = true; + normal_sort_valid = true; + } else { + use_hash_valid = !groupby_helper.force_use_merge_; + use_merge_valid = !groupby_helper.force_use_hash_; + part_sort_valid = !groupby_helper.force_normal_sort_; + normal_sort_valid = !groupby_helper.force_part_sort_; + } + if (OB_ISNULL(get_stmt()) || OB_ISNULL(optimizer_context_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(get_stmt()), K(session), K(ret)); + LOG_WARN("get unexpected null", K(get_stmt()), K(optimizer_context_.get_query_ctx()), K(ret)); } else if (get_stmt()->has_rollup() || group_by_exprs.empty() || get_stmt()->has_distinct_or_concat_agg()) { @@ -387,6 +402,41 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray &group_by_ex //too much memory consumption for hash aggregate. use_hash_valid = false; } + + if (OB_FAIL(ret)) { + } else if (!use_merge_valid) { + part_sort_valid = false; + normal_sort_valid = false; + } else if (group_by_exprs.empty()) { + part_sort_valid = false; + /* if normal_sort_valid set as false by hint, will retry by ignore hint */ + } else if (OB_FAIL(update_part_sort_method(part_sort_valid, normal_sort_valid))) { + LOG_WARN("fail to update part sort method", K(ret)); + } + return ret; +} + +// update partition sort method with opt_param hint +int ObSelectLogPlan::update_part_sort_method(bool &part_sort_valid, + bool &normal_sort_valid) +{ + int ret = OB_SUCCESS; + if (!part_sort_valid || !normal_sort_valid) { + /* has sort method in no_use_hash_aggregation hint, ignore opt_param hint */ + } else { + bool use_part_sort = false; + bool is_exists_opt = false; + if (OB_FAIL(optimizer_context_.get_query_ctx()->get_global_hint() + .opt_params_.get_bool_opt_param( + ObOptParamHint::USE_PART_SORT_MGB, use_part_sort, is_exists_opt))) { + LOG_WARN("fail to check partition sort merge group by enabled", K(ret)); + } else if (!is_exists_opt) { + /* has no opt_param hint */ + } else { + part_sort_valid = use_part_sort; + normal_sort_valid = !use_part_sort; + } + } return ret; } @@ -406,8 +456,11 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & CandidatePlan candidate_plan; bool use_hash_valid = false; bool use_merge_valid = false; + bool part_sort_valid = false; + bool normal_sort_valid = false; if (OB_FAIL(get_valid_aggr_algo(group_by_exprs, groupby_helper, ignore_hint, - use_hash_valid, use_merge_valid))) { + use_hash_valid, use_merge_valid, + part_sort_valid, normal_sort_valid))) { LOG_WARN("failed to get valid aggr algo", K(ret)); } // create hash group by plans @@ -436,7 +489,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & } // create merge group by plans if (OB_SUCC(ret) && use_merge_valid) { - bool can_ignore_merge_plan = !(groupby_plans.empty() || (!ignore_hint && groupby_helper.force_use_merge_)); + bool can_ignore_merge_plan = !groupby_plans.empty(); for (int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) { candidate_plan = candidates_.candidate_plans_.at(i); bool is_needed = false; @@ -469,6 +522,8 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray & groupby_helper, candidate_plan, groupby_plans, + part_sort_valid, + normal_sort_valid, can_ignore_merge_plan))) { LOG_WARN("failed to create merge group by plan", K(ret)); } @@ -628,6 +683,8 @@ int ObSelectLogPlan::create_rollup_pushdown_plan(const ObIArray &gro } else if (rollup_collector->set_rollup_info(ObRollupStatus::ROLLUP_COLLECTOR, groupby_helper.rollup_id_expr_)) { LOG_WARN("failed to set rollup id expr", K(ret)); + } else { + rollup_collector->set_group_by_outline_info(false, true); } return ret; } @@ -665,6 +722,7 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e is_partition_wise))) { LOG_WARN("failed to allocate group by as top", K(ret)); } + static_cast(top)->set_group_by_outline_info(true, false); } else { // allocate push down group by if (groupby_helper.can_basic_pushdown_) { @@ -719,7 +777,9 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e groupby_helper.group_ndv_, origin_child_card))) { LOG_WARN("failed to allocate scala group by as top", K(ret)); - } else { /*do nothing*/ } + } else { + static_cast(top)->set_group_by_outline_info(true, groupby_helper.can_basic_pushdown_); + } } } return ret; @@ -865,65 +925,43 @@ int ObSelectLogPlan::create_merge_group_plan(const ObIArray &reduce_ GroupingOpHelper &groupby_helper, CandidatePlan &candidate_plan, ObIArray &candidate_plans, + bool part_sort_valid, + bool normal_sort_valid, bool can_ignore_merge_plan) { int ret = OB_SUCCESS; - bool use_part_sort = true; - bool is_exists_opt = false; - bool is_plan_valid = true; - bool gen_part_sort_mgb_plan = false; - const int64_t group_by_expr_cnt = group_by_exprs.count(); CandidatePlan part_sort_mgb_plan = candidate_plan; - if (OB_ISNULL(optimizer_context_.get_query_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(optimizer_context_.get_query_ctx()->get_global_hint() - .opt_params_.get_bool_opt_param( - ObOptParamHint::USE_PART_SORT_MGB, use_part_sort, is_exists_opt))) { - LOG_WARN("fail to check partition sort merge group by enabled", K(ret)); - } else if (group_by_expr_cnt > 0 && - (!is_exists_opt || use_part_sort) && - OB_FAIL(inner_create_merge_group_plan(reduce_exprs, - group_by_exprs, - group_directions, - rollup_exprs, - rollup_directions, - aggr_items, - having_exprs, - is_from_povit, - groupby_helper, - part_sort_mgb_plan.plan_tree_, - is_plan_valid, - can_ignore_merge_plan, - use_part_sort))) { - LOG_WARN("failed to create partition sort merge group by plan", K(ret)); - } else if (group_by_expr_cnt > 0 && - NULL != part_sort_mgb_plan.plan_tree_ && - is_plan_valid && - (!is_exists_opt || use_part_sort) && - FALSE_IT(gen_part_sort_mgb_plan = true)) { - } else if (gen_part_sort_mgb_plan && - OB_FAIL(candidate_plans.push_back(part_sort_mgb_plan))) { + if (part_sort_valid && OB_FAIL(inner_create_merge_group_plan(reduce_exprs, + group_by_exprs, + group_directions, + rollup_exprs, + rollup_directions, + aggr_items, + having_exprs, + is_from_povit, + groupby_helper, + part_sort_mgb_plan.plan_tree_, + true, + can_ignore_merge_plan))) { + LOG_WARN("failed to create partition sort merge group by plan", K(ret)); + } else if (part_sort_valid && NULL != part_sort_mgb_plan.plan_tree_ + && OB_FAIL(candidate_plans.push_back(part_sort_mgb_plan))) { LOG_WARN("failed to push merge group by", K(ret)); - } else if (FALSE_IT(can_ignore_merge_plan = can_ignore_merge_plan ? true : gen_part_sort_mgb_plan)) { - } else if (is_plan_valid && - (!is_exists_opt || !use_part_sort) && - OB_FAIL(inner_create_merge_group_plan(reduce_exprs, - group_by_exprs, - group_directions, - rollup_exprs, - rollup_directions, - aggr_items, - having_exprs, - is_from_povit, - groupby_helper, - candidate_plan.plan_tree_, - is_plan_valid, - can_ignore_merge_plan))) { + } else if (normal_sort_valid && OB_FAIL(inner_create_merge_group_plan(reduce_exprs, + group_by_exprs, + group_directions, + rollup_exprs, + rollup_directions, + aggr_items, + having_exprs, + is_from_povit, + groupby_helper, + candidate_plan.plan_tree_, + false, + can_ignore_merge_plan))) { LOG_WARN("failed to create merge group by plan", K(ret)); - } else if (is_plan_valid && - (!is_exists_opt || !use_part_sort) && - OB_FAIL(candidate_plans.push_back(candidate_plan))) { + } else if (normal_sort_valid && NULL != candidate_plan.plan_tree_ && + OB_FAIL(candidate_plans.push_back(candidate_plan))) { LOG_WARN("failed to push merge group by", K(ret)); } return ret; @@ -939,9 +977,8 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top, - bool &is_plan_valid, - bool can_ignore_merge_plan, - bool use_part_sort) + bool use_part_sort, + bool can_ignore_merge_plan) { int ret = OB_SUCCESS; int64_t prefix_pos = 0; @@ -950,8 +987,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r bool push_need_sort = false; bool is_partition_wise = false; bool is_fetch_with_ties = false; - is_plan_valid = true; - int64_t part_cnt = 0; + int64_t part_cnt = use_part_sort ? group_by_exprs.count() : 0; OrderItem hash_sortkey; ObSEArray dummy_exprs; ObSEArray dummy_aggrs; @@ -963,9 +999,9 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r ObSEArray adjusted_group_directions; int64_t interesting_order_info = OrderingFlag::NOT_MATCH; double origin_child_card = 0.0; - if (OB_ISNULL(top)) { + if (OB_ISNULL(top) || OB_UNLIKELY(0 == part_cnt && use_part_sort)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); + LOG_WARN("get unexpected params", K(top), K(part_cnt), K(use_part_sort), K(ret)); } else if (OB_FALSE_IT(origin_child_card = top->get_card())) { } else if (OB_FAIL(adjusted_group_by_exprs.assign(group_by_exprs)) || OB_FAIL(adjusted_group_directions.assign(group_directions))) { @@ -996,7 +1032,6 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r OrderingCheckScope::CHECK_ALL & (~OrderingCheckScope::CHECK_GROUP), interesting_order_info))) { LOG_WARN("failed to compute stmt interesting order", K(ret)); - } else if (FALSE_IT(part_cnt = use_part_sort ? group_by_exprs.count() : 0)) { } else if (OB_FAIL(ObOptimizerUtil::check_need_sort(sort_exprs, &sort_directions, top->get_op_ordering(), @@ -1009,9 +1044,12 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r prefix_pos, part_cnt))) { LOG_WARN("failed to check if need sort", K(ret)); - } else if (need_sort && can_ignore_merge_plan && OrderingFlag::NOT_MATCH == interesting_order_info) { - // if need sort and no further op needs the output order, not generate merge groupby - is_plan_valid = false; + } else if ((!need_sort && use_part_sort) || + (need_sort && can_ignore_merge_plan && + OrderingFlag::NOT_MATCH == interesting_order_info)) { + // 1. need generate partition sort plan, but need not sort + // 2. if need sort and no further op needs the output order, not generate merge groupby + top = NULL; } else if (top->is_distributed() && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { @@ -1030,8 +1068,10 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r origin_child_card, is_partition_wise))) { LOG_WARN("failed to allocate group by as top", K(ret)); + } else { + static_cast(top)->set_group_by_outline_info(false, false, use_part_sort); } - } else if (need_sort && part_cnt > 0 && + } else if (use_part_sort && OB_FAIL(create_hash_sortkey(part_cnt, sort_keys, hash_sortkey))) { LOG_WARN("failed to create hash sort key", K(ret), K(part_cnt), K(sort_keys)); } else { @@ -1063,7 +1103,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r !need_sort && top_is_local_order, nullptr, is_fetch_with_ties, - (need_sort && part_cnt > 0) ? &hash_sortkey : NULL))) { + use_part_sort ? &hash_sortkey : NULL))) { LOG_WARN("failed to allcoate sort as top", K(ret)); } else if (OB_FAIL(allocate_group_by_as_top(top, MERGE_AGGREGATE, @@ -1101,7 +1141,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r top->get_is_local_order(), nullptr, is_fetch_with_ties, - (need_sort && part_cnt > 0) ? &hash_sortkey : NULL))) { + use_part_sort ? &hash_sortkey : NULL))) { LOG_WARN("failed to allocate sort as top", K(ret)); } else if (OB_FAIL(allocate_group_by_as_top(top, MERGE_AGGREGATE, @@ -1113,7 +1153,10 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r groupby_helper.group_ndv_, origin_child_card))) { LOG_WARN("failed to allocate group by as top", K(ret)); - } else { /*do nothing*/ } + } else { + static_cast(top)->set_group_by_outline_info(false, + groupby_helper.can_basic_pushdown_, use_part_sort); + } } } return ret; diff --git a/src/sql/optimizer/ob_select_log_plan.h b/src/sql/optimizer/ob_select_log_plan.h index 9a0bd19a48f382570d8e2ec1eaee934734d140e7..dfa5c52f6d31fe3d25067996576d7c5724fcd8f9 100644 --- a/src/sql/optimizer/ob_select_log_plan.h +++ b/src/sql/optimizer/ob_select_log_plan.h @@ -85,7 +85,10 @@ private: const GroupingOpHelper &groupby_helper, const bool ignore_hint, bool &use_hash_valid, - bool &use_merge_valid); + bool &use_merge_valid, + bool &part_sort_valid, + bool &normal_sort_valid); + int update_part_sort_method(bool &part_sort_valid, bool &normal_sort_valid); int candi_allocate_normal_group_by(const ObIArray &reduce_exprs, const ObIArray &group_by_exprs, const ObIArray &group_directions, @@ -160,6 +163,8 @@ private: GroupingOpHelper &groupby_helper, CandidatePlan &candidate_plan, ObIArray &candidate_plans, + bool part_sort_valid, + bool normal_sort_valid, bool can_ignore_merge = false); int generate_merge_group_sort_keys(ObLogicalOperator *top, @@ -671,9 +676,8 @@ private: const bool is_from_povit, GroupingOpHelper &groupby_helper, ObLogicalOperator *&top, - bool &is_plan_valid, - bool can_ignore_merge = false, - bool use_part_sort = false); + bool use_part_sort, + bool can_ignore_merge = false); DISALLOW_COPY_AND_ASSIGN(ObSelectLogPlan); }; diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 0d6aec8182136e67f82850ab5e1e06290483292d..b275785d254ab5e27842c10dc9012b6035f7e1d4 100755 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -950,6 +950,8 @@ Timestamp{whitespace}?\"[^\"]*\" { USE_PLAN_CACHE { return USE_PLAN_CACHE; } USE_HASH_AGGREGATION { return USE_HASH_AGGREGATION; } NO_USE_HASH_AGGREGATION { return NO_USE_HASH_AGGREGATION; } +PARTITION_SORT { return PARTITION_SORT; } +NO_PARTITION_SORT { return NO_PARTITION_SORT; } USE_LATE_MATERIALIZATION { return USE_LATE_MATERIALIZATION; } NO_USE_LATE_MATERIALIZATION { return NO_USE_LATE_MATERIALIZATION; } TRACE_LOG { return TRACE_LOG; } diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 31dc5bb23a43533bae18b80c5d7fa6db61b29526..aac814c5fbb7a42a0b87b16eb0352071c2e510e4 100755 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -171,7 +171,8 @@ INDEX_SS_HINT INDEX_SS_ASC_HINT INDEX_SS_DESC_HINT LEADING_HINT ORDERED USE_NL USE_MERGE USE_HASH NO_USE_HASH NO_USE_MERGE NO_USE_NL USE_NL_MATERIALIZATION NO_USE_NL_MATERIALIZATION -USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION +USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION +PARTITION_SORT NO_PARTITION_SORT USE_LATE_MATERIALIZATION NO_USE_LATE_MATERIALIZATION PX_JOIN_FILTER NO_PX_JOIN_FILTER PX_PART_JOIN_FILTER NO_PX_PART_JOIN_FILTER PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET RANDOM_LOCAL BROADCAST BC2HOST LIST @@ -8850,6 +8851,17 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB ')' | NO_USE_HASH_AGGREGATION opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $2); + $$->value_ = -1; +} +| NO_USE_HASH_AGGREGATION '(' qb_name_option NO_PARTITION_SORT ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $3); + $$->value_ = 0; +} +| NO_USE_HASH_AGGREGATION '(' qb_name_option PARTITION_SORT ')' +{ + malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $3); + $$->value_ = 1; } | USE_LATE_MATERIALIZATION opt_qb_name { diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index dc2835badd72de10058682f20ae60496b2543096..127d6031792ef678ccd0a4543ed409c1db2b697b 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -11982,8 +11982,6 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } case T_USE_LATE_MATERIALIZATION: case T_NO_USE_LATE_MATERIALIZATION: - case T_USE_HASH_AGGREGATE: - case T_NO_USE_HASH_AGGREGATE: case T_GBY_PUSHDOWN: case T_NO_GBY_PUSHDOWN: case T_USE_HASH_DISTINCT: @@ -11999,6 +11997,13 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } break; } + case T_USE_HASH_AGGREGATE: + case T_NO_USE_HASH_AGGREGATE: { + if (OB_FAIL(resolve_aggregation_hint(hint_node, opt_hint))) { + LOG_WARN("failed to resolve aggregation hint.", K(ret)); + } + break; + } case T_TABLE_PARALLEL: { // PARALLEL(qb_name tablespec 4) if (OB_FAIL(resolve_table_parallel_hint(hint_node, opt_hint))) { LOG_WARN("fail to resolve parallel in hint", K(ret)); @@ -12751,6 +12756,34 @@ int ObDMLResolver::resolve_normal_optimize_hint(const ParseNode &hint_node, return ret; } +int ObDMLResolver::resolve_aggregation_hint(const ParseNode &hint_node, + ObOptHint *&hint) +{ + int ret = OB_SUCCESS; + hint = NULL; + ObAggHint *agg_hint = NULL; + ObString qb_name; + if (OB_UNLIKELY(1 != hint_node.num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("hint with qb name param has no one children.", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, agg_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) { + LOG_WARN("failed to resolve qb name node.", K(ret)); + } else { + hint = agg_hint; + agg_hint->set_qb_name(qb_name); + if (T_NO_USE_HASH_AGGREGATE == hint_node.type_) { + if (1 == hint_node.value_) { + agg_hint->set_use_partition_sort(true); + } else if (0 == hint_node.value_) { + agg_hint->set_use_partition_sort(false); + } + } + } + return ret; +} + int ObDMLResolver::resolve_monitor_ids(const ParseNode &tracing_node, ObIArray &monitoring_ids) { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index a8ad0e2978c7f433571b658e795783d4dc127555..473b1a7d0649e924fbd565d66121033cfe50115b 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -840,6 +840,7 @@ private: int resolve_pq_distribute_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_pq_set_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_filter_hint(const ParseNode &join_node, ObOptHint *&opt_hint); + int resolve_aggregation_hint(const ParseNode &hint_node, ObOptHint *&hint); int resolve_normal_transform_hint(const ParseNode &hint_node, ObTransHint *&hint); int resolve_normal_optimize_hint(const ParseNode &hint_node, ObOptHint *&hint); int resolve_view_merge_hint(const ParseNode &hint_node, ObTransHint *&hint); diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 6b42ffee1295829f29de83db3bb1311c704a3aab..41663e88c4a297689f40a3af13a5e96cabe4cfae 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -2511,5 +2511,29 @@ int ObWindowDistHint::print_hint_desc(PlanText &plan_text) const return ret; } +int ObAggHint::assign(const ObAggHint &other) +{ + int ret = OB_SUCCESS; + sort_method_valid_ = other.sort_method_valid_; + use_partition_sort_ = other.use_partition_sort_; + return ret; +} + +int ObAggHint::print_hint_desc(PlanText &plan_text) const +{ + int ret = OB_SUCCESS; + if (sort_method_valid_) { + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + if (use_partition_sort_ && OB_FAIL(BUF_PRINTF("PARTITION_SORT"))) { + LOG_WARN("print failed", K(ret)); + } else if (!use_partition_sort_ && OB_FAIL(BUF_PRINTF("NO_PARTITION_SORT"))) { + LOG_WARN("print failed", K(ret)); + } + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 1d70a3735525c159941b8ddec3a5038c21bbc8b2..4886dc227c06e7c1a85d895525e962afff89322a 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -931,6 +931,31 @@ private: Algos algos_; }; +class ObAggHint : public ObOptHint +{ +public: + ObAggHint(ObItemType hint_type) + : ObOptHint(hint_type), + sort_method_valid_(false), + use_partition_sort_(false) + { + } + int assign(const ObAggHint &other); + virtual ~ObAggHint() {} + + virtual int print_hint_desc(PlanText &plan_text) const override; + void set_use_partition_sort(bool use_part_sort) { sort_method_valid_ = is_disable_hint(); use_partition_sort_ = sort_method_valid_ && use_part_sort; } + void reset_use_partition_sort() { sort_method_valid_ = false; use_partition_sort_ = false; } + bool force_partition_sort() const { return is_disable_hint() && sort_method_valid_ && use_partition_sort_; } + bool force_normal_sort() const { return is_disable_hint() && sort_method_valid_ && !use_partition_sort_; } + + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(sort_method_valid), K_(use_partition_sort)); + +private: + bool sort_method_valid_; + bool use_partition_sort_; +}; + struct ObDDLSchemaVersionHint { ObDDLSchemaVersionHint() : schema_version_(0) {} diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 5d6fe51af82e24f4bc7f822e823c68be4a443660..74708ba8fd51b790bf5433f7d362594320d804de 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -1756,6 +1756,32 @@ bool ObLogPlanHint::has_disable_hint(ObItemType hint_type) const return NULL != cur_hint ? cur_hint->is_disable_hint() : is_outline_data_; } +int ObLogPlanHint::get_aggregation_info(bool &force_use_hash, + bool &force_use_merge, + bool &force_part_sort, + bool &force_normal_sort) const +{ + int ret = OB_SUCCESS; + force_use_hash = false; + force_use_merge = false; + force_part_sort = false; + force_normal_sort = false; + const ObAggHint *agg_hint = static_cast(get_normal_hint(T_USE_HASH_AGGREGATE)); + if (NULL != agg_hint) { + force_use_hash = agg_hint->is_enable_hint(); + force_use_merge = agg_hint->is_disable_hint(); + force_part_sort = agg_hint->force_partition_sort(); + force_normal_sort = agg_hint->force_normal_sort(); + if (force_use_merge && !force_part_sort && !force_normal_sort && is_outline_data_) { + force_normal_sort = true; + } + } else if (is_outline_data_) { + force_use_merge = true; + force_normal_sort = true; + } + return ret; +} + const ObWindowDistHint *ObLogPlanHint::get_window_dist() const { return static_cast( diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index f54b6c3a06f023cdf48e7ec71ca94935efc66f9a..e0c320e93c60d868277faba9299fa4b533e66c3d 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -435,6 +435,10 @@ struct ObLogPlanHint bool has_disable_hint(ObItemType hint_type) const; bool use_join_filter(const ObRelIds &table_set) const; bool no_use_join_filter(const ObRelIds &table_set) const; + int get_aggregation_info(bool &force_use_hash, + bool &force_use_merge, + bool &force_part_sort, + bool &force_normal_sort) const; bool use_late_material() const { return has_enable_hint(T_USE_LATE_MATERIALIZATION); } bool no_use_late_material() const { return has_disable_hint(T_USE_LATE_MATERIALIZATION); } diff --git a/src/sql/rewrite/ob_transform_groupby_pullup.cpp b/src/sql/rewrite/ob_transform_groupby_pullup.cpp index eae127fe06d7d68e3030f30fc366bca6f19acf22..fa917e6d628d0657c28a5e1e66c9c02593cf6a64 100644 --- a/src/sql/rewrite/ob_transform_groupby_pullup.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pullup.cpp @@ -305,7 +305,7 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt, } else if (OB_FAIL(check_hint_valid(*stmt, *table->ref_query_, hint_valid))) { LOG_WARN("check hint failed", K(ret)); } else if (!hint_valid) { - is_valid = false; + // can not set is_valid as false, may pullup other table OPT_TRACE("hint reject transform"); } else if (OB_FALSE_IT(myhint = static_cast(sub_stmt->get_stmt_hint().get_normal_hint(T_MERGE_HINT)))) { } else if (ignore_tables.has_member(stmt->get_table_bit_index(table->table_id_))) { @@ -1071,12 +1071,12 @@ int ObTransformGroupByPullup::need_transform(const common::ObIArrayis_valid_outline_transform(ctx_->trans_list_loc_, get_hint(table->ref_query_->get_stmt_hint())); - if (!need_trans) { - OPT_TRACE("outline reject transform"); - } LOG_DEBUG("need trans pullup0", K(need_trans)); } } + if (OB_SUCC(ret) && !need_trans) { + OPT_TRACE("outline reject transform"); + } } LOG_DEBUG("need trans pullup", K(need_trans)); return ret; diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.cpp b/src/sql/rewrite/ob_transform_predicate_move_around.cpp index 0b8e372346e7c9212fd4f20594d01b100d726263..cf457ee873e71f413e6b2902be37bb2059559c32 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.cpp +++ b/src/sql/rewrite/ob_transform_predicate_move_around.cpp @@ -47,44 +47,52 @@ int ObTransformPredicateMoveAround::transform_one_stmt( ObArray dummy_pullup; ObArray dummy_pushdown; ObArray dummy_list; - bool is_happened = false; transed_stmts_.reset(); + real_happened_ = false; UNUSED(parent_stmts); if (OB_ISNULL(stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("stmt is null", K(ret), K(stmt)); } else if (!stmt_map_.created() && OB_FAIL(stmt_map_.create(20, ObModIds::OB_SQL_COMPILE))) { LOG_WARN("failed to create stmt map", K(ret)); + } else if (OB_FAIL(pullup_predicates(stmt, dummy_list, dummy_pullup))) { + LOG_WARN("failed to pull up predicates", K(ret)); + } else if ((stmt->is_insert_stmt() || stmt->is_merge_stmt()) + && (OB_FAIL(create_equal_exprs_for_insert(static_cast(stmt))))) { + LOG_WARN("failed to create equal exprs for insert", K(ret)); + } else if (OB_FAIL(pushdown_predicates(stmt, dummy_pushdown))) { + LOG_WARN("failed to push down predicates", K(ret)); + } else if (transed_stmts_.empty() || !real_happened_) { + // transform not happened actually + transed_stmts_.reuse(); + } else if (OB_FAIL(sort_transed_stmts())) { + LOG_WARN("sort sort transed stmts failed", K(ret)); + } else if (OB_FAIL(add_transform_hint(*stmt, &transed_stmts_))) { + LOG_WARN("add transform hint failed", K(ret)); + } else { + trans_happened = true; } - if (OB_SUCC(ret)) { - if (OB_FAIL(pullup_predicates(stmt, dummy_list, dummy_pullup))) { - LOG_WARN("failed to pull up predicates", K(ret)); - } else if (!stmt->is_insert_stmt() && !stmt->is_merge_stmt()) { - // do nothing - } else if (OB_FAIL(create_equal_exprs_for_insert(static_cast(stmt), - is_happened))) { - LOG_WARN("failed to create equal exprs for insert", K(ret)); - } else if (is_happened) { - trans_happened = true; - } - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(pushdown_predicates(stmt, dummy_pushdown))) { - LOG_WARN("failed to push down predicates", K(ret)); - } else if (!trans_happened && !trans_happened_) { - // do nothing - } else if (OB_FAIL(add_transform_hint(*stmt, &transed_stmts_))) { - LOG_WARN("add transform hint failed", K(ret)); + return ret; +} + +int ObTransformPredicateMoveAround::sort_transed_stmts() +{ + int ret = OB_SUCCESS; + auto cmp_func = [](ObDMLStmt* l_stmt, ObDMLStmt* r_stmt) { + if (OB_ISNULL(l_stmt) || OB_ISNULL(r_stmt)) { + return false; } else { - trans_happened = true; + return l_stmt->get_stmt_id() < r_stmt->get_stmt_id(); } - } + }; + std::sort(transed_stmts_.begin(), transed_stmts_.end(), cmp_func); return ret; } ObTransformPredicateMoveAround::ObTransformPredicateMoveAround(ObTransformerCtx *ctx) : ObTransformRule(ctx, TransMethod::ROOT_ONLY, T_PRED_DEDUCE), - allocator_("PredDeduce") + allocator_("PredDeduce"), + real_happened_(false) {} ObTransformPredicateMoveAround::~ObTransformPredicateMoveAround() @@ -213,6 +221,17 @@ int ObTransformPredicateMoveAround::get_stmt_to_trans(ObDMLStmt *stmt, ObIArray< } else if (OB_FAIL(stmt->get_from_subquery_stmts(child_stmts))) { LOG_WARN("failed to get from subquery stmts", K(ret)); } else { + ObIArray &subquery_exprs = stmt->get_subquery_exprs(); + for (int64_t i = 0; OB_SUCC(ret) && i < subquery_exprs.count(); i++) { + if (OB_ISNULL(subquery_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(SMART_CALL(get_stmt_to_trans(subquery_exprs.at(i)->get_ref_stmt(), + stmt_to_trans)))) { + LOG_WARN("get stmt to trans from subquery failed", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); i++) { if (OB_FAIL(SMART_CALL(get_stmt_to_trans(child_stmts.at(i), stmt_to_trans)))) { LOG_WARN("get stmt to trans failed", K(ret)); @@ -252,6 +271,7 @@ int ObTransformPredicateMoveAround::pullup_predicates_from_set_stmt(ObDMLStmt *s return ret; } +//pullup predicates do not change stmt, need not hint check and allowed pullup pred from temp table int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt, ObIArray &sel_ids, ObIArray &output_pullup_preds) @@ -259,7 +279,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt, int ret = OB_SUCCESS; bool is_overflow = false; ObIArray *input_pullup_preds = NULL; - bool enable_no_pred_deduce = false; OPT_TRACE_BEGIN_SECTION; OPT_TRACE("try to pullup predicates"); if (OB_ISNULL(stmt)) { @@ -270,8 +289,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt, } else if (is_overflow) { ret = OB_SIZE_OVERFLOW; LOG_WARN("too deep recursive", K(ret), K(is_overflow)); - } else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) { - LOG_WARN("check_enable_no_pred_deduce failed", K(ret)); } else if (OB_FAIL(acquire_transform_params(stmt, input_pullup_preds))) { LOG_WARN("failed to acquire pullup preds", K(ret)); } else if (OB_ISNULL(input_pullup_preds)) { @@ -281,9 +298,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt, if (OB_FAIL(pullup_predicates_from_set_stmt(stmt, sel_ids, output_pullup_preds))) { LOG_WARN("process set stmt failed", K(ret)); } - } else if (!enable_no_pred_deduce && - OB_FAIL(preprocess(*stmt))) { - LOG_WARN("failed to preprocess stmt", K(ret)); } else if (OB_FAIL(generate_basic_table_pullup_preds(stmt, *input_pullup_preds))) { LOG_WARN("add stmt check constraints", K(ret)); } else if (OB_FAIL(pullup_predicates_from_view(*stmt, sel_ids, *input_pullup_preds))) { @@ -303,107 +317,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt, return ret; } -int ObTransformPredicateMoveAround::preprocess(ObDMLStmt &stmt) -{ - int ret = OB_SUCCESS; - OPT_TRACE("try pullup semi condition"); - for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_semi_infos().count(); ++i) { - if (OB_FAIL(preprocess_semi_info(stmt, stmt.get_semi_infos().at(i), - stmt.get_condition_exprs()))) { - LOG_WARN("failed to preprocess joined table", K(ret)); - } - } - OPT_TRACE("try pullup on condition"); - for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_joined_tables().count(); ++i) { - if (OB_FAIL(preprocess_joined_table(stmt, stmt.get_joined_tables().at(i), - stmt.get_condition_exprs()))) { - LOG_WARN("failed to preprocess joined table", K(ret)); - } - } - return ret; -} - -int ObTransformPredicateMoveAround::preprocess_joined_table(ObDMLStmt &stmt, - JoinedTable *join_table, - ObIArray &upper_conds) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(join_table) || - OB_ISNULL(join_table->left_table_) || - OB_ISNULL(join_table->right_table_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("joined table is null", K(ret)); - } else if (join_table->is_inner_join() && !join_table->join_conditions_.empty()) { - if (OB_FAIL(append(upper_conds, join_table->join_conditions_))) { - LOG_WARN("failed to append conditions", K(ret)); - } else { - join_table->join_conditions_.reset(); - trans_happened_ = true; - if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, &stmt))) { - LOG_WARN("append failed", K(ret)); - } - } - } - if (OB_SUCC(ret) && join_table->left_table_->is_joined_table() && - (join_table->is_inner_join() ||join_table->is_left_join())) { - if (OB_FAIL(SMART_CALL(preprocess_joined_table(stmt, - static_cast(join_table->left_table_), upper_conds)))) { - LOG_WARN("failed to process joined table", K(ret)); - } - } - if (OB_SUCC(ret) && join_table->right_table_->is_joined_table() && - (join_table->is_inner_join() || join_table->is_right_join())) { - if (OB_FAIL(SMART_CALL(preprocess_joined_table(stmt, - static_cast(join_table->right_table_), upper_conds)))) { - LOG_WARN("failed to process joined table", K(ret)); - } - } - return ret; -} - -int ObTransformPredicateMoveAround::preprocess_semi_info(ObDMLStmt &stmt, - SemiInfo *semi_info, - ObIArray &upper_conds) -{ - int ret = OB_SUCCESS; - ObSqlBitSet<> left_rel_ids; - if (OB_ISNULL(semi_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("semi info is null", K(ret)); - } else if (semi_info->is_anti_join()) { - // do not pull up anti conditions to upper conds - } else if (OB_FAIL(stmt.get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) { - LOG_WARN("failed to get table rel ids", K(ret)); - } else { - ObRawExpr *expr = NULL; - ObIArray &semi_conds = semi_info->semi_conditions_; - ObSEArray left_filters; - for (int64_t i = 0; OB_SUCC(ret) && i < semi_conds.count(); ++i) { - if (OB_ISNULL(expr = semi_conds.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (!expr->has_flag(CNT_COLUMN) || - !expr->get_relation_ids().is_subset(left_rel_ids)) { - /*do nothing*/ - } else if (OB_FAIL(left_filters.push_back(expr))) { - LOG_WARN("failed to push back expr", K(ret)); - } - } - if (OB_FAIL(ret) || left_filters.empty()) { - /*do nothing*/ - } else if (OB_FAIL(append(upper_conds, left_filters))) { - LOG_WARN("failed to append conditions", K(ret)); - } else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_conds, left_filters))) { - LOG_WARN("failed to remove item", K(ret)); - } else { - trans_happened_ = true; - if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, &stmt))) { - LOG_WARN("append failed", K(ret)); - } - } - } - return ret; -} int ObTransformPredicateMoveAround::pullup_predicates_from_view( ObDMLStmt &stmt, ObIArray &sel_ids, ObIArray &input_pullup_preds) @@ -907,7 +820,9 @@ int ObTransformPredicateMoveAround::generate_pullup_predicates( int ret = OB_SUCCESS; ObSEArray local_preds; ObSEArray filter_preds; - if (OB_FAIL(append(filter_preds, select_stmt.get_condition_exprs()))) { + if (OB_FAIL(gather_pullup_preds_from_semi_outer_join(select_stmt, filter_preds))) { + LOG_WARN("failed to pullup preds from semi outer join", K(ret)); + } else if (OB_FAIL(append(filter_preds, select_stmt.get_condition_exprs()))) { LOG_WARN("failed to append conditions", K(ret)); } else if (OB_FAIL(append(filter_preds, select_stmt.get_having_exprs()))) { LOG_WARN("failed to append having conditions", K(ret)); @@ -924,6 +839,77 @@ int ObTransformPredicateMoveAround::generate_pullup_predicates( return ret; } +int ObTransformPredicateMoveAround::gather_pullup_preds_from_semi_outer_join(ObDMLStmt &stmt, + ObIArray &preds, + bool remove_preds /* default false*/) +{ + int ret = OB_SUCCESS; + ObSqlBitSet<> left_rel_ids; + SemiInfo *semi_info = NULL; + ObRawExpr *expr = NULL; + ObSEArray new_semi_conditions; + for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_semi_infos().count(); ++i) { + left_rel_ids.reuse(); + new_semi_conditions.reuse(); + if (OB_ISNULL(semi_info = stmt.get_semi_infos().at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("semi info is null", K(ret)); + } else if (semi_info->is_anti_join()) { + // do not pull up anti conditions to upper conds + } else if (OB_FAIL(stmt.get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) { + LOG_WARN("failed to get table rel ids", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < semi_info->semi_conditions_.count(); ++j) { + if (OB_ISNULL(expr = semi_info->semi_conditions_.at(j))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!expr->has_flag(CNT_COLUMN) || !expr->get_relation_ids().is_subset(left_rel_ids)) { + if (remove_preds && OB_FAIL(new_semi_conditions.push_back(expr))) { + LOG_WARN("failed to push back expr", K(ret)); + } + } else if (OB_FAIL(preds.push_back(expr))) { + LOG_WARN("failed to push back expr", K(ret)); + } + } + if (OB_SUCC(ret) && remove_preds && semi_info->semi_conditions_.count() != new_semi_conditions.count() + && OB_FAIL(semi_info->semi_conditions_.assign(new_semi_conditions))) { + LOG_WARN("failed to assign semi conditions", K(ret)); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_joined_tables().count(); ++i) { + if (OB_FAIL(gather_pullup_preds_from_join_table(stmt.get_joined_tables().at(i), preds, remove_preds))) { + LOG_WARN("failed to pullup preds from joined table", K(ret)); + } + } + return ret; +} + +int ObTransformPredicateMoveAround::gather_pullup_preds_from_join_table(TableItem *table, + ObIArray &preds, + bool remove_preds) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (table->is_joined_table()) { + JoinedTable *join_table = static_cast(table); + if (join_table->is_inner_join() && OB_FAIL(append(preds, join_table->join_conditions_))) { + LOG_WARN("failed to append conditions", K(ret)); + } else if ((join_table->is_inner_join() || join_table->is_left_join()) + && OB_FAIL(SMART_CALL(gather_pullup_preds_from_join_table(join_table->left_table_, preds, remove_preds)))) { + LOG_WARN("failed to pullup preds from joined table", K(ret)); + } else if ((join_table->is_inner_join() || join_table->is_right_join()) + && OB_FAIL(SMART_CALL(gather_pullup_preds_from_join_table(join_table->right_table_, preds, remove_preds)))) { + LOG_WARN("failed to pullup preds from joined table", K(ret)); + } else if (join_table->is_inner_join() && remove_preds) { + join_table->join_conditions_.reset(); + } + } + return ret; +} + // be careful about anti join right table, // do not pullup a filter which does not contain any column of the right table. int ObTransformPredicateMoveAround::compute_pullup_predicates( @@ -1086,7 +1072,6 @@ int ObTransformPredicateMoveAround::pushdown_predicates( bool has_group = false; bool has_winfunc = false; ObSEArray candi_preds; - ObSEArray old_where_preds; ObIArray *pullup_preds = NULL; ObSelectStmt *sel_stmt = static_cast(stmt); bool enable_no_pred_deduce = false; @@ -1099,8 +1084,6 @@ int ObTransformPredicateMoveAround::pushdown_predicates( LOG_WARN("failed to acquire pull up preds", K(ret)); } else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) { LOG_WARN("check_enable_no_pred_deduce failed", K(ret)); - } else if (OB_FAIL(old_where_preds.assign(stmt->get_condition_exprs()))) { - LOG_WARN("failed to assign conditions", K(ret)); } else if (OB_ISNULL(pullup_preds)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("pullup predicate array is null", K(ret)); @@ -1114,7 +1097,12 @@ int ObTransformPredicateMoveAround::pushdown_predicates( pushdown_preds))) { LOG_WARN("recursive pushdown preds into set stmt failed", K(ret)); } else {/*do nothing*/} - } else if (stmt->is_hierarchical_query()) { + } else if (stmt->is_hierarchical_query() || enable_no_pred_deduce) { + // do not transform for current level stmt, but need call function to transform child query + if (enable_no_pred_deduce) { + LOG_TRACE("NO PRED DEDUCE"); + OPT_TRACE("hint disable transform"); + } ObArray dummy_preds; for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) { TableItem *table = stmt->get_table_item(i); @@ -1127,10 +1115,9 @@ int ObTransformPredicateMoveAround::pushdown_predicates( LOG_WARN("failed to push down predicates", K(ret)); } else {/*do nothing*/} } - } else if (enable_no_pred_deduce) { - LOG_TRACE("NO PRED DEDUCE"); - OPT_TRACE("hint disable transform"); } else { + const uint64_t pushdown_pred_count = pushdown_preds.count(); + bool is_happened = false; if (OB_FAIL(stmt->has_rownum(has_rownum))) { LOG_WARN("failed to check stmt has rownum", K(ret)); } else if (stmt->is_select_stmt()) { @@ -1155,8 +1142,10 @@ int ObTransformPredicateMoveAround::pushdown_predicates( } else { pushdown_preds.reset(); } + if (OB_SUCC(ret) && pushdown_preds.count() != pushdown_pred_count) { + is_happened = true; + } } - if (OB_SUCC(ret) && has_group) { ObSEArray old_having_exprs; if (OB_FAIL(old_having_exprs.assign(sel_stmt->get_having_exprs()))) { @@ -1169,39 +1158,41 @@ int ObTransformPredicateMoveAround::pushdown_predicates( } else if (OB_FAIL(pushdown_through_groupby(*sel_stmt, candi_preds))) { LOG_WARN("failed to pushdown predicate", K(ret)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(check_transform_happened(stmt, old_having_exprs, sel_stmt->get_having_exprs()))) { + if (OB_SUCC(ret) && OB_FAIL(check_conds_deduced(old_having_exprs, sel_stmt->get_having_exprs(), is_happened))) { LOG_WARN("failed to check transform happened", K(ret)); } } if (OB_SUCC(ret)) { - if (OB_FAIL(pushdown_into_where(*stmt, *pullup_preds, candi_preds))) { - LOG_WARN("failed to push down predicates into where", K(ret)); - } else if (OB_FAIL(check_transform_happened(stmt, old_where_preds, stmt->get_condition_exprs()))) { - LOG_WARN("failed to check transform happened", K(ret)); - } - } - } - if (OB_SUCC(ret) && !stmt->is_set_stmt() && !stmt->is_hierarchical_query()) { - ObIArray &from_items = stmt->get_from_items(); - ObIArray &semi_infos = stmt->get_semi_infos(); - ObSEArray empty; - for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { - if (OB_FAIL(pushdown_into_table(stmt, stmt->get_table_item(from_items.at(i)), - *pullup_preds, enable_no_pred_deduce ? empty : stmt->get_condition_exprs()))) { - LOG_WARN("failed to push down predicates", K(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) { - if (OB_FAIL(pushdown_into_semi_info(stmt, semi_infos.at(i), *pullup_preds, - enable_no_pred_deduce ? empty : stmt->get_condition_exprs()))) { - LOG_WARN("failed to push down into semi info", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(check_transform_happened(stmt, old_where_preds, stmt->get_condition_exprs()))) { - LOG_WARN("failed to check transform happened", K(ret)); + typedef ObSEArray, 4> PredsArray; + SMART_VAR(PredsArray, all_old_preds) { + ObIArray &from_items = stmt->get_from_items(); + ObIArray &semi_infos = stmt->get_semi_infos(); + if ((!is_happened || !real_happened_) && OB_FAIL(store_all_preds(*stmt, all_old_preds))) { + LOG_WARN("failed to store all preds", K(ret)); + } else if (OB_FAIL(gather_pullup_preds_from_semi_outer_join(*stmt, stmt->get_condition_exprs(), true))) { + LOG_WARN("failed to pullup preds from semi outer join", K(ret)); + } else if (OB_FAIL(pushdown_into_where(*stmt, *pullup_preds, candi_preds))) { + LOG_WARN("failed to push down predicates into where", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { + if (OB_FAIL(pushdown_into_table(stmt, stmt->get_table_item(from_items.at(i)), + *pullup_preds, stmt->get_condition_exprs()))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) { + if (OB_FAIL(pushdown_into_semi_info(stmt, semi_infos.at(i), *pullup_preds, + stmt->get_condition_exprs()))) { + LOG_WARN("failed to push down into semi info", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(check_transform_happened(*stmt, all_old_preds, is_happened))) { + LOG_WARN("failed to check transform happened", K(ret)); + } else if (is_happened && OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) { + LOG_WARN("append transed stmt failed", K(ret)); + } } } } @@ -1222,6 +1213,163 @@ int ObTransformPredicateMoveAround::pushdown_predicates( return ret; } +int ObTransformPredicateMoveAround::store_all_preds(const ObDMLStmt &stmt, + ObIArray> &all_preds) +{ + int ret = OB_SUCCESS; + const ObIArray &join_tables = stmt.get_joined_tables(); + const ObIArray &semi_infos = stmt.get_semi_infos(); + ObIArray *preds = NULL; + all_preds.reuse(); + if (OB_ISNULL(preds = all_preds.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret)); + } else if (OB_FAIL(preds->assign(stmt.get_condition_exprs()))) { + LOG_WARN("failed to assign conditions", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < join_tables.count(); ++i) { + if (OB_FAIL(store_join_conds(join_tables.at(i), all_preds))) { + LOG_WARN("failed to store join conds", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) { + if (OB_ISNULL(semi_infos.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_ISNULL(preds = all_preds.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret)); + } else if (OB_FAIL(preds->assign(semi_infos.at(i)->semi_conditions_))) { + LOG_WARN("failed to assign conditions", K(ret)); + } + } + return ret; +} + +int ObTransformPredicateMoveAround::store_join_conds(const TableItem *table, + ObIArray> &all_preds) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (table->is_joined_table()) { + const JoinedTable *join_table = static_cast(table); + ObIArray *preds = NULL; + if (OB_ISNULL(preds = all_preds.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret)); + } else if (OB_FAIL(preds->assign(join_table->join_conditions_))) { + LOG_WARN("failed to assign conditions", K(ret)); + } else if (OB_FAIL(SMART_CALL(store_join_conds(join_table->left_table_, all_preds)))) { + LOG_WARN("failed to pullup preds from joined table", K(ret)); + } else if (OB_FAIL(SMART_CALL(store_join_conds(join_table->right_table_, all_preds)))) { + LOG_WARN("failed to pullup preds from joined table", K(ret)); + } + } + return ret; +} + +int ObTransformPredicateMoveAround::check_transform_happened(const ObDMLStmt &stmt, + const ObIArray> &all_preds, + bool &is_happened) +{ + int ret = OB_SUCCESS; + const ObIArray &join_tables = stmt.get_joined_tables(); + const ObIArray &semi_infos = stmt.get_semi_infos(); + bool real_happened = is_happened && real_happened_; + if (real_happened) { + // do nothing + } else if (OB_UNLIKELY(all_preds.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty", K(ret), K(all_preds.count())); + } else if (OB_FAIL(check_conds_deduced(all_preds.at(0), stmt.get_condition_exprs(), real_happened))) { + LOG_WARN("failed to push back conditions", K(ret)); + } else { + uint64_t idx = 1; + for (int64_t i = 0; !real_happened && OB_SUCC(ret) && i < join_tables.count(); ++i) { + if (OB_FAIL(check_join_conds_deduced(join_tables.at(i), all_preds, idx, real_happened))) { + LOG_WARN("failed to check join conds deduced", K(ret)); + } + } + if (OB_SUCC(ret) && OB_UNLIKELY(semi_infos.count() + idx > all_preds.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected count", K(ret), K(semi_infos.count()), K(idx), K(all_preds.count())); + } + for (int64_t i = 0; !real_happened && OB_SUCC(ret) && i < semi_infos.count(); ++i) { + if (OB_ISNULL(semi_infos.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(check_conds_deduced(all_preds.at(idx), + semi_infos.at(i)->semi_conditions_, + real_happened))) { + LOG_WARN("failed to check conds deduced", K(ret)); + } else { + ++idx; + } + } + if (OB_SUCC(ret)) { + is_happened |= real_happened; + } + } + return ret; +} + +int ObTransformPredicateMoveAround::check_join_conds_deduced(const TableItem *table, + const ObIArray> &all_preds, + uint64_t &idx, + bool &is_happened) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!is_happened && table->is_joined_table()) { + const JoinedTable *join_table = static_cast(table); + if (OB_UNLIKELY(all_preds.count() <= idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(all_preds.count())); + } else if (OB_FAIL(check_conds_deduced(all_preds.at(idx), join_table->join_conditions_, is_happened))) { + LOG_WARN("failed to check conds deduced", K(ret)); + } else { + ++idx; + } + if (OB_FAIL(ret) || is_happened) { + } else if (OB_FAIL(SMART_CALL(check_join_conds_deduced(join_table->left_table_, all_preds, idx, is_happened)))) { + LOG_WARN("failed to check join conds deduced", K(ret)); + } else if (OB_FAIL(SMART_CALL(check_join_conds_deduced(join_table->right_table_, all_preds, idx, is_happened)))) { + LOG_WARN("failed to check join conds deduced", K(ret)); + } + } + return ret; +} + +int ObTransformPredicateMoveAround::check_conds_deduced(const ObIArray &old_conditions, + const ObIArray &new_conditions, + bool &is_happened) +{ + int ret = OB_SUCCESS; + if (!is_happened || !real_happened_) { + bool happened = false; + for (int64_t i = 0; OB_SUCC(ret) && !happened && i < new_conditions.count(); ++i) { + if (!ObPredicateDeduce::find_equal_expr(old_conditions, new_conditions.at(i))) { + happened = true; + } + } + for (int64_t i = 0; OB_SUCC(ret) && !happened && i < old_conditions.count(); ++i) { + if (!ObPredicateDeduce::find_equal_expr(new_conditions, old_conditions.at(i))) { + happened = true; + } + } + if (OB_SUCC(ret) && happened) { + is_happened = true; + real_happened_ = true; + } + LOG_DEBUG("check transform happened", K(old_conditions), K(new_conditions), K(is_happened)); + } + return ret; +} + int ObTransformPredicateMoveAround::pushdown_into_set_stmt(ObSelectStmt *stmt, ObIArray &pullup_preds, ObIArray &pushdown_preds) @@ -1281,7 +1429,6 @@ int ObTransformPredicateMoveAround::check_pushdown_predicates(ObSelectStmt *stmt ObIArray &output_pushdown_preds) { int ret = OB_SUCCESS; - bool enable_no_pred_deduce = false; if (OB_ISNULL(stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("stmt is null", K(ret)); @@ -2088,9 +2235,7 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table( ObIArray &pushdown_preds) { int ret = OB_SUCCESS; - bool enable_no_pred_deduce = false; ObSEArray all_preds; - ObSEArray old_join_condition; /// STEP 1. deduce new join conditions bool is_stack_overflow = false; if (OB_FAIL(check_stack_overflow(is_stack_overflow))) { @@ -2102,8 +2247,6 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table( OB_ISNULL(joined_table->right_table_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("params are invalid", K(ret), K(stmt), K(joined_table)); - } else if (OB_FAIL(old_join_condition.assign(joined_table->join_conditions_))) { - LOG_WARN("failed to assign join condition", K(ret)); } else if (joined_table->is_left_join() || joined_table->is_right_join() || joined_table->is_inner_join()) { @@ -2146,10 +2289,6 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table( LOG_WARN("failed to get related columns", K(ret)); } else if (OB_FAIL(transform_predicates(*stmt, all_preds, cols, new_preds))) { LOG_WARN("failed to deduce predicates", K(ret)); - } else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) { - LOG_WARN("check_enable_no_pred_deduce failed", K(ret)); - } else if (enable_no_pred_deduce) { - OPT_TRACE("hint disable transform"); } else if (joined_table->is_inner_join()) { if (OB_FAIL(accept_predicates(*stmt, joined_table->join_conditions_, @@ -2270,58 +2409,10 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table( //do nothing for full join } } - if (OB_FAIL(ret)) { - } else if (check_transform_happened(stmt, old_join_condition, joined_table->join_conditions_)) { - LOG_WARN("failed to check transform happened", K(ret)); - } } return ret; } -/** - * 只有当condition出现了新的谓词才说明发生了改写 - * 如果condition的谓词减少了,定义不发生改写,整个查询是否 - * 发生改写交由后续的推导判断,如果减少的谓词成功下推到下层视图了,才说明发生了改写 - * 否则视为没改写 - */ -int ObTransformPredicateMoveAround::check_transform_happened(ObDMLStmt *stmt, - ObIArray &old_conditions, - ObIArray &new_conditions) -{ - int ret = OB_SUCCESS; - const ObQueryHint *query_hint = stmt->get_stmt_hint().query_hint_; - const ObTransHint *hint = static_cast(get_hint(stmt->get_stmt_hint())); - bool is_happened = false; - if (OB_ISNULL(query_hint)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(ctx_), K(query_hint)); - } else if (query_hint->has_outline_data() && hint == NULL) { - if (OB_FAIL(new_conditions.assign(old_conditions))) { - LOG_WARN("assign failed", K(ret)); - } - } else { - for (int64_t i = 0; OB_SUCC(ret) && !is_happened && i < new_conditions.count(); ++i) { - if (!ObPredicateDeduce::find_equal_expr(old_conditions, new_conditions.at(i))) { - is_happened = true; - } - } - for (int64_t i = 0; OB_SUCC(ret) && !is_happened && i < old_conditions.count(); ++i) { - if (!ObPredicateDeduce::find_equal_expr(new_conditions, old_conditions.at(i))) { - is_happened = true; - } - } - - if (OB_SUCC(ret) && is_happened) { - trans_happened_ = true; - if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) { - LOG_WARN("append failed", K(ret)); - } - } - } - LOG_DEBUG("check_transform_happened", K(old_conditions), K(new_conditions), K(is_happened)); - return ret; -} - int ObTransformPredicateMoveAround::get_pushdown_predicates( ObDMLStmt &stmt, TableItem &table, ObIArray &preds, @@ -2391,7 +2482,6 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt, { int ret = OB_SUCCESS; ObSEArray all_preds; - ObSEArray old_semi_conds; ObSqlBitSet<> left_rel_ids; ObSqlBitSet<> right_rel_ids; TableItem *right_table = NULL; @@ -2400,14 +2490,11 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt, ObSEArray all_cols; ObSEArray new_preds; ObSEArray empty; - bool enable_no_pred_deduce = false; OPT_TRACE("try to transform semi conditions"); if (OB_ISNULL(stmt) || OB_ISNULL(semi_info) || OB_ISNULL(right_table = stmt->get_table_item_by_id(semi_info->right_table_id_))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("params have null", K(ret), K(stmt), K(semi_info), K(right_table)); - } else if (OB_FAIL(old_semi_conds.assign(semi_info->semi_conditions_))) { - LOG_WARN("failed to assign exprs", K(ret)); } else if (OB_FAIL(stmt->get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) { LOG_WARN("failed to get left rel ids", K(ret)); } else if (OB_FAIL(stmt->get_table_rel_ids(semi_info->right_table_id_, right_rel_ids))) { @@ -2429,22 +2516,14 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt, LOG_WARN("failed to get related columns", K(ret)); } else if (OB_FAIL(transform_predicates(*stmt, all_preds, cols, new_preds))) { LOG_WARN("failed to deduce predicates", K(ret)); - } else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) { - LOG_WARN("check_enable_no_pred_deduce failed", K(ret)); - } else if (!enable_no_pred_deduce && OB_FAIL(accept_predicates(*stmt, semi_info->semi_conditions_, + } else if (OB_FAIL(accept_predicates(*stmt, semi_info->semi_conditions_, properites, new_preds))) { LOG_WARN("failed to check different", K(ret)); - } else if (OB_FAIL(pushdown_into_table(stmt, right_table, pullup_preds, - enable_no_pred_deduce ? empty : semi_info->semi_conditions_))) { + } else if (OB_FAIL(pushdown_into_table(stmt, right_table, pullup_preds, semi_info->semi_conditions_))) { LOG_WARN("failed to push down predicates", K(ret)); } else if (OB_FAIL(pushdown_semi_info_right_filter(stmt, ctx_, semi_info))) { LOG_WARN("failed to pushdown semi info right filter", K(ret)); } - - if (OB_SUCC(ret) && OB_FAIL(check_transform_happened(stmt, old_semi_conds, - semi_info->semi_conditions_))) { - LOG_WARN("failed to check transform happened", K(ret)); - } return ret; } @@ -2503,7 +2582,6 @@ int ObTransformPredicateMoveAround::check_has_shared_query_ref(ObRawExpr *expr, // pushdown right table filter in semi condition: // 1. if right table is a basic table, create a generate table. // 2. pushdown the right table filters into the generate table. -// subquery in right filter can not be a shared subquery int ObTransformPredicateMoveAround::pushdown_semi_info_right_filter(ObDMLStmt *stmt, ObTransformerCtx *ctx, SemiInfo *semi_info) @@ -2523,21 +2601,15 @@ int ObTransformPredicateMoveAround::pushdown_semi_info_right_filter(ObDMLStmt *s LOG_WARN("failed to extract table exprs", K(ret)); } else if (right_filters.empty()) { // do nothing - } else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_info->semi_conditions_, right_filters))) { - LOG_WARN("failed to remove right filters", K(ret)); } else if (OB_FAIL(ObTransformUtils::can_push_down_filter_to_table(*right_table, can_push))) { LOG_WARN("failed to check can push down", K(ret), K(*right_table)); } else if (can_push) { - ObArray new_right_filters; - if (OB_FAIL(ObTransformUtils::move_expr_into_view(*ctx_->expr_factory_, - *stmt, - *right_table, - right_filters, - new_right_filters))) { - LOG_WARN("failed to move expr into view", K(ret)); - } else if (OB_FAIL(right_table->ref_query_->add_condition_exprs(new_right_filters))) { - LOG_WARN("failed to add condition exprs", K(ret)); - } + // if a right filter can push down to right_table, has pushdown in pushdown_into_table, + // do not reture error here + } else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_info->semi_conditions_, right_filters))) { + LOG_WARN("failed to remove right filters", K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) { + LOG_WARN("append transed stmt failed", K(ret)); } else if (OB_FAIL(ObTransformUtils::replace_with_empty_view(ctx_, stmt, view_table, @@ -2603,12 +2675,17 @@ int ObTransformPredicateMoveAround::pushdown_into_table(ObDMLStmt *stmt, if (OB_SUCC(ret) && table_item->is_generated_table()) { // if predicates are pushed into the view, we can remove them from the upper stmt ObSEArray invalid_preds; + uint64_t old_candi_preds_count = 0; if (OB_FAIL(rename_pushdown_predicates(*stmt, *table_item, rename_preds))) { LOG_WARN("failed to rename predicates", K(ret)); } else if (OB_FAIL(choose_pushdown_preds(rename_preds, invalid_preds, candi_preds))) { LOG_WARN("failed to choose predicates for pushdown", K(ret)); + } else if (OB_FALSE_IT(old_candi_preds_count = candi_preds.count())) { } else if (OB_FAIL(SMART_CALL(pushdown_predicates(table_item->ref_query_, candi_preds)))) { LOG_WARN("failed to push down predicates", K(ret)); + } else if (candi_preds.count() != old_candi_preds_count + && OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) { + LOG_WARN("append transed stmt failed", K(ret)); } else if (OB_FAIL(append(candi_preds, invalid_preds))) { LOG_WARN("failed to append predicates", K(ret)); } @@ -2958,8 +3035,7 @@ int ObTransformPredicateMoveAround::get_columns_in_filters( * @param preds * @return */ -int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt, - bool &is_happened) +int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt) { int ret = OB_SUCCESS; ObSEArray part_exprs; @@ -3114,11 +3190,10 @@ int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt * LOG_WARN("failed to add condition expr", K(ret)); } else if (is_not_null && OB_FAIL(ObTransformUtils::add_param_not_null_constraint(*ctx_, constraints))) { LOG_WARN("failed to add param not null constraint", K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, static_cast(del_upd_stmt)))) { + LOG_WARN("append failed", K(ret)); } else { - is_happened = true; - if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, static_cast(del_upd_stmt)))) { - LOG_WARN("append failed", K(ret)); - } + real_happened_ = true; } } } @@ -3271,6 +3346,7 @@ int ObTransformPredicateMoveAround::construct_transform_hint(ObDMLStmt &stmt, vo LOG_WARN("adjust stmt id failed", K(ret)); } else { hint->set_qb_name(qb_name); + OPT_TRACE("transformed query blocks:", qb_name); } } } diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.h b/src/sql/rewrite/ob_transform_predicate_move_around.h index c080446956a16df384eb365970e16fb2d6643911..502bb74127b687b9a1f038df403fa9e71e6bf28d 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.h +++ b/src/sql/rewrite/ob_transform_predicate_move_around.h @@ -43,21 +43,14 @@ private: const ObDMLStmt &stmt, bool &need_trans) override; + int sort_transed_stmts(); + int check_outline_valid_to_transform(const ObDMLStmt &stmt, bool &need_trans); int pullup_predicates(ObDMLStmt *stmt, ObIArray &select_list, ObIArray &properties); - int preprocess(ObDMLStmt &stmt); - - int preprocess_semi_info(ObDMLStmt &stmt, - SemiInfo *semi_info, - ObIArray &upper_conds); - int preprocess_joined_table(ObDMLStmt &stmt, - JoinedTable *join_table, - ObIArray &upper_conds); - int pullup_predicates_from_view(ObDMLStmt &stmt, ObIArray &sel_ids, ObIArray &input_pullup_preds); @@ -93,6 +86,12 @@ private: ObIArray &sel_ids, ObIArray &input_pullup_preds, ObIArray &output_pullup_preds); + int gather_pullup_preds_from_semi_outer_join(ObDMLStmt &stmt, + ObIArray &preds, + bool remove_preds = false); + int gather_pullup_preds_from_join_table(TableItem *table, + ObIArray &preds, + bool remove_preds); int remove_pullup_union_predicates(ObIArray &exprs); @@ -220,9 +219,18 @@ private: ObIArray &pullup_preds, ObIArray &pushdown_preds); - int check_transform_happened(ObDMLStmt *stmt, - ObIArray &old_conditions, - ObIArray &new_conditions); + int store_all_preds(const ObDMLStmt &stmt, ObIArray> &all_preds); + int store_join_conds(const TableItem *table, ObIArray> &all_preds); + int check_transform_happened(const ObDMLStmt &stmt, + const ObIArray> &all_preds, + bool &is_happened); + int check_join_conds_deduced(const TableItem *table, + const ObIArray> &all_preds, + uint64_t &idx, + bool &is_happened); + int check_conds_deduced(const ObIArray &old_conditions, + const ObIArray &new_conditions, + bool &is_happened); int pushdown_through_winfunc(ObSelectStmt &stmt, ObIArray &predicates, @@ -277,7 +285,7 @@ private: ObIArray &sel_items, ObIArray &columns); - int create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt, bool &is_happened); + int create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt); int print_debug_info(const char *str, ObDMLStmt *stmt, ObIArray &preds); @@ -325,6 +333,7 @@ private: Ob2DArray stmt_pullup_preds_; ObSEArray transed_stmts_; ObSEArray applied_hints_; + bool real_happened_; }; } diff --git a/tools/deploy/mysql_test/test_suite/groupby/r/mysql/group_by_basic.result b/tools/deploy/mysql_test/test_suite/groupby/r/mysql/group_by_basic.result index 317c67147aedc12b8bca03a69f3e4d1de41aaa83..dfbe33b1a567fcd5980ebaee0835417289e7155e 100644 --- a/tools/deploy/mysql_test/test_suite/groupby/r/mysql/group_by_basic.result +++ b/tools/deploy/mysql_test/test_suite/groupby/r/mysql/group_by_basic.result @@ -360,15 +360,15 @@ insert into t1 (c1, c2, c3) values (2, 3, 2); insert into t1 (c1, c2, c3) values (2, 3, 3); select c2, avg(distinct c3) from t1 group by c2; c2 avg(distinct c3) -2 2.5000 1 1.5000 +2 2.5000 3 2.5000 select c1,c2, group_concat(distinct c3 order by c3 desc) from t1 group by c1,c2; c1 c2 group_concat(distinct c3 order by c3 desc) 1 1 1 -2 3 3,2 -2 1 2 1 2 3,2 +2 1 2 +2 3 3,2 select c1,c2, group_concat(distinct c3 order by c3 desc) from t1 group by c1,c2 with rollup; c1 c2 group_concat(distinct c3 order by c3 desc) 1 1 1