From 5ae88eacbdaf93d00cc4596fa4ddc0bcf17e0a2f Mon Sep 17 00:00:00 2001 From: gjw2284740 Date: Mon, 23 Aug 2021 21:24:34 +0800 Subject: [PATCH] [opensource] add clog tool --- CMakeLists.txt | 7 +- src/storage/transaction/ob_trans_log.cpp | 72 + src/storage/transaction/ob_trans_log.h | 97 + tools/CMakeLists.txt | 3 +- tools/ob_admin/CMakeLists.txt | 32 + tools/ob_admin/clog_tool/cmd_args_parser.h | 150 ++ .../clog_tool/ob_admin_clog_v2_executor.cpp | 587 +++++ .../clog_tool/ob_admin_clog_v2_executor.h | 63 + tools/ob_admin/clog_tool/ob_func_utils.cpp | 205 ++ tools/ob_admin/clog_tool/ob_func_utils.h | 49 + .../clog_tool/ob_ilog_entry_parser.cpp | 118 + .../ob_admin/clog_tool/ob_ilog_entry_parser.h | 58 + .../clog_tool/ob_log_entry_filter.cpp | 76 + .../ob_admin/clog_tool/ob_log_entry_filter.h | 91 + .../clog_tool/ob_log_entry_parser.cpp | 2299 +++++++++++++++++ .../ob_admin/clog_tool/ob_log_entry_parser.h | 252 ++ tools/ob_admin/main.cpp | 60 + tools/ob_admin/ob_admin_executor.cpp | 57 + tools/ob_admin/ob_admin_executor.h | 43 + 19 files changed, 4317 insertions(+), 2 deletions(-) create mode 100644 tools/ob_admin/CMakeLists.txt create mode 100644 tools/ob_admin/clog_tool/cmd_args_parser.h create mode 100644 tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.cpp create mode 100644 tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.h create mode 100644 tools/ob_admin/clog_tool/ob_func_utils.cpp create mode 100644 tools/ob_admin/clog_tool/ob_func_utils.h create mode 100644 tools/ob_admin/clog_tool/ob_ilog_entry_parser.cpp create mode 100644 tools/ob_admin/clog_tool/ob_ilog_entry_parser.h create mode 100644 tools/ob_admin/clog_tool/ob_log_entry_filter.cpp create mode 100644 tools/ob_admin/clog_tool/ob_log_entry_filter.h create mode 100644 tools/ob_admin/clog_tool/ob_log_entry_parser.cpp create mode 100644 tools/ob_admin/clog_tool/ob_log_entry_parser.h create mode 100644 tools/ob_admin/main.cpp create mode 100644 tools/ob_admin/ob_admin_executor.cpp create mode 100644 tools/ob_admin/ob_admin_executor.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c9cfaec32..864a2abc12 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,6 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_subdirectory(deps/easy) add_subdirectory(deps/oblib) add_subdirectory(src) -add_subdirectory(tools) include(CMakeDependentOption) # OB_BUILD_RPM => include tools and build them. @@ -59,4 +58,10 @@ elseif(OB_INCLUDE_UNITTEST) add_subdirectory(unittest EXCLUDE_FROM_ALL) endif() +if (OB_BUILD_TOOLS) + add_subdirectory(tools) +elseif (OB_INCLUDE_TOOLS) + add_subdirectory(tools EXCLUDE_FROM_ALL) +endif() + include(cmake/RPM.cmake) diff --git a/src/storage/transaction/ob_trans_log.cpp b/src/storage/transaction/ob_trans_log.cpp index d7239b8b30..c83c39cf9c 100644 --- a/src/storage/transaction/ob_trans_log.cpp +++ b/src/storage/transaction/ob_trans_log.cpp @@ -382,6 +382,30 @@ bool ObTransRedoLog::is_xa_trans() const return xid_.is_valid() && !xid_.empty(); } +int ObTransRedoLog::init_for_deserialize(const bool use_mutator_buf) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(mutator_.init(use_mutator_buf))) { + TRANS_LOG(WARN, "redo log mutator init failed", K(ret)); + } + return ret; +} + +int ObTransRedoLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + UNUSED(real_tenant_id); + // no implementation + return ret; +} + +int ObTransRedoLog::decrypt_table_key() +{ + int ret = OB_SUCCESS; + // no implementation + return ret; +} + // prepare log generated by observers before version 2.0 is replayed on observers of version 2.0 // checkpoint has not been deserialized, no need to check at this time bool ObTransPrepareLog::is_valid() const @@ -707,6 +731,30 @@ int ObSpTransRedoLog::replace_tenant_id(const uint64_t new_tenant_id) return ret; } +int ObSpTransRedoLog::init_for_deserialize(const bool use_mutator_buf) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(mutator_.init(use_mutator_buf))) { + TRANS_LOG(WARN, "sp trans redo log mutator init failed", K(ret)); + } + return ret; +} + +int ObSpTransRedoLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + UNUSED(real_tenant_id); + // no implementation + return ret; +} + +int ObSpTransRedoLog::decrypt_table_key() +{ + int ret = OB_SUCCESS; + // no implementation + return ret; +} + int ObSpTransCommitLog::init(const int64_t log_type, const ObPartitionKey& partition, const uint64_t tenant_id, const ObTransID& trans_id, const uint64_t checksum, const uint64_t cluster_id, const ObRedoLogIdArray& redo_log_ids, const ObStartTransParam& trans_param, const int64_t log_no, const ObString& app_trace_id_str, @@ -933,6 +981,30 @@ int ObTransMutatorLog::replace_tenant_id(const uint64_t new_tenant_id) return ret; } +int ObTransMutatorLog::init_for_deserialize(const bool use_mutator_buf) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(mutator_.init(use_mutator_buf))) { + TRANS_LOG(WARN, "mutator log mutator init failed", K(ret)); + } + return ret; +} + +int ObTransMutatorLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + UNUSED(real_tenant_id); + // no implementation + return ret; +} + +int ObTransMutatorLog::decrypt_table_key() +{ + int ret = OB_SUCCESS; + // no implementation + return ret; +} + int ObTransMutatorAbortLog::init( const int64_t log_type, const ObPartitionKey& pkey, const ObTransID& trans_id, const uint64_t cluster_id) { diff --git a/src/storage/transaction/ob_trans_log.h b/src/storage/transaction/ob_trans_log.h index e76cd68075..abc80db848 100644 --- a/src/storage/transaction/ob_trans_log.h +++ b/src/storage/transaction/ob_trans_log.h @@ -319,6 +319,9 @@ public: return xid_; } bool is_xa_trans() const; + int init_for_deserialize(const bool use_mutator_buf = true); + int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id); + int decrypt_table_key(); VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(scheduler), K_(coordinator), K_(participants), K_(trans_param), K_(cluster_id), K_(active_memstore_version), @@ -664,6 +667,30 @@ public: bool is_valid() const override; }; +class ObSpTransRedoLogHelper { +public: + ObSpTransRedoLogHelper() + : tenant_id_(common::OB_INVALID_TENANT_ID), + log_no_(-1), + trans_param_(), + mutator_(), + active_memstore_version_(), + prev_trans_arr_(), + can_elr_(false) + {} + ~ObSpTransRedoLogHelper() + {} + +public: + uint64_t tenant_id_; + int64_t log_no_; + ObStartTransParam trans_param_; + ObTransMutator mutator_; + common::ObVersion active_memstore_version_; + ObElrTransInfoArray prev_trans_arr_; + bool can_elr_; +}; + class ObSpTransRedoLog : public ObTransLog { OB_UNIS_VERSION(1); @@ -678,6 +705,16 @@ public: prev_trans_arr_(), can_elr_(false) {} + ObSpTransRedoLog(ObSpTransRedoLogHelper& helper) + : ObTransLog(), + tenant_id_(helper.tenant_id_), + log_no_(helper.log_no_), + trans_param_(helper.trans_param_), + mutator_(), + active_memstore_version_(helper.active_memstore_version_), + prev_trans_arr_(helper.prev_trans_arr_), + can_elr_(helper.can_elr_) + {} ~ObSpTransRedoLog() {} int init(const int64_t log_type, const common::ObPartitionKey& partition, const ObTransID& trans_id, @@ -721,6 +758,9 @@ public: } bool is_valid() const override; virtual int replace_tenant_id(const uint64_t new_tenant_id) override; + int init_for_deserialize(const bool use_mutator_buf = true); + int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id); + int decrypt_table_key(); VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(trans_param), K_(cluster_id), K_(active_memstore_version), K_(prev_trans_arr), K_(can_elr)); @@ -738,6 +778,14 @@ protected: bool can_elr_; }; +class ObSpTransCommitLogHelper : public ObSpTransRedoLogHelper { +public: + ObSpTransCommitLogHelper() : ObSpTransRedoLogHelper() + {} + ~ObSpTransCommitLogHelper() + {} +}; + class ObSpTransCommitLog : public ObSpTransRedoLog { OB_UNIS_VERSION(1); @@ -751,6 +799,15 @@ public: checkpoint_(0), app_trace_info_() {} + ObSpTransCommitLog(ObSpTransCommitLogHelper& helper) + : ObSpTransRedoLog(helper), + global_trans_version_(-1), + checksum_(0), + prev_redo_log_ids_(common::ObModIds::OB_TRANS_REDO_LOG_ID_ARRAY, common::OB_MALLOC_NORMAL_BLOCK_SIZE), + app_trace_id_str_(), + checkpoint_(0), + app_trace_info_() + {} ~ObSpTransCommitLog() {} int init(const int64_t log_type, const common::ObPartitionKey& partition, const uint64_t tenant_id, @@ -1009,6 +1066,32 @@ private: ObXATransID xid_; }; +class ObTransMutatorLogHelper { +public: + ObTransMutatorLogHelper() + : tenant_id_(common::OB_INVALID_TENANT_ID), + trans_expired_time_(0), + trans_param_(), + log_no_(0), + mutator_(), + prev_trans_arr_(), + can_elr_(false), + cluster_version_(0) + {} + ~ObTransMutatorLogHelper() + {} + +public: + int64_t tenant_id_; + int64_t trans_expired_time_; + ObStartTransParam trans_param_; + int64_t log_no_; + ObTransMutator mutator_; + ObElrTransInfoArray prev_trans_arr_; + bool can_elr_; + uint64_t cluster_version_; +}; + class ObTransMutatorLog : public ObTransLog { OB_UNIS_VERSION(1); @@ -1024,6 +1107,17 @@ public: can_elr_(false), cluster_version_(0) {} + ObTransMutatorLog(ObTransMutatorLogHelper& helper) + : ObTransLog(), + tenant_id_(helper.tenant_id_), + trans_expired_time_(helper.trans_expired_time_), + trans_param_(), + log_no_(helper.log_no_), + mutator_(), + prev_trans_arr_(), + can_elr_(helper.can_elr_), + cluster_version_(helper.cluster_version_) + {} ~ObTransMutatorLog() { destroy(); @@ -1073,6 +1167,9 @@ public: return cluster_version_; } virtual int replace_tenant_id(const uint64_t new_tenant_id) override; + int init_for_deserialize(const bool use_mutator_buf = true); + int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id); + int decrypt_table_key(); public: TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(cluster_id), K_(tenant_id), K_(trans_expired_time), diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e27d0f2b62..87e6e16891 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1 +1,2 @@ -add_subdirectory(ob_error) \ No newline at end of file +add_subdirectory(ob_error) +add_subdirectory(ob_admin) diff --git a/tools/ob_admin/CMakeLists.txt b/tools/ob_admin/CMakeLists.txt new file mode 100644 index 0000000000..e83e39134c --- /dev/null +++ b/tools/ob_admin/CMakeLists.txt @@ -0,0 +1,32 @@ +add_executable(ob_admin + clog_tool/cmd_args_parser.h + clog_tool/ob_admin_clog_v2_executor.cpp + clog_tool/ob_admin_clog_v2_executor.h + clog_tool/ob_func_utils.cpp + clog_tool/ob_func_utils.h + clog_tool/ob_ilog_entry_parser.cpp + clog_tool/ob_ilog_entry_parser.h + clog_tool/ob_log_entry_filter.cpp + clog_tool/ob_log_entry_filter.h + clog_tool/ob_log_entry_parser.cpp + clog_tool/ob_log_entry_parser.h + ob_admin_executor.h + ob_admin_executor.cpp + main.cpp) + +if (OB_STATIC_LINK_LGPL_DEPS) + set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -l:libmariadbclient.a -l:libaio.a") +endif() + +target_link_libraries(ob_admin + PRIVATE + oceanbase_static + -static-libgcc + -static-libstdc++ + ${LGPL_DEPS} + ) + + +target_include_directories(ob_admin + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tools/ob_admin/clog_tool/cmd_args_parser.h b/tools/ob_admin/clog_tool/cmd_args_parser.h new file mode 100644 index 0000000000..a67ca76d6b --- /dev/null +++ b/tools/ob_admin/clog_tool/cmd_args_parser.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef __OB_TOOLS_CMD_ARGS_PARSER_H__ +#define __OB_TOOLS_CMD_ARGS_PARSER_H__ +class CmdArgsParser { + const static int64_t MAX_N_ARGS = 1 << 10; + struct arg_t { + arg_t() : name_(NULL), value_(null_), default_value_(NULL) + {} + ~arg_t() + {} + const char* name_; + const char* value_; + const char* default_value_; + }; + +public: + CmdArgsParser() : parse_seq_(0), n_args_(0) + { + default_arg_.name_ = "*default*"; + default_arg_.value_ = null_; + default_arg_.default_value_ = null_; + } + ~CmdArgsParser() + {} + bool reset() + { + memset(args_, 0, sizeof(args_)); + n_args_ = 0; + parse_seq_ |= 1; + return true; + } + bool check(int argc, char** argv, ...) + { + bool args_is_valid = true; + char* p = NULL; + parse_seq_ = (parse_seq_ & ~1) + 2; + + for (int64_t i = 0; i < n_args_ / 2; i++) { + arg_t arg = args_[i]; + args_[i] = args_[n_args_ - 1 - i]; + args_[n_args_ - 1 - i] = arg; + } + for (int64_t i = 0; i < argc; i++) { + if (argv[i][0] == ':' || NULL == (p = strchr(argv[i], '='))) + continue; + *p++ = 0; + arg_t* arg = get_arg(argv[i]); + if (arg && &default_arg_ != arg) + arg->value_ = p; + *--p = '='; + } + for (int64_t i = 0; i < argc; i++) { + if (argv[i][0] != ':' && (p = strchr(argv[i], '='))) + continue; + p = argv[i][0] == ':' ? argv[i] + 1 : argv[i]; + arg_t* arg = get_next_unset_arg(); + if (arg && arg->name_) + arg->value_ = p; + } + for (int64_t i = 0; i < n_args_; i++) { + if (null_ == args_[i].value_ && args_[i].default_value_) + args_[i].value_ = args_[i].default_value_; + if (null_ == args_[i].value_) + args_is_valid = false; + } + if (0 == strcmp("true", getenv("dump_args") ?: "false")) { + dump(argc, argv); + } + return args_is_valid; + } + + void dump(int argc, char** argv) + { + printf("cmd_args_parser.dump:\n"); + for (int64_t i = 0; i < argc; i++) { + printf("argv[%ld]=%s\n", i, argv[i]); + } + for (int64_t i = 0; i < n_args_; i++) { + printf( + "args[%ld]={name=%s, value=%s, default=%s}\n", i, args_[i].name_, args_[i].value_, args_[i].default_value_); + } + } + + arg_t* get_next_unset_arg() + { + for (int64_t i = 0; i < n_args_; i++) { + if (null_ == args_[i].value_) + return args_ + i; + } + return NULL; + } + + arg_t* get_arg(const char* name, const char* default_value = NULL) + { + assert(n_args_ < MAX_N_ARGS && name); + if (parse_seq_ & 1) { + args_[n_args_].name_ = name; + args_[n_args_].default_value_ = default_value; + args_[n_args_].value_ = null_; + return args_ + (n_args_++); + } + for (int64_t i = 0; i < n_args_; i++) { + if (0 == strcmp(args_[i].name_, name)) + return args_ + i; + } + return &default_arg_; + } + +private: + static const char* null_; + int64_t parse_seq_; + int64_t n_args_; + arg_t default_arg_; + arg_t args_[MAX_N_ARGS]; +}; +inline bool argv1_match_func(const char* argv1, const char* func) +{ + const char* last_part = strrchr(func, '.'); + if (NULL != last_part) { + last_part++; + } else { + last_part = func; + } + return 0 == strcmp(last_part, argv1); +} + +const char* CmdArgsParser::null_ __attribute__((weak)) = "*null*"; +CmdArgsParser __cmd_args_parser __attribute__((weak)); +#define _Arg(name, ...) __cmd_args_parser.get_arg(#name, ##__VA_ARGS__) +#define BoolArg(name, ...) (0 == atoll(__cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_)) ? true : false +#define IntArg(name, ...) atoll(__cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_) +#define StrArg(name, ...) __cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_ +#define CmdCall(argc, argv, func, ...) \ + (argc >= 2 && argv1_match_func(argv[1], #func) && __cmd_args_parser.reset() && \ + __cmd_args_parser.check(argc - 2, argv + 2, ##__VA_ARGS__)) \ + ? func(__VA_ARGS__) + +#define CmdCallSimple(argc, argv, func) (argc >= 3 && argv1_match_func(argv[1], #func)) ? func(argc - 2, argv + 2) +#endif /* __OB_TOOLS_CMD_ARGS_PARSER_H__ */ diff --git a/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.cpp b/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.cpp new file mode 100644 index 0000000000..b964d4f62f --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.cpp @@ -0,0 +1,587 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX CLOG +#include +#include +#include +#include +#include +#include "ob_log_entry_parser.h" +#include "ob_ilog_entry_parser.h" +#include "cmd_args_parser.h" + +#include "ob_admin_clog_v2_executor.h" +#include "ob_func_utils.h" +#include "share/ob_srv_rpc_proxy.h" +#include "share/ob_version.h" + +using namespace oceanbase::common; +using namespace oceanbase::share; +using namespace oceanbase::clog; + +namespace oceanbase { +namespace tools { + +#define getcfg(key) getenv(key) + +ObAdminClogV2Executor::ObAdminClogV2Executor() : DB_port_(-1), is_ofs_open_(false) +{} + +ObAdminClogV2Executor::~ObAdminClogV2Executor() +{} + +int ObAdminClogV2Executor::execute(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(argc < 4) || OB_ISNULL(argv)) { + print_usage(); + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(parse_options(argc, argv))) { + LOG_WARN("failed to parse options", K(ret)); + } else { + int new_argc = argc - optind; + char** new_argv = argv + optind; + if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_all) : OB_NEED_RETRY)) { + LOG_INFO("finish dump_all ", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_filter) : OB_NEED_RETRY)) { + LOG_INFO("finish dump_filter ", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_hex) : OB_NEED_RETRY)) { + LOG_INFO("finish dump_hex ", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_format) : OB_NEED_RETRY)) { + LOG_INFO("finish dump_format ", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, stat_clog) : OB_NEED_RETRY)) { + LOG_INFO("finish stat_clog ", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, grep) : OB_NEED_RETRY)) { + LOG_INFO("finish encode", K(ret)); + } else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_ilog) : OB_NEED_RETRY)) { + LOG_INFO("finish encode", K(ret)); + } else { + fprintf(stderr, "failed %d", ret); + print_usage(); + } + } + return ret; +} + +int ObAdminClogV2Executor::parse_options(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + int option_index = 0; + struct option long_options[] = {{"host", 1, NULL, 'h'}, {"port", 1, NULL, 'p'}, {NULL, 0, NULL, 0}}; + int c; + while (-1 != (c = getopt_long(argc, argv, "h:p:", long_options, &option_index))) { + switch (c) { + case 'h': + DB_host_.assign_ptr(optarg, strlen(optarg)); + break; + case 'p': + DB_port_ = static_cast(strtol(optarg, NULL, 10)); + break; + case '?': + case ':': + ret = OB_ERR_UNEXPECTED; + break; + default: + break; + } + } + return ret; +} + +void ObAdminClogV2Executor::print_usage() +{ + fprintf(stdout, + "Usages:\n" + "$ob_admin clog_tool dump_ilog ilog_files ## ./ob_admin clog_tool dump_ilog 1 2 3\n" + "$ob_admin clog_tool dump_all log_files ## ./ob_admin clog_tool dump_all 1 2 3\n" + "$ob_admin clog_tool dump_filter filter_str log_files ## ./ob_admin clog_tool dump_filter " + "'table_id=123;partition_id=123;log_id=123' 1 2 3\n" + "$ob_admin clog_tool dump_hex log_files ## ./ob_admin clog_tool dump_hex 1 2 3\n" + "$ob_admin clog_tool dump_format log_files ## ./ob_admin clog_tool dump_format 1 2 3\n"); +} + +int ObAdminClogV2Executor::dump_all(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + const bool is_hex = false; + if (OB_FAIL(dump_inner(argc, argv, is_hex))) { + LOG_WARN("failed to dump all", K(ret)); + } + return ret; +} + +int ObAdminClogV2Executor::dump_filter(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + const bool is_hex = false; + if (OB_FAIL(filter_.parse(argv[0]))) { + LOG_WARN("parse filter failed", K(ret), K(argv[0])); + } else { + LOG_INFO("dump with filter", K_(filter), K(argv[0])); + if (OB_FAIL(dump_inner(argc - 1, argv + 1, is_hex))) { + LOG_WARN("failed to dump all", K(ret)); + } + } + return ret; +} + +int ObAdminClogV2Executor::dump_hex(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + const bool is_hex = true; + if (OB_FAIL(dump_inner(argc, argv, is_hex))) { + LOG_WARN("failed to dump hex", K(ret)); + } + return ret; +} + +int ObAdminClogV2Executor::dump_inner(int argc, char* argv[], bool is_hex) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(argc), K(ret)); + } else { + LOG_INFO("begin to dump all ", K(is_hex), K(ret)); + for (int64_t i = 0; i < argc; ++i) { + if (OB_FAIL(dump_single_clog(argv[i], is_hex)) && OB_ITER_END != ret) { + LOG_WARN("failed to dump log ", K(argv[i]), K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { /*do nothing*/ + } + } + } + return ret; +} + +int ObAdminClogV2Executor::dump_format(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(argc), K(ret)); + } else { + LOG_INFO("begin to dump format", K(ret)); + for (int64_t i = 0; i < argc; ++i) { + if (OB_FAIL(dump_format_single_file(argv[i])) && (OB_ITER_END != ret)) { + LOG_WARN("failed to dump format ", K(argv[i]), K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { /*do nothing*/ + } + } + } + return ret; +} + +int ObAdminClogV2Executor::stat_clog(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(argc), K(ret)); + } else { + LOG_INFO("begin to stat_clog", K(ret)); + for (int64_t i = 0; i < argc; ++i) { + if (OB_FAIL(stat_single_log(argv[i])) && (OB_ITER_END != ret)) { + LOG_WARN("failed to stat_single_log", K(argv[i]), K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { /*do nothing*/ + } + } + } + return ret; +} + +int ObAdminClogV2Executor::dump_ilog(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(argc), K(ret)); + } else { + LOG_INFO("begin to dump ilog ", K(ret)); + for (int64_t i = 0; i < argc; ++i) { + if (OB_FAIL(dump_single_ilog(argv[i])) && OB_ITER_END != ret) { + LOG_WARN("failed to dump ilog ", K(argv[i]), K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { /*do nothing*/ + } + } + } + return ret; +} + +int ObAdminClogV2Executor::grep(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + const int64_t BUF_LEN = 1024; + char result[BUF_LEN] = {0}; + if (OB_UNLIKELY(1 > argc) || OB_ISNULL(argv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(argc), K(ret)); + } else { + LOG_INFO("begin to grep", K(ret)); + char arg_str[BUF_LEN] = {0}; + int64_t pos = 0; + memcpy(arg_str, argv[0], strlen(argv[0])); + char* input_str = arg_str; + char encode_type[64] = {0}; + int64_t int_value = 0; + char origin_str[BUF_LEN] = {0}; + char* token = NULL; + int32_t local_ret = 0; + while (OB_SUCC(ret) && NULL != (token = STRSEP(&input_str, "%"))) { + MEMSET(origin_str, '\0', BUF_LEN); + if (0 == STRLEN(token)) { + // do nothing + } else if (0 == (local_ret = sscanf(token, "%[0-9a-z]:%ld%s", encode_type, &int_value, origin_str)) || + 1 == local_ret) { + if (0 > (local_ret = snprintf(result + pos, BUF_LEN - pos, "%s", token))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("buf is not enough", "token_len", STRLEN(token), K(local_ret), K(BUF_LEN), K(ret)); + } else { + pos += local_ret; + LOG_DEBUG("match normal str", K(token), K(pos)); + } + } else if (2 == local_ret || 3 == local_ret) { + if (0 != STRCMP("i4", encode_type) && 0 != STRCMP("i8", encode_type) && 0 != STRCMP("v4", encode_type) && + 0 != STRCMP("v8", encode_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid encode type", K(token), K(encode_type), K(ret)); + print_usage(); + } else if (OB_FAIL(encode_int(result, pos, BUF_LEN, encode_type, int_value))) { + LOG_WARN("failed to encode_int", K(result), K(pos), K(BUF_LEN), K(encode_type), K(int_value), K(ret)); + } else { + if (0 > (local_ret = snprintf(result + pos, BUF_LEN - pos, "%s", origin_str))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("buf is not enough", "token_len", STRLEN(token), K(BUF_LEN), K(ret)); + } else { + pos += local_ret; + LOG_DEBUG("match normal str", K(token), K(pos)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid token", K(token), K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (1 == argc) { + int fd = 0; + struct stat sb; + if (-1 == fstat(fd, &sb)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to fstat", K(errno), K(ret)); + } else { + if (S_ISFIFO(sb.st_mode) || S_ISREG(sb.st_mode)) { + execl("/usr/bin/xargs", "xargs", "-n", "1", "-P", "30", "grep", "-UP", result, NULL); + if (-1 == errno) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execve", K(errno), K(ret)); + } + } else { + fprintf(stdout, "ls store/clog/{1..3} |xargs -n 1 -P 30 grep -UP '%s' \n", result); + } + } + } else if (argc > 1) { + const int64_t COMMAND_BUF_SIZE = 1024; + char command[COMMAND_BUF_SIZE] = {0}; + snprintf(command, COMMAND_BUF_SIZE, "xargs -n 1 -P 30 grep -UP '%s' \n", result); + FILE* file = popen(command, "w"); + + if (NULL == file) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to popen", K(errno), K(ret)); + } else { + for (int64_t i = 1; OB_SUCC(ret) && i < argc; ++i) { + size_t w_len = 0; + size_t arg_len = STRLEN(argv[i]); + const size_t element_size = 1; + if (arg_len != (w_len = fwrite(argv[i], element_size, arg_len, file))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to fwrite", K(argv[i]), K(i), K(ret)); + } else { + fwrite((char*)"\n", 1, 1, file); + } + } + if (OB_SUCC(ret)) { + if (-1 == pclose(file)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to pclose", K(errno), K(ret)); + } + } + } + } + } + return ret; +} + +int ObAdminClogV2Executor::encode_int( + char* buf, int64_t& pos, int64_t buf_len, const char* encode_type, int64_t int_value) +{ + int ret = OB_SUCCESS; + const int64_t BUF_LEN = 128; + char buf_local[BUF_LEN] = {0}; + int64_t local_pos = 0; + if (0 == STRCMP("i8", encode_type)) { + if (OB_FAIL(serialization::encode_i64(buf_local, BUF_LEN, local_pos, int_value))) { + LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret)); + } else { + CLOG_LOG(DEBUG, "succ to encode i64", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret)); + } + } else if (0 == STRCMP("i4", encode_type)) { + if (OB_FAIL(serialization::encode_i32(buf_local, BUF_LEN, local_pos, static_cast(int_value)))) { + LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret)); + } else { + CLOG_LOG(DEBUG, "succ to encode i32", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret)); + } + } else if (0 == STRCMP("v8", encode_type)) { + if (OB_FAIL(serialization::encode_vi64(buf_local, BUF_LEN, local_pos, int_value))) { + LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret)); + } else { + CLOG_LOG(DEBUG, "succ to encode vi64", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret)); + } + } else if (0 == STRCMP("v4", encode_type)) { + if (OB_FAIL(serialization::encode_vi32(buf_local, BUF_LEN, local_pos, static_cast(int_value)))) { + LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret)); + } else { + CLOG_LOG(DEBUG, "succ to encode vi32", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid encode_type", K(encode_type), K(ret)); + } + + if (OB_SUCC(ret)) { + int ret_len = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < local_pos; ++i) { + if (4 != (ret_len = snprintf(buf + pos, buf_len - pos, "\\x%02x", (unsigned int)(unsigned char)(buf_local[i])))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to snprinf", K(buf), K(buf_local[i]), K(ret_len), K(ret)); + } else { + pos += 4; + } + } + } + + return ret; +} + +int ObAdminClogV2Executor::dump_single_clog(const char* path, bool is_hex) +{ + int ret = OB_SUCCESS; + int fd = -1; + char* buf = NULL; + int64_t buf_len = -1; + uint64_t file_id = -1; + if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) { + LOG_WARN("failed to mmap_log_file", K(path), K(ret)); + } else if (OB_FAIL(file_name_parser(path, file_id))) { + LOG_WARN("failed to parse file name", K(path), K(ret)); + } else { + const bool is_ofs = is_ofs_file(path); + ObLogEntryParser entry_parser; + if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) { + LOG_WARN("failed to init entry parser", K(path), K(ret)); + } else if (OB_FAIL(entry_parser.dump_all_entry(is_hex))) { + if (OB_ITER_END == ret) { + LOG_INFO("succ to dump_all_entry", K(path)); + } else { + LOG_WARN("failed to dump_all_entry", K(path), K(ret)); + } + } else { /*do nothing*/ + } + } + + close_fd(path, fd, buf, buf_len); + return ret; +} + +int ObAdminClogV2Executor::dump_single_ilog(const char* path) +{ + int ret = OB_SUCCESS; + int fd = -1; + char* buf = NULL; + int64_t buf_len = -1; + uint64_t file_id = -1; + if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) { + LOG_WARN("failed to mmap_log_file", K(path), K(ret)); + } else if (OB_FAIL(file_name_parser(path, file_id))) { + LOG_WARN("failed to parse file name", K(path), K(ret)); + } else { + ObILogEntryParser entry_parser; + if (OB_FAIL(entry_parser.init(file_id, buf, buf_len))) { + LOG_WARN("failed to init entry parser", K(path), K(ret)); + } else if (OB_FAIL(entry_parser.parse_all_entry())) { + if (OB_ITER_END == ret) { + LOG_INFO("succ to dump_all_entry", K(path)); + } else { + LOG_WARN("failed to dump_all_entry", K(path), K(ret)); + } + } else { /*do nothing*/ + } + } + + close_fd(path, fd, buf, buf_len); + return ret; +} + +int ObAdminClogV2Executor::dump_format_single_file(const char* path) +{ + int ret = OB_SUCCESS; + int fd = -1; + char* buf = NULL; + int64_t buf_len = -1; + uint64_t file_id = -1; + if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) { + LOG_WARN("failed to mmap_log_file", K(path), K(ret)); + } else if (OB_FAIL(file_name_parser(path, file_id))) { + LOG_WARN("failed to parser file name ", K(path), K(ret)); + } else { + const bool is_ofs = is_ofs_file(path); + ObLogEntryParser entry_parser; + if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) { + LOG_WARN("failed to init entry parser", K(path), K(ret)); + } else if (OB_FAIL(entry_parser.format_dump_entry())) { + if (OB_ITER_END == ret) { + LOG_INFO("succ to format_dump_all_entry", K(path)); + } else { + LOG_WARN("failed to format_dump_entry", K(path), K(ret)); + } + } else { /*do nothing*/ + } + } + + close_fd(path, fd, buf, buf_len); + return ret; +} + +int ObAdminClogV2Executor::stat_single_log(const char* path) +{ + int ret = OB_SUCCESS; + int fd = -1; + char* buf = NULL; + int64_t buf_len = -1; + uint64_t file_id = -1; + if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) { + LOG_WARN("failed to mmap_log_file", K(path), K(ret)); + } else if (OB_FAIL(file_name_parser(path, file_id))) { + LOG_WARN("failed to parser file name", K(path), K(ret)); + } else { + const bool is_ofs = is_ofs_file(path); + ObLogEntryParser entry_parser; + if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) { + LOG_WARN("failed to init entry parser", K(ret)); + } else if (OB_FAIL(entry_parser.stat_log()) && (OB_ITER_END != ret)) { + LOG_WARN("failed to stat log", K(path), K(ret)); + } else { + const ObLogStat& log_stat = entry_parser.get_log_stat(); + fprintf(stdout, "log_file:%s\t stat_info:%s\n ", path, to_cstring(log_stat)); + fprintf(stdout, + "log_file:%s\t stat_info:\n data_block_header_size = %lf M;\n log_header_size = %lf M;\n " + " log_size = %lf M;\n " + " trans_log_size = %lf M;\n mutator_size = %lf M;\n padding_size = %lf M;\n " + " new_row_size = %lf M;\n old_row_size = %lf M;\n total_row_size = %lf M;\n" + " new_primary_row_size = %lf M;\n primary_row_count = %ld;\n total_row_count = " + "%ld;\n total_log_count = %ld;\n dist_trans_count = %ld;\n sp_trans_count = %ld;\n" + " non_compressed_log_cnt = %ld;\n compressed_log_cnt = %ld;\n compressed_log_size " + "= %lf M;\n original_log_size = %lf M;\n compress_ratio = %lf;\n " + "compressed_tenant_ids:[%s];\n", + path, + (double)log_stat.data_block_header_size_ / 1024 / 1024, + (double)log_stat.log_header_size_ / 1024 / 1024, + (double)log_stat.log_size_ / 1024 / 1024, + (double)log_stat.trans_log_size_ / 1024 / 1024, + (double)log_stat.mutator_size_ / 1024 / 1024, + (double)log_stat.padding_size_ / 1024 / 1024, + (double)log_stat.new_row_size_ / 1024 / 1024, + (double)log_stat.old_row_size_ / 1024 / 1024, + (double)(log_stat.old_row_size_ + log_stat.new_row_size_) / 1024 / 1024, + (double)log_stat.new_primary_row_size_ / 1024 / 1024, + log_stat.primary_row_count_, + log_stat.total_row_count_, + log_stat.total_log_count_, + log_stat.dist_trans_count_, + log_stat.sp_trans_count_, + log_stat.non_compressed_log_cnt_, + log_stat.compressed_log_cnt_, + (double)log_stat.compressed_log_size_ / 1024 / 1024, + (double)log_stat.original_log_size_ / 1024 / 1024, + (0 == log_stat.original_log_size_) ? 1 : (double)log_stat.compressed_log_size_ / log_stat.original_log_size_, + to_cstring(log_stat.compressed_tenant_ids_)); + } + } + + close_fd(path, fd, buf, buf_len); + return ret; +} + +int ObAdminClogV2Executor::mmap_log_file(const char* path, char*& buf_out, int64_t& buf_len, int& fd) +{ + int ret = OB_SUCCESS; + void* buf = NULL; + struct stat stat_buf; + if (OB_ISNULL(path)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid log file path is NULL", K(ret)); + } else if (is_ofs_file(path)) { // OFS file + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support ofs", K(ret)); + } else { // Local file + if (-1 == (fd = open(path, O_RDONLY))) { + ret = OB_IO_ERROR; + CLOG_LOG(ERROR, "open file fail", K(path), KERRMSG, K(ret)); + } else if (-1 == fstat(fd, &stat_buf)) { + ret = OB_IO_ERROR; + CLOG_LOG(ERROR, "stat_buf error", K(path), KERRMSG, K(ret)); + } else if (stat_buf.st_size > LOG_FILE_MAX_SIZE) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "invalid file size", K(path), K(stat_buf.st_size), K(ret)); + } else if (MAP_FAILED == (buf = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_SHARED, fd, 0)) || NULL == buf) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to mmap file", K(path), K(errno), KERRMSG, K(ret)); + } else { + buf_out = static_cast(buf); + buf_len = stat_buf.st_size; + } + } + return ret; +} + +int ObAdminClogV2Executor::close_fd(const char* path, const int fd, char* buf, const int64_t buf_len) +{ + int ret = OB_SUCCESS; + if (is_ofs_file(path)) { + if (fd > 0) { + // close fd for ofs + } + if (nullptr != buf) { + ob_free(buf); + } + } else { + if (nullptr != buf) { + munmap(buf, buf_len); + } + if (fd >= 0) { + close(fd); + } + } + return ret; +} +} // namespace tools +} // namespace oceanbase diff --git a/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.h b/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.h new file mode 100644 index 0000000000..8898aad5b6 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.h @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ADMIN_CLOG_V2_EXECUTOR_H_ +#define OB_ADMIN_CLOG_V2_EXECUTOR_H_ + +#include "../ob_admin_executor.h" +#include "ob_log_entry_filter.h" +#include "lib/string/ob_string.h" + +namespace oceanbase { +using namespace clog; + +namespace tools { +class ObAdminClogV2Executor : public ObAdminExecutor { +public: + ObAdminClogV2Executor(); + virtual ~ObAdminClogV2Executor(); + virtual int execute(int argc, char* argv[]); + +private: + int dump_all(int argc, char* argv[]); + int dump_filter(int argc, char* argv[]); + int dump_hex(int argc, char* argv[]); + int dump_inner(int argc, char* argv[], bool is_hex); + int dump_format(int argc, char* argv[]); + int stat_clog(int argc, char* argv[]); + int dump_ilog(int argc, char* argv[]); + + void print_usage(); + int parse_options(int argc, char* argv[]); + + int grep(int argc, char* argv[]); + int encode_int(char* buf, int64_t& pos, int64_t buf_len, const char* encode_type, int64_t int_value); + + int dump_single_clog(const char* path, bool is_hex); + int dump_single_ilog(const char* path); + int dump_format_single_file(const char* path); + int stat_single_log(const char* path); + + int mmap_log_file(const char* path, char*& buf_out, int64_t& buf_len, int& fd); + int close_fd(const char* path, const int fd, char* buf, const int64_t buf_len); + +private: + ObLogEntryFilter filter_; + common::ObString DB_host_; + int32_t DB_port_; + bool is_ofs_open_; +}; + +} // namespace tools +} // namespace oceanbase + +#endif /* OB_ADMIN_CLOG_EXECUTOR_V2_H_ */ diff --git a/tools/ob_admin/clog_tool/ob_func_utils.cpp b/tools/ob_admin/clog_tool/ob_func_utils.cpp new file mode 100644 index 0000000000..5f1122b7b9 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_func_utils.cpp @@ -0,0 +1,205 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_func_utils.h" + +const char* get_submit_log_type(const int64_t submit_log_type) +{ + const char* char_ret = NULL; + switch (submit_log_type) { + case OB_LOG_SP_TRANS_REDO: + char_ret = "OB_LOG_SP_TRANS_REDO"; + break; + case OB_LOG_TRANS_REDO: + char_ret = "OB_LOG_TRANS_REDO"; + break; + case OB_LOG_TRANS_PREPARE: + char_ret = "OB_LOG_TRANS_PREPARE"; + break; + case OB_LOG_TRANS_REDO_WITH_PREPARE: + char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE"; + break; + case OB_LOG_TRANS_PREPARE_WITH_COMMIT: + char_ret = "OB_LOG_TRANS_PREPARE_WITH_COMMIT"; + break; + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT: + char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT"; + break; + case OB_LOG_SP_TRANS_COMMIT: + char_ret = "OB_LOG_SP_TRANS_COMMIT"; + break; + case OB_LOG_SP_ELR_TRANS_COMMIT: + char_ret = "OB_LOG_SP_ELR_TRANS_COMMIT"; + break; + case OB_LOG_TRANS_COMMIT: + char_ret = "OB_LOG_TRANS_COMMIT"; + break; + case OB_LOG_SP_TRANS_ABORT: + char_ret = "OB_LOG_SP_TRANS_ABORT"; + break; + case OB_LOG_TRANS_ABORT: + char_ret = "OB_LOG_TRANS_ABORT"; + break; + case OB_LOG_TRANS_CLEAR: + char_ret = "OB_LOG_TRANS_CLEAR"; + break; + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR: + char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR"; + break; + case OB_LOG_MUTATOR: + char_ret = "OB_LOG_MUTATOR"; + break; + case OB_LOG_TRANS_STATE: + char_ret = "OB_LOG_TRANS_STATE"; + break; + case OB_LOG_MUTATOR_WITH_STATE: + char_ret = "OB_LOG_MUTATOR_WITH_STATE"; + break; + case OB_LOG_MUTATOR_ABORT: + char_ret = "OB_LOG_MUTATOR_ABORT"; + break; + case OB_LOG_SPLIT_SOURCE_PARTITION: + char_ret = "OB_LOG_SPLIT_SOURCE_PARTITION"; + break; + case OB_LOG_SPLIT_DEST_PARTITION: + char_ret = "OB_LOG_SPLIT_DEST_PARTITION"; + break; + case OB_LOG_TRANS_CHECKPOINT: + char_ret = "OB_LOG_TRANS_CHECKPOINT"; + break; + case OB_LOG_MAJOR_FREEZE: + char_ret = "OB_LOG_MAJOR_FREEZE"; + break; + case OB_LOG_ADD_PARTITION_TO_PG: + char_ret = "OB_LOG_ADD_PARTITION_TO_PG"; + break; + case OB_LOG_REMOVE_PARTITION_FROM_PG: + char_ret = "OB_LOG_REMOVE_PARTITION_FROM_PG"; + break; + case OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG: + char_ret = "OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG"; + break; + default: + char_ret = "OB_LOG_UNKNOWN"; + break; + } + return char_ret; +} + +int file_name_parser(const char* path, uint64_t& file_id) +{ + int ret = oceanbase::OB_SUCCESS; + + struct stat _stat; + if (OB_ISNULL(path)) { + ret = oceanbase::common::OB_INVALID_ARGUMENT; + } else if (!is_ofs_file(path) && 0 != stat(path, &_stat)) { + ret = OB_IO_ERROR; + _LOGTOOL_LOG(ERROR, "fstate:%s", strerror(errno)); + } else { + file_id = 0; + int i = 0; + int path_len = static_cast(strlen(path)); + for (--path_len; path_len >= 0 && path[path_len] >= '0' && path[path_len] <= '9'; --path_len) { + file_id += (path[path_len] - '0') * static_cast(pow(10, i++)); + } + } + return ret; +} + +const char* get_log_type(const enum ObLogType log_type) +{ + const char* char_ret = NULL; + switch (log_type) { + case OB_LOG_SUBMIT: + char_ret = "OB_LOG_SUBMIT"; + break; + case OB_LOG_MEMBERSHIP: + char_ret = "OB_LOG_MEMBERSHIP"; + break; + case OB_LOG_PREPARED: + char_ret = "OB_LOG_PREPARED"; + break; + case oceanbase::clog::OB_LOG_NOP: + char_ret = "OB_LOG_NOP"; + break; + case OB_LOG_START_MEMBERSHIP: + char_ret = "OB_LOG_START_MEMBERSHIP"; + break; + case OB_LOG_NOT_EXIST: + char_ret = "OB_LOG_NOT_EXIST"; + break; + case OB_LOG_AGGRE: + char_ret = "OB_LOG_AGGRE"; + break; + case OB_LOG_ARCHIVE_CHECKPOINT: + char_ret = "OB_LOG_ARCHIVE_CHECKPOINT"; + break; + case OB_LOG_ARCHIVE_KICKOFF: + char_ret = "OB_LOG_ARCHIVE_KICKOFF"; + break; + default: + char_ret = "OB_LOG_UNKNOWN"; + break; + } + return char_ret; +} + +const char* get_freeze_type(ObFreezeType freeze_type) +{ + const char* char_ret = NULL; + switch (freeze_type) { + case INVALID_FREEZE: + char_ret = "INVALID_FREEZE"; + break; + case MAJOR_FREEZE: + char_ret = "MAJOR_FREEZE"; + break; + case MINOR_FREEZE: + char_ret = "MINOR_FREEZE"; + break; + } + return char_ret; +} + +const char* get_row_dml_type_str(const ObRowDml& dml_type) +{ + const char* dml_str = "UNKNOWN"; + switch (dml_type) { + case T_DML_INSERT: + dml_str = "INSERT"; + break; + case T_DML_UPDATE: + dml_str = "UPDATE"; + break; + case T_DML_DELETE: + dml_str = "DELETE"; + break; + case T_DML_REPLACE: + dml_str = "REPLACE"; + break; + case T_DML_LOCK: + dml_str = "LOCK"; + break; + default: + dml_str = "UNKNOWN"; + CLOG_LOG(ERROR, "unknown dml_type", K(dml_type)); + break; + } + return dml_str; +} + +bool is_ofs_file(const char* path) +{ + UNUSED(path); + return false; +} diff --git a/tools/ob_admin/clog_tool/ob_func_utils.h b/tools/ob_admin/clog_tool/ob_func_utils.h new file mode 100644 index 0000000000..cea311cfa1 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_func_utils.h @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOG_TOOL_OB_FUN_H +#define OCEANBASE_LOG_TOOL_OB_FUN_H + +#include +#include +#include +#include +#include +#include "share/ob_define.h" +#include "common/storage/ob_freeze_define.h" +#include "common/cell/ob_cell_reader.h" +#include "clog/ob_log_entry.h" +#include "clog/ob_log_type.h" +#include "storage/transaction/ob_trans_log.h" +#include "storage/memtable/ob_memtable_mutator.h" + +using namespace oceanbase; +using namespace oceanbase::clog; +using namespace oceanbase::common; +using namespace oceanbase::transaction; +using namespace oceanbase::memtable; +using namespace oceanbase::storage; + +// get the type of clog entry +const char* get_log_type(const enum ObLogType log_type); +const char* get_submit_log_type(const int64_t submit_log_type); +const char* get_freeze_type(ObFreezeType freeze_type); + +// this func is used to parse file name +int file_name_parser(const char* path, uint64_t& file_id); +const char* get_row_dml_type_str(const ObRowDml& dml_type); + +bool is_ofs_file(const char* path); + +const int64_t LOG_FILE_MAX_SIZE = 64 * 1024 * 1024; + +#endif // OCEANBASE_LOG_TOOL_OB_FUN_H diff --git a/tools/ob_admin/clog_tool/ob_ilog_entry_parser.cpp b/tools/ob_admin/clog_tool/ob_ilog_entry_parser.cpp new file mode 100644 index 0000000000..0b90c46557 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_ilog_entry_parser.cpp @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "ob_ilog_entry_parser.h" + +namespace oceanbase { +using namespace common; +using namespace memtable; +namespace clog { + +class ObILogEntryParser::DumpIlogEntryFunctor { +public: + DumpIlogEntryFunctor(file_id_t file_id, char* buf, int64_t len) : file_id_(file_id), buf_(buf), buf_len_(len) + {} + ~DumpIlogEntryFunctor() + {} + bool operator()(const common::ObPartitionKey& partition_key, const IndexInfoBlockEntry& index_info_block_entry) + { + int ret = OB_SUCCESS; + bool bool_ret = true; + offset_t start_offset = index_info_block_entry.start_offset_; + int64_t cur_pos = start_offset; + ObLogCursorExt cursor; + // int64_t cursor_size = cursor.get_serialize_size(); + for (uint64_t log_id = index_info_block_entry.min_log_id_; + OB_SUCC(ret) && bool_ret && log_id <= index_info_block_entry.max_log_id_; + ++log_id) { + if (OB_FAIL(cursor.deserialize(buf_, buf_len_, cur_pos))) { + bool_ret = false; + } else { + fprintf(stdout, + "ilog_file_id: %d INDEX_LOG: pk:%s log_id:%lu %s||\n", + file_id_, + to_cstring(partition_key), + log_id, + to_cstring(cursor)); + } + } + return bool_ret; + } + +private: + file_id_t file_id_; + char* buf_; + int64_t buf_len_; +}; + +int ObILogEntryParser::init(file_id_t file_id, char* buf, int64_t buf_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0) || OB_UNLIKELY(OB_INVALID_FILE_ID == file_id)) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalie buf or buf_len", KP(buf), K(buf_len), K(file_id), K(ret)); + } else { + is_inited_ = true; + file_id_ = file_id; + buf_ = buf; + buf_len_ = buf_len; + } + return ret; +} + +int ObILogEntryParser::parse_all_entry() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(resolve_trailer_and_info_block_map())) { + CLOG_LOG(ERROR, "failed to resolve trailer and info block map", K(ret)); + } else if (OB_FAIL(dump_all_entry())) { + CLOG_LOG(ERROR, "failed to dump all entry", K(ret)); + } + return ret; +} + +int ObILogEntryParser::resolve_trailer_and_info_block_map() +{ + int ret = OB_SUCCESS; + int64_t pos = buf_len_ - CLOG_TRAILER_SIZE; + if (OB_FAIL(trailer_.deserialize(buf_, buf_len_, pos))) { + CLOG_LOG(ERROR, "index_info_block_map init failed", K(ret)); + } else { + CLOG_LOG(INFO, "DEBUG", K(trailer_)); + const int64_t info_block_size = upper_align(trailer_.get_info_block_size(), CLOG_DIO_ALIGN_SIZE); + const int64_t MAX_ENTRY_NUM = info_block_size / index_info_block_map_.item_size() + 1000; + int64_t local_pos = 0; + if (OB_FAIL(index_info_block_map_.init(ObModIds::OB_CLOG_INFO_BLK_HNDLR, MAX_ENTRY_NUM))) { + CLOG_LOG(ERROR, "index_info_block_map init failed", K(ret)); + } else if (OB_FAIL(index_info_block_map_.deserialize( + buf_ + trailer_.get_info_block_start_offset(), info_block_size, local_pos))) { + CLOG_LOG(ERROR, "index_info_block_map deserialize failed", K(ret)); + } else { /*do nothing*/ + } + } + return ret; +} + +int ObILogEntryParser::dump_all_entry() +{ + int ret = OB_SUCCESS; + DumpIlogEntryFunctor functor(file_id_, buf_, trailer_.get_info_block_start_offset()); + if (OB_FAIL(index_info_block_map_.for_each(functor))) { + CLOG_LOG(ERROR, "index_info_block_map_ for_each failed", K(ret)); + } + return ret; +} +} // namespace clog +} // end of namespace oceanbase diff --git a/tools/ob_admin/clog_tool/ob_ilog_entry_parser.h b/tools/ob_admin/clog_tool/ob_ilog_entry_parser.h new file mode 100644 index 0000000000..7ab9682d49 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_ilog_entry_parser.h @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_TOOL_iLOG_ENTRY_PARSER +#define OCEANBASE_TOOL_iLOG_ENTRY_PARSER + +#include "ob_func_utils.h" +#include "clog/ob_log_block.h" +#include "clog/ob_log_file_trailer.h" +#include "clog/ob_info_block_handler.h" +//#include "lib/allocator/page_arena.h" + +namespace oceanbase { +namespace clog { + +class ObILogEntryParser { +public: + ObILogEntryParser() + : is_inited_(false), file_id_(OB_INVALID_FILE_ID), buf_(NULL), buf_len_(0), trailer_(), index_info_block_map_() + {} + virtual ~ObILogEntryParser() + {} + + int init(file_id_t file_id, char* buf, int64_t buf_len); + int parse_all_entry(); + +private: + class DumpIlogEntryFunctor; + +private: + int resolve_trailer_and_info_block_map(); + int dump_all_entry(); + +private: + static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024; + bool is_inited_; + file_id_t file_id_; + char* buf_; + int64_t buf_len_; + ObIlogFileTrailerV2 trailer_; + IndexInfoBlockMap index_info_block_map_; + + DISALLOW_COPY_AND_ASSIGN(ObILogEntryParser); +}; + +} // end namespace clog +} // end namespace oceanbase + +#endif // OCEANBASE_CLOG_OB_RAW_ENTRY_ITERATOR_ diff --git a/tools/ob_admin/clog_tool/ob_log_entry_filter.cpp b/tools/ob_admin/clog_tool/ob_log_entry_filter.cpp new file mode 100644 index 0000000000..cd9aafb765 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_log_entry_filter.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include "lib/ob_errno.h" +#include "ob_log_entry_filter.h" + +namespace oceanbase { + +namespace clog { + +int ObLogEntryFilter::parse(const char* str) +{ + int ret = OB_SUCCESS; + char* ptr1 = NULL; + char* saveptr1 = NULL; + char* token1 = NULL; + char buf[1024]; + char tmp[128]; + const char* TABLE_ID_STR = "table_id"; + const char* PARTITION_ID_STR = "partition_id"; + const char* LOG_ID_STR = "log_id"; + const char* TRANS_ID_STR = "trans_id"; + if (NULL != str) { + strncpy(buf, str, sizeof(buf)); + for (ptr1 = buf;; ptr1 = NULL) { + token1 = strtok_r(ptr1, ";", &saveptr1); + if (NULL == token1) { + break; + } else { + int i = 0; + char* ptr2 = NULL; + char* saveptr2 = NULL; + char* token2 = NULL; + for (i = 1, ptr2 = token1;; ptr2 = NULL, i++) { + token2 = strtok_r(ptr2, "=", &saveptr2); + if (NULL == token2) { + break; + } else if (1 == (i % 2)) { + strncpy(tmp, token2, sizeof(tmp)); + } else { + if (0 == strcmp(tmp, TABLE_ID_STR)) { + table_id_ = atol(token2); + is_table_id_valid_ = true; + } else if (0 == strcmp(tmp, PARTITION_ID_STR)) { + partition_id_ = atol(token2); + is_partition_id_valid_ = true; + } else if (0 == strcmp(tmp, LOG_ID_STR)) { + log_id_ = atol(token2); + is_log_id_valid_ = true; + } else if (0 == strcmp(tmp, TRANS_ID_STR)) { + trans_id_ = atol(token2); + is_trans_id_valid_ = true; + } else { + // do nothing + } + } + } + } + } + } + return ret; +} + +} // end of namespace clog +} // end of namespace oceanbase diff --git a/tools/ob_admin/clog_tool/ob_log_entry_filter.h b/tools/ob_admin/clog_tool/ob_log_entry_filter.h new file mode 100644 index 0000000000..27c8cc9faf --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_log_entry_filter.h @@ -0,0 +1,91 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_ +#define OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_ + +#include +#include "lib/utility/utility.h" + +namespace oceanbase { +namespace clog { + +class ObLogEntryFilter { +public: + ObLogEntryFilter() + : table_id_(0), + partition_id_(0), + trans_id_(0), + log_id_(0), + is_table_id_valid_(false), + is_partition_id_valid_(false), + is_trans_id_valid_(false), + is_log_id_valid_(false) + {} + ~ObLogEntryFilter() + {} + int parse(const char* str); + bool is_table_id_valid() const + { + return is_table_id_valid_; + } + bool is_partition_id_valid() const + { + return is_partition_id_valid_; + } + bool is_trans_id_valid() const + { + return is_trans_id_valid_; + } + bool is_log_id_valid() const + { + return is_log_id_valid_; + } + uint64_t get_table_id() const + { + return table_id_; + } + int64_t get_partition_id() const + { + return partition_id_; + } + uint64_t get_trans_id() const + { + return trans_id_; + } + int64_t get_log_id() const + { + return log_id_; + } + bool is_valid() const + { + return is_table_id_valid_ || is_partition_id_valid_ || is_trans_id_valid_ || is_log_id_valid_; + } + +public: + TO_STRING_KV(K_(table_id), K_(partition_id), K_(trans_id), K_(log_id), K_(is_table_id_valid), + K_(is_partition_id_valid), K_(is_trans_id_valid), K_(is_log_id_valid)); + +private: + uint64_t table_id_; + int64_t partition_id_; + uint64_t trans_id_; + int64_t log_id_; + bool is_table_id_valid_; + bool is_partition_id_valid_; + bool is_trans_id_valid_; + bool is_log_id_valid_; +}; +} // namespace clog +} // end namespace oceanbase + +#endif // OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_ diff --git a/tools/ob_admin/clog_tool/ob_log_entry_parser.cpp b/tools/ob_admin/clog_tool/ob_log_entry_parser.cpp new file mode 100644 index 0000000000..d7b03ae553 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_log_entry_parser.cpp @@ -0,0 +1,2299 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "ob_log_entry_parser.h" +#include "clog/ob_info_block_handler.h" +#include "storage/memtable/ob_memtable_context.h" +#include "storage/ob_pg_log.h" +#include "share/ob_rpc_struct.h" +#include "clog/ob_log_compress.h" +#include "share/ob_encrypt_kms.h" +#include "share/ob_encryption_util.h" + +namespace oceanbase { +using namespace common; +using namespace share; +using namespace memtable; +using namespace storage; +namespace clog { + +int ObLogStat::init() +{ + int ret = OB_SUCCESS; + const int64_t HASH_BUCKET_NUM = 241; + if (OB_FAIL(compressed_tenant_ids_.create(HASH_BUCKET_NUM))) { + CLOG_LOG(ERROR, "failed to init"); + } + return ret; +} + +bool ObInfoEntryDumpFunctor::operator()(const ObPartitionKey& partition_key, const uint64_t min_log_id) +{ + int bool_ret = true; + int ret = OB_SUCCESS; + if ((!partition_key.is_valid()) || OB_INVALID_FILE_ID == file_id_) { + ret = OB_INVALID_ARGUMENT; + bool_ret = false; + CLOG_LOG(ERROR, "ObInfoEntryDumpFunctor get invalid argument", K(ret), K(partition_key), K(file_id_)); + } else { + fprintf(stdout, + "||IndexInfoBlock|file_id:%lu|partition_key:%s|min_log_id:%ld|\n", + file_id_, + to_cstring(partition_key), + min_log_id); + } + return bool_ret; +} + +int ObLogEntryParser::init(uint64_t file_id, char* buf, int64_t buf_len, const ObLogEntryFilter& filter, + const common::ObString& host, const int32_t port, const char* config_file, const bool is_ofs) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + } else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0) || OB_UNLIKELY(OB_INVALID_FILE_ID == file_id)) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid buf or buf_len", KP(buf), K(buf_len), K(file_id), K(ret)); + } else if (OB_ISNULL(compress_rbuf_.buf_ = static_cast( + allocator_.alloc_aligned(OB_MALLOC_BIG_BLOCK_SIZE, CLOG_DIO_ALIGN_SIZE)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + CLOG_LOG(WARN, "failed to allocate memory", K(ret)); + } else if (OB_FAIL(log_stat_.init())) { + CLOG_LOG(WARN, "failed to init log_stat", K(file_id)); + } else if (OB_FAIL(ObLogEntryParserImpl::init(file_id, filter, host, port, config_file))) { + CLOG_LOG(WARN, "failed to init ObLogEntryParserImpl"); + } else { + is_inited_ = true; + log_file_type_ = OB_UNKNOWN_FILE_TYPE; + buf_cur_ = buf; + buf_end_ = buf + buf_len; + is_ofs_ = is_ofs; + compress_rbuf_.buf_len_ = OB_MALLOC_BIG_BLOCK_SIZE; + } + return ret; +} + +void ObLogEntryParser::advance_(const int64_t step) +{ + cur_offset_ += static_cast(step); + buf_cur_ += step; + CLOG_LOG(DEBUG, "advance", K(step)); +} + +int ObLogEntryParser::advance_to_next_align_() +{ + int ret = OB_SUCCESS; + bool break_on_fail = true; + CLOG_LOG(INFO, "DEBUG for break", K(cur_offset_)); + const char* env_val = getenv("CLOG_TOOL_BREAK_ON_FAIL"); + if (NULL != env_val) { + break_on_fail = strcasecmp(env_val, "false") ? true : false; + } + if (break_on_fail) { + ret = OB_ITER_END; + } else { + ObLogBlockMetaV2 block_meta; + const int64_t step = SKIP_STEP - cur_offset_ % SKIP_STEP; + if ((buf_end_ - buf_cur_) < step + block_meta.get_serialize_size()) { + ret = OB_ITER_END; + } else { + advance_(step); + } + } + return ret; +} + +int ObLogEntryParser::dump_block_(const ObLogBlockMetaV2& meta) +{ + + int ret = OB_SUCCESS; + if (!meta.check_meta_checksum()) { + // fatal error, data corrupt, but go on + CLOG_LOG(ERROR, "check block meta checksum error", K(ret), K(meta)); + advance_(meta.get_serialize_size()); + } else { + if (!filter_.is_valid()) { + fprintf(stdout, + "$$$file_id:%lu offset:%ld len:%ld BlockMeta: %s|\n", + file_id_, + cur_offset_, + meta.get_serialize_size(), + to_cstring(meta)); + } + ObBlockType type = meta.get_block_type(); + switch (type) { + case OB_DATA_BLOCK: + advance_(meta.get_serialize_size()); + break; + case OB_HEADER_BLOCK: // not used any more + // file header is CLOG_DIO_ALIGN_SIZE bytes, + // skip it + advance_(meta.get_total_len()); + break; + case OB_INFO_BLOCK: { + if (OB_UNKNOWN_FILE_TYPE == log_file_type_) { + log_file_type_ = OB_CLOG_FILE_TYPE; + } + if (0 == cur_offset_) { + is_ofs_ = true; + } + advance_(meta.get_serialize_size()); + int64_t pos = 0; + if (OB_CLOG_FILE_TYPE == log_file_type_) { + ObCommitInfoBlockHandler commit_info_block_handler; + if (OB_FAIL(commit_info_block_handler.init())) { + CLOG_LOG(WARN, "failed to init clog handler", K(ret)); + } else if (OB_FAIL(commit_info_block_handler.resolve_info_block(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(WARN, "failed to resolve_info_block in clog file", K(ret)); + } else { + CLOG_LOG(INFO, "dump info block success", K(meta), K(is_ofs_)); + advance_(meta.get_data_len()); + } + } else if (OB_ILOG_FILE_TYPE == log_file_type_) { + ObIndexInfoBlockHandler index_info_block_handler; + if (OB_FAIL(index_info_block_handler.init())) { + CLOG_LOG(WARN, "failed to init clog handler", K(ret)); + } else if (OB_FAIL(index_info_block_handler.resolve_info_block(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(WARN, "failed to resolve_info_block in ilog file", K(ret)); + } else { + ObInfoEntryDumpFunctor fn(file_id_); + ObIndexInfoBlockHandler::MinLogIdInfo min_log_id_info; + if (OB_FAIL(index_info_block_handler.get_all_min_log_id_info(min_log_id_info))) { + CLOG_LOG(WARN, "get_all_min_log_id_info failed", K(ret)); + } else if (OB_FAIL(min_log_id_info.for_each(fn))) { + CLOG_LOG(ERROR, "hash_map parse fail: for_each", K(ret)); + } else { + CLOG_LOG(INFO, "index log info block print complete", K(meta)); + } + } + } else { + CLOG_LOG(ERROR, "invalid clog type while dump info block", K(log_file_type_)); + } + } + if (!is_ofs_) { + ret = OB_ITER_END; // info block is the end for local file + } + break; + case OB_TRAILER_BLOCK: + ret = OB_ITER_END; + // iter will never encounter a trailer for InfoBlock lies ahead + break; + default: + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "unknown block meta type", K(ret), K(meta), K(file_id_), K(cur_offset_)); + } + } + return ret; +} + +// if return ILOG_ENTRY, that means ilog_entry prepared well +int ObLogEntryParser::get_type_(ObCLogItemType& type) +{ + int ret = OB_SUCCESS; + type = UNKNOWN_TYPE; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log entry parser is not inited", K(ret)); + } else if (OB_ISNULL(buf_cur_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "buf cur is null", KP(buf_cur_), K(ret)); + } else if (0 == buf_end_ - buf_cur_) { + CLOG_LOG(INFO, "reach end", K(cur_offset_)); + ret = OB_ITER_END; + } else if (OB_UNLIKELY((buf_end_ - buf_cur_) < 2)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "buf not enough", K_(cur_offset), K(ret)); + } else if (OB_FAIL(clog::parse_log_item_type(buf_cur_, buf_end_ - buf_cur_, type))) { + CLOG_LOG(ERROR, "parse_log_item_type fail", K(ret), KP_(buf_cur), K_(buf_end), K(type)); + } + + return ret; +} + +int ObLogEntryParser::dump_all_entry(bool is_hex) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else { + dump_hex_ = is_hex; + while (OB_SUCC(ret)) { + if (OB_FAIL(parse_next_entry())) { + if (OB_ITER_END != ret) { + CLOG_LOG(WARN, "failed to parse_next_entry", K(ret)); + } + } + } + } + return ret; +} + +int ObLogEntryParser::format_dump_entry() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(format_dump_next_entry())) { + if (OB_ITER_END != ret) + CLOG_LOG(WARN, "failed to format_dump_entry", K(ret)); + } + } + } + return ret; +} + +int ObLogEntryParser::stat_log() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(stat_next_entry())) { + if (OB_ITER_END != ret) + CLOG_LOG(WARN, "failed to stat_log", K(ret)); + } + } + } + return ret; +} + +// return OB_EAGAIN: to prepare buffer and do get_next_entry_ again +int ObLogEntryParser::parse_next_entry() +{ + int ret = OB_SUCCESS; + ObCLogItemType item_type = UNKNOWN_TYPE; + if (OB_FAIL(get_type_(item_type)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "failed to get item type", K(ret)); + } else if (OB_ITER_END == ret) { + // reach end + } else if (ILOG_ENTRY == item_type) { + if (OB_ILOG_FILE_TYPE != log_file_type_) { + log_file_type_ = OB_ILOG_FILE_TYPE; + } + CLOG_LOG(DEBUG, "ilog entry magic", K(cur_offset_), K(item_type)); + int64_t pos = 0; + ObIndexEntry entry; + if (OB_FAIL(entry.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "ilog entry deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + // just go to next item + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + advance_(pos); + if (OB_FAIL(dump_ilog_entry(entry))) { + CLOG_LOG(WARN, "failed to dump_ilog_entry", K(ret), K(cur_offset_), KP(buf_cur_), KP(buf_end_), K(entry)); + ret = OB_SUCCESS; // skip this entry, go on to handle next entry + } + } + } else if (CLOG_ENTRY == item_type) { + if (OB_CLOG_FILE_TYPE != log_file_type_) { + log_file_type_ = OB_CLOG_FILE_TYPE; + } + CLOG_LOG(DEBUG, "clog entry magic", K(cur_offset_), K(item_type)); + int64_t pos = 0; + ObLogEntry entry; + if (OB_FAIL(entry.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "clog entry deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + // just go to next item + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (OB_FAIL(dump_clog_entry(entry, pos))) { + // fatal error, data corrupt + CLOG_LOG(WARN, "failed to dump_clog_entry", K(ret), K(cur_offset_), KP(buf_cur_), KP(buf_end_), K(entry)); + ret = OB_SUCCESS; // skip this entry, go on to handle next entry + } + advance_(pos); + } + } else if (EOF_BUF == item_type) { + CLOG_LOG(DEBUG, "eof magic", K(cur_offset_)); + // pointing to start of EOF + ret = OB_ITER_END; + CLOG_LOG(INFO, "reach eof", K(file_id_), K(cur_offset_)); + } else if (BLOCK_META == item_type) { + CLOG_LOG(DEBUG, "block magic", K(cur_offset_)); + ObLogBlockMetaV2 meta; + int64_t pos = 0; + if (OB_FAIL(meta.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "log block meta deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else if (OB_FAIL(dump_block_(meta))) { + if (OB_ITER_END != ret) { + CLOG_LOG(WARN, "dump block error", K(ret), K(meta), K(file_id_), K(cur_offset_), KP(buf_cur_), KP(buf_end_)); + } + } + } else if (PADDING_ENTRY == item_type) { + CLOG_LOG(DEBUG, "padding entry magic", K(cur_offset_)); + // data written by new writer + ObPaddingEntry pe; + int64_t pos = 0; + if (OB_FAIL(pe.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, "padding entry deserialize error"); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (!filter_.is_valid()) { + fprintf(stdout, "Padding: %s|\n", to_cstring(pe)); + } + // skip padding entry + advance_(pe.get_entry_size()); + } + } else if (CLOG_ENTRY_COMPRESSED_ZSTD == item_type || CLOG_ENTRY_COMPRESSED_LZ4 == item_type || + CLOG_ENTRY_COMPRESSED_ZSTD_138 == item_type) { + int64_t local_pos = 0; + int64_t uncompress_len = 0; + int64_t pos = 0; + ObLogEntry entry; + if (OB_FAIL(clog::uncompress( + buf_cur_, buf_end_ - buf_cur_, compress_rbuf_.buf_, compress_rbuf_.buf_len_, uncompress_len, pos))) { + CLOG_LOG(ERROR, "failed to uncompress clog entry", K(ret), K(item_type)); + } else if (OB_FAIL(entry.deserialize(compress_rbuf_.buf_, uncompress_len, local_pos))) { + CLOG_LOG(ERROR, "failed to deserialize clog entry", K(ret)); + } + if (OB_FAIL(ret)) { + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (OB_FAIL(dump_clog_entry(entry, pos))) { + CLOG_LOG(WARN, "failed to dump_clog_entry", K(ret), K(entry)); + ret = OB_SUCCESS; + } + advance_(pos); + } + } else { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "error unexpected", K(ret), K(item_type)); + } + return ret; +} + +int ObLogEntryParserImpl::init(const int64_t file_id, const ObLogEntryFilter& filter, const common::ObString& host, + const int32_t port, const char* config_file) +{ + int ret = OB_SUCCESS; + UNUSED(config_file); + if (OB_UNLIKELY(file_id <= 0)) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_UNLIKELY(NULL == (print_buf_ = static_cast(allocator_.alloc(PRINT_BUF_SIZE))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + CLOG_LOG(WARN, "failed to allocate memory", K(ret)); + } else { + file_id_ = file_id; + cur_offset_ = 0; + filter_ = filter; + is_inited_ = true; + } + if (!host.empty() && port > 0) { + if (OB_SUCCESS != client_.init()) { + CLOG_LOG(WARN, "failed to init net client", K(ret)); + } else if (OB_SUCCESS != client_.get_proxy(rpc_proxy_)) { + CLOG_LOG(WARN, "failed to get_proxy", K(ret)); + } else { + host_addr_.set_ip_addr(host, port); + rpc_proxy_.set_server(host_addr_); + } + } + return ret; +} + +int ObLogEntryParser::format_dump_next_entry() +{ + int ret = OB_SUCCESS; + ObCLogItemType item_type = UNKNOWN_TYPE; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else if (OB_FAIL(get_type_(item_type)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "failed to get item type", K(ret)); + } else if (OB_ITER_END == ret) { + // reach end + } else if (ILOG_ENTRY == item_type) { + ret = OB_ITER_END; + CLOG_LOG(ERROR, "should be clog file rather than ilog file", K(ret)); + } else if (CLOG_ENTRY == item_type) { + if (OB_CLOG_FILE_TYPE != log_file_type_) { + log_file_type_ = OB_CLOG_FILE_TYPE; + } + CLOG_LOG(DEBUG, "clog entry magic", K(cur_offset_), K(item_type)); + int64_t pos = 0; + ObLogEntry entry; + if (OB_FAIL(entry.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "entry deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (OB_FAIL(format_dump_clog_entry(entry))) { + // fatal error, data corrupt + CLOG_LOG(ERROR, "failed to dump_clog_entry", K(ret), K(cur_offset_), KP(buf_cur_), KP(buf_end_), K(entry)); + ret = OB_SUCCESS; // skip this entry, go on to handle next entry + } else { /*do nothing*/ + } + advance_(pos); + } + } else if (EOF_BUF == item_type) { + CLOG_LOG(DEBUG, "eof magic", K(cur_offset_)); + // pointing to start of EOF + ret = OB_ITER_END; + CLOG_LOG(INFO, "reach eof", K(file_id_), K(cur_offset_)); // FIXME + } else if (BLOCK_META == item_type) { + CLOG_LOG(DEBUG, "block magic", K(cur_offset_)); + ObLogBlockMetaV2 meta; + int64_t pos = 0; + if (OB_FAIL(meta.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "log block meta deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else if (OB_FAIL(skip_block_(meta))) { + if (OB_ITER_END != ret) { + CLOG_LOG(WARN, "dump block error", K(ret), K(meta), K(file_id_), K(cur_offset_), KP(buf_cur_), KP(buf_end_)); + } + } + } else if (PADDING_ENTRY == item_type) { + CLOG_LOG(DEBUG, "padding entry magic", K(cur_offset_)); + // data written by new writer + ObPaddingEntry pe; + int64_t pos = 0; + if (OB_FAIL(pe.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, "padding entry deserialize error"); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + // skip padding entry + advance_(pe.get_entry_size()); + } + } else { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "error unexpected", K(ret), K(item_type)); + } + return ret; +} + +int ObLogEntryParser::stat_next_entry() +{ + int ret = OB_SUCCESS; + ObCLogItemType item_type = UNKNOWN_TYPE; + if (OB_FAIL(get_type_(item_type)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "failed to get item type", K(ret)); + } else if (OB_ITER_END == ret) { + // reach end + } else if (ILOG_ENTRY == item_type) { + ret = OB_ITER_END; + // fprintf(stdout, "invalid argument is ilog file, stat only support clog file"); + } else if (CLOG_ENTRY == item_type) { + if (OB_CLOG_FILE_TYPE != log_file_type_) { + log_file_type_ = OB_CLOG_FILE_TYPE; + } + CLOG_LOG(DEBUG, "clog entry magic", K(cur_offset_), K(item_type)); + int64_t pos = 0; + ObLogEntry entry; + if (OB_FAIL(entry.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "entry deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (OB_FAIL(stat_clog_entry(entry, pos, false /*is_compressed=false*/))) { + CLOG_LOG(ERROR, "fail to stat_clog_entry", K(ret)); + } + advance_(pos); + } + } else if (EOF_BUF == item_type) { + CLOG_LOG(DEBUG, "eof magic", K(cur_offset_)); + // pointing to start of EOF + ret = OB_ITER_END; + CLOG_LOG(INFO, "reach eof", K(file_id_), K(cur_offset_), K(ret)); + } else if (BLOCK_META == item_type) { + CLOG_LOG(DEBUG, "block magic", K(cur_offset_)); + ObLogBlockMetaV2 meta; + int64_t pos = 0; + if (OB_FAIL(meta.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, + "log block meta deserialize error, maybe corrupted", + K(ret), + K(item_type), + KP(buf_cur_), + KP(buf_end_), + K(pos), + K(cur_offset_), + K(file_id_)); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else if (OB_FAIL(stat_block_(meta))) { + if (OB_ITER_END != ret) { + CLOG_LOG(WARN, "dump block error", K(ret), K(meta), K(file_id_), K(cur_offset_), KP(buf_cur_), KP(buf_end_)); + } + } + } else if (PADDING_ENTRY == item_type) { + CLOG_LOG(DEBUG, "padding entry magic", K(cur_offset_)); + // data written by new writer + ObPaddingEntry pe; + int64_t pos = 0; + if (OB_FAIL(pe.deserialize(buf_cur_, buf_end_ - buf_cur_, pos))) { + CLOG_LOG(ERROR, "padding entry deserialize error"); + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + // skip padding entry + advance_(pe.get_entry_size()); + log_stat_.padding_size_ += pe.get_entry_size(); + } + } else if (CLOG_ENTRY_COMPRESSED_ZSTD == item_type || CLOG_ENTRY_COMPRESSED_LZ4 == item_type || + CLOG_ENTRY_COMPRESSED_ZSTD_138 == item_type) { + int64_t local_pos = 0; + int64_t uncompress_len = 0; + int64_t pos = 0; + ObLogEntry entry; + if (OB_FAIL(clog::uncompress( + buf_cur_, buf_end_ - buf_cur_, compress_rbuf_.buf_, compress_rbuf_.buf_len_, uncompress_len, pos))) { + CLOG_LOG(ERROR, "failed to uncompress clog entry", K(ret), K(item_type)); + } else if (OB_FAIL(entry.deserialize(compress_rbuf_.buf_, uncompress_len, local_pos))) { + CLOG_LOG(ERROR, "failed to deserialize clog entry", K(ret)); + } + if (OB_FAIL(ret)) { + if (OB_FAIL(advance_to_next_align_())) { + if (OB_ITER_END != ret) { + CLOG_LOG(ERROR, "unexpected error", K(ret)); + } + } + } else { + if (OB_FAIL(stat_clog_entry(entry, pos, true /*is_compressed=true*/))) { + CLOG_LOG(WARN, "failed to stat_clog_entry", K(ret), K(entry)); + ret = OB_SUCCESS; + } + advance_(pos); + } + } else { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "error unexpected", K(ret), K(item_type)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_memtable_mutator(const char* buf, int64_t len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObMemtableMutatorIterator mmi; + ObMemtableMutatorMeta meta; + int64_t meta_pos = 0; + if (OB_FAIL(meta.deserialize(buf, len, meta_pos))) { + CLOG_LOG(ERROR, "meta.deserialize fail", K(ret)); + } else if (ObTransRowFlag::is_normal_row(meta.get_flags())) { + if (OB_FAIL(mmi.deserialize(buf, len, pos))) { + CLOG_LOG(ERROR, "mmi.deserialize fail", K(ret)); + } else { + fprintf(stdout, " %s", to_cstring(mmi.get_meta())); + } + + while (OB_SUCCESS == ret) { + ObMemtableMutatorRow row; + if (OB_FAIL(mmi.get_next_row(row)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "mmi.get_next_row fail", K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + if (dump_hex_) { + int64_t pos = row.rowkey_.to_smart_string(print_buf_, PRINT_BUF_SIZE - 1); + print_buf_[pos] = '\0'; + fprintf(stdout, " MutatorRow={%s} HexedRowkey={%s} | OLD_ROW={", to_cstring(row), print_buf_); + } else { + fprintf(stdout, " MutatorRow={%s} | OLD_ROW={", to_cstring(row)); + } + ObCellReader new_cci; + ObCellReader old_cci; + if (NULL != row.old_row_.data_ && row.old_row_.size_ > 0) { + if (OB_FAIL(old_cci.init(row.old_row_.data_, row.old_row_.size_, SPARSE))) { + CLOG_LOG(ERROR, "old cci.init fail", K(ret)); + } + while (OB_SUCC(ret) && OB_SUCC(old_cci.next_cell())) { + uint64_t column_id = OB_INVALID_ID; + const ObObj* value = NULL; + if (OB_FAIL(old_cci.get_cell(column_id, value))) { + CLOG_LOG(ERROR, "failed to get cell", K(ret)); + } else if (NULL == value) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "got value is NULL", K(ret)); + } else if (ObExtendType == value->get_type() && ObActionFlag::OP_END_FLAG == value->get_ext()) { + ret = OB_ITER_END; + } else if (OB_FAIL(dump_obj(*value, column_id))) { + CLOG_LOG(ERROR, "failed to dump obj", K(*value), K(ret)); + } else { /*do nothing*/ + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + } else if (NULL == row.old_row_.data_ && 0 == row.old_row_.size_) { + // no data of old row + } else { + CLOG_LOG(ERROR, "old cci.init fail", K(row), K(ret)); + } + + fprintf(stdout, "} | NEW_ROW={"); + if (OB_SUCC(ret) && OB_FAIL(new_cci.init(row.new_row_.data_, row.new_row_.size_, SPARSE))) { + CLOG_LOG(ERROR, "new cci.init fail", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(new_cci.next_cell())) { + uint64_t column_id = OB_INVALID_ID; + const ObObj* value = NULL; + if (OB_FAIL(new_cci.get_cell(column_id, value))) { + } else if (NULL == value) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "value is NULL", K(ret)); + } else if (ObExtendType == value->get_type() && ObActionFlag::OP_END_FLAG == value->get_ext()) { + ret = OB_ITER_END; + } else if (OB_FAIL(dump_obj(*value, column_id))) { + CLOG_LOG(ERROR, "failed to dump obj", K(*value), K(ret)); + } else { /*do nothing*/ + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + } + fprintf(stdout, "}"); + } + } + } else if (ObTransRowFlag::is_big_row_start(meta.get_flags())) { + fprintf(stdout, "BIG_ROW_START: %s", to_cstring(meta)); + } else if (ObTransRowFlag::is_big_row_mid(meta.get_flags())) { + fprintf(stdout, "BIG_ROW_MID: %s", to_cstring(meta)); + } else if (ObTransRowFlag::is_big_row_end(meta.get_flags())) { + fprintf(stdout, "BIG_ROW_END: %s", to_cstring(meta)); + } + + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + return ret; +} + +int ObLogEntryParserImpl::dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransRedoLogHelper helper; + ObSpTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + fprintf(stdout, " %s|||", to_cstring(log)); + dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + fprintf(stdout, " %s|||", to_cstring(log)); + dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_prepare_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObTransPrepareLogHelper helper; + ObTransPrepareLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransCommitLogHelper helper; + ObSpTransCommitLog log(helper); + int64_t pos = 0; + // an sp commit log may contain redo data + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "sp redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + fprintf(stdout, " %s|||", to_cstring(log)); + dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_commit_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + PartitionLogInfoArray partition_log_info_array; + ObTransCommitLog log(partition_log_info_array); + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_sp_trans_abort_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObSpTransAbortLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "sp abort log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_abort_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObTransAbortLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_clear_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObTransClearLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "clear log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_prepare_commit_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + + ObTransPrepareLogHelper helper; + ObTransPrepareLog p_log(helper); + PartitionLogInfoArray partition_log_info_array; + ObTransCommitLog c_log(partition_log_info_array); + if (OB_FAIL(p_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(c_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(p_log)); + fprintf(stdout, " %s|||", to_cstring(c_log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + int64_t pos = 0; + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key()) || ret == OB_EAGAIN) { + if (OB_SUCC(ret)) { + fprintf(stdout, " %s|||", to_cstring(r_log)); + dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else { + // impossible branch + } + ObTransPrepareLogHelper helper; + ObTransPrepareLog p_log(helper); + if (OB_FAIL(p_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(p_log)); + } + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_redo_prepare_commit_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserializde log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key()) || ret == OB_EAGAIN) { + if (OB_SUCC(ret)) { + fprintf(stdout, " %s|||", to_cstring(r_log)); + dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else { + // impossible branch + } + ObTransPrepareLogHelper helper; + ObTransPrepareLog p_log(helper); + PartitionLogInfoArray partition_log_info_array; + ObTransCommitLog c_log(partition_log_info_array); + if (OB_FAIL(p_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(c_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(p_log)); + fprintf(stdout, " %s|||", to_cstring(c_log)); + } + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_prepare_commit_clear_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransPrepareLogHelper helper; + ObTransPrepareLog p_log(helper); + PartitionLogInfoArray partition_log_info_array; + ObTransCommitLog c_log(partition_log_info_array); + ObTransClearLog cl_log; + if (OB_FAIL(p_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(c_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(cl_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "clear log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(p_log)); + fprintf(stdout, " %s|||", to_cstring(c_log)); + fprintf(stdout, " %s|||", to_cstring(cl_log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_redo_prepare_commit_clear_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key()) || ret == OB_EAGAIN) { + if (OB_SUCC(ret)) { + fprintf(stdout, " %s|||", to_cstring(r_log)); + dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else { + // impossible branch + } + ObTransPrepareLogHelper helper; + ObTransPrepareLog p_log(helper); + PartitionLogInfoArray partition_log_info_array; + ObTransCommitLog c_log(partition_log_info_array); + ObTransClearLog cl_log; + if (OB_FAIL(p_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "prepare log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(c_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "commit log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else if (OB_FAIL(cl_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "clear log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(p_log)); + fprintf(stdout, " %s|||", to_cstring(c_log)); + fprintf(stdout, " %s|||", to_cstring(cl_log)); + } + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_mutator_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransMutatorLogHelper helper; + ObTransMutatorLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "trans mutator log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + fprintf(stdout, " %s|||", to_cstring(log)); + dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_mutator_abort_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObTransMutatorAbortLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_state_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObTransStateLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_mutator_state_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransMutatorLogHelper helper; + ObTransMutatorLog m_log(helper); + int64_t pos = 0; + if (OB_FAIL(m_log.init_for_deserialize())) { + CLOG_LOG(WARN, "trans mutator log init for deserialize fail", K(ret)); + } else if (OB_FAIL(m_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(m_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(m_log.decrypt_table_key()) || ret == OB_EAGAIN) { + if (OB_SUCC(ret)) { + fprintf(stdout, " %s|||", to_cstring(m_log)); + dump_memtable_mutator(m_log.get_mutator().get_data(), m_log.get_mutator().get_position()); + } else { + // impossible branch + } + ObTransStateLog s_log; + if (OB_FAIL(s_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(s_log)); + } + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_part_split_src_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObPartitionSplitSourceLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_part_split_dest_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObPartitionSplitDestLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_checkpoint_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObCheckpointLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_partition_schema_version_change_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObPGSchemaChangeLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "ObPGSchemaChangeLog deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " OB_LOG_PARTITION_SCHEMA_VERSION_CHANGE %s|||", to_cstring(log)); + } + return ret; +} +int ObLogEntryParserImpl::dump_new_offline_partition_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObOfflinePartitionLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "ObOfflinePartitionLog log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " OB_LOG_OFFLINE_PARTITION_LOG_V2 %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_add_partition_to_pg_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObAddPartitionToPGLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "ObAddPartitionToPGLog log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " OB_LOG_ADD_PARTITION_TO_PG %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_remove_partition_from_pg_log(const char* data, int64_t len) +{ + int ret = OB_SUCCESS; + ObRemovePartitionFromPGLog log; + int64_t pos = 0; + if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "ObRemovePartitionFroPGLog log deserialize error", K(ret), KP(data), K(len), K(pos)); + } else { + fprintf(stdout, " OB_LOG_REMOVE_PARTITION_FROM_PG %s|||", to_cstring(log)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_clog_entry(ObLogEntry& entry, int64_t pos) +{ + int ret = OB_SUCCESS; + bool need_print = false; + ObLogEntry* entry_ptr = &entry; + ObLogEntry decrypted_entry; + if (!entry.check_integrity(true /*ignore_batch_commited*/)) { + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "entry.check_integrity() fail", K(entry), K(ret)); + } else if (OB_FAIL(check_filter(entry, need_print))) { + CLOG_LOG(ERROR, "check filter failed", K(ret), K(entry)); + } else if (!need_print) { + CLOG_LOG(DEBUG, "ignore log", K(entry)); + } else { + if (OB_SUCC(ret)) { + ret = dump_clog_entry_(*entry_ptr, pos); + } + } + return ret; +} + +int ObLogEntryParserImpl::dump_clog_entry_(ObLogEntry& entry, int64_t pos) +{ + int ret = OB_SUCCESS; + clog::ObLogType log_type = entry.get_header().get_log_type(); + fprintf(stdout, + "$$$file_id:%lu offset:%ld len:%ld %s|log_id:%lu|%s |||", + file_id_, + cur_offset_, + pos, + get_log_type(log_type), + entry.get_header().get_log_id(), + to_cstring(entry)); + if (OB_LOG_MEMBERSHIP == log_type) { + int64_t pos = 0; + ObMembershipLog member_log; + if (OB_FAIL(member_log.deserialize(entry.get_buf(), entry.get_header().get_data_len(), pos))) { + CLOG_LOG(ERROR, "failed to deserialize member_log", K(entry), K(ret)); + } else { + fprintf(stdout, "||ObMembershipLog:%s", to_cstring(member_log)); + } + } else if (OB_LOG_RENEW_MEMBERSHIP == log_type) { + int64_t pos = 0; + ObRenewMembershipLog renew_ms_log; + if (OB_FAIL(renew_ms_log.deserialize(entry.get_buf(), entry.get_header().get_data_len(), pos))) { + CLOG_LOG(ERROR, "failed to deserialize ObRenewMembershipLog", K(entry), K(ret)); + } else { + fprintf(stdout, "||ObRenewMembershipLog:%s", to_cstring(renew_ms_log)); + } + } else if (OB_LOG_SUBMIT == log_type) { + int64_t trans_inc = -1; + int64_t submit_log_type_tmp = -1; + const char* buf = entry.get_buf(); + int64_t buf_len = entry.get_header().get_data_len(); + int64_t pos = 0; + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &submit_log_type_tmp))) { + CLOG_LOG(ERROR, "failed to decode submit_log_type", K(entry), K(ret)); + } else { + ObStorageLogType submit_log_type = static_cast(submit_log_type_tmp); + const uint64_t real_tenant_id = entry.get_header().get_partition_key().get_tenant_id(); + if (ObStorageLogTypeChecker::is_trans_log(submit_log_type)) { + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &trans_inc))) { + CLOG_LOG(ERROR, "failed to decode trans_inc", K(ret), KP(buf), K(pos), K(buf_len)); + } else if (OB_FAIL(dump_trans_log(submit_log_type, trans_inc, real_tenant_id, buf, buf_len, pos))) { + CLOG_LOG(ERROR, "failed to dump_trans_log", K(ret), KP(buf), K(pos), K(buf_len)); + } + } else if (submit_log_type == OB_LOG_MAJOR_FREEZE) { + fprintf(stdout, "\t|||freeze log type:%s ", get_submit_log_type(submit_log_type)); + ObStorageLogType log_type = storage::OB_LOG_UNKNOWN; + ObFreezeType freeze_type = INVALID_FREEZE; + ObPartitionKey partition_key; + ObSavedStorageInfo info; + int64_t frozen_version = -1; + if (OB_FAIL(dump_freeze_log(buf, buf_len, log_type, freeze_type, partition_key, frozen_version, info))) { + CLOG_LOG(ERROR, "failed to dump freeze log", K(ret)); + } else { + fprintf(stdout, + "|FreezeType:%s|frozen_version:%ld|frozen_timestamp:%s|schema_version:%ld|saved_storage_info:%s |||", + get_freeze_type(freeze_type), + frozen_version, + time2str(info.get_frozen_timestamp()), + info.get_schema_version(), + to_cstring(info)); + } + } else if (OB_LOG_OFFLINE_PARTITION == submit_log_type) { + fprintf(stdout, "|| OB_LOG_OFFLINE_PARTITION"); + } else if (OB_LOG_OFFLINE_PARTITION_V2 == submit_log_type) { + if (OB_FAIL(dump_new_offline_partition_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_new_offline_partition_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + } else if (OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG == submit_log_type) { + if (OB_FAIL(dump_partition_schema_version_change_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump partition schema change log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + } else if (OB_LOG_ADD_PARTITION_TO_PG == submit_log_type) { + if (OB_FAIL(dump_add_partition_to_pg_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_add_partition_to_pg_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + } else if (OB_LOG_REMOVE_PARTITION_FROM_PG == submit_log_type) { + if (OB_FAIL(dump_remove_partition_from_pg_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_remove_partition_from_pg_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + } else if (ObStorageLogTypeChecker::is_split_log(submit_log_type)) { + switch (submit_log_type) { + case OB_LOG_SPLIT_SOURCE_PARTITION: { + if (OB_FAIL(dump_part_split_src_log(buf + pos, buf_len - pos))) { + CLOG_LOG(ERROR, "dump_part_split_src_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SPLIT_DEST_PARTITION: { + if (OB_FAIL(dump_part_split_dest_log(buf + pos, buf_len - pos))) { + CLOG_LOG(ERROR, "dump_part_split_dest_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + default: { + CLOG_LOG(ERROR, "unknown log type", K(submit_log_type)); + break; + } + } + } else if (submit_log_type == OB_LOG_TRANS_CHECKPOINT) { + if (OB_FAIL(dump_trans_checkpoint_log(buf + pos, buf_len - pos))) { + CLOG_LOG(ERROR, "dump_trans_checkpoint_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + } else { /*todo may be other logs need be printed?*/ + } + } + } else if (OB_LOG_AGGRE == log_type) { + const char* log_buf = entry.get_buf(); + int64_t buf_len = entry.get_header().get_data_len(); + int32_t next_log_offset = 0; + const uint64_t real_tenant_id = entry.get_header().get_partition_key().get_tenant_id(); + while (OB_SUCC(ret) && next_log_offset < buf_len) { + int64_t saved_pos = next_log_offset; + int64_t pos = next_log_offset; + int64_t log_type_in_buf = storage::OB_LOG_UNKNOWN; + ObStorageLogType log_type = storage::OB_LOG_UNKNOWN; + int64_t trans_inc = 0; + int64_t submit_timestamp = 0; + if (OB_FAIL(serialization::decode_i32(log_buf, buf_len, pos, &next_log_offset))) { + REPLAY_LOG(ERROR, "serialization decode_i32 failed", K(ret)); + } else if (OB_FAIL(serialization::decode_i64(log_buf, buf_len, pos, &submit_timestamp))) { + REPLAY_LOG(ERROR, "serialization::decode_i64 failed", K(ret)); + } else if (OB_FAIL(serialization::decode_i64(log_buf, buf_len, pos, &log_type_in_buf))) { + REPLAY_LOG(ERROR, "serialization::decode_i64 failed", K(ret)); + } else if (OB_FAIL(serialization::decode_i64(log_buf, buf_len, pos, &trans_inc))) { + REPLAY_LOG(ERROR, "serialization::decode_i64 failed", K(ret)); + } else { + const int64_t buf_len = next_log_offset - saved_pos - AGGRE_LOG_RESERVED_SIZE; + log_type = static_cast(log_type_in_buf); + const char* buf = log_buf + saved_pos + AGGRE_LOG_RESERVED_SIZE; + int64_t pos = 16; + if (ObStorageLogTypeChecker::is_trans_log(log_type)) { + if (OB_FAIL(dump_trans_log(log_type, trans_inc, real_tenant_id, buf, buf_len, pos))) { + CLOG_LOG(ERROR, "failed to dump_trans_log", K(entry), K(ret), KP(buf), K(pos), K(buf_len)); + } + } + } + } + } else if (OB_LOG_ARCHIVE_CHECKPOINT == log_type || OB_LOG_ARCHIVE_KICKOFF == log_type) { + int64_t pos = 0; + ObLogArchiveInnerLog log; + if (OB_FAIL(log.deserialize(entry.get_buf(), entry.get_header().get_data_len(), pos))) { + CLOG_LOG(ERROR, "failed to deserialize archive_inner_log", K(entry), K(ret)); + } else { + fprintf(stdout, "|| ObArchiveInnerLog:%s", to_cstring(log)); + } + } else { /*todo may be other logs need be printed?*/ + } + fprintf(stdout, "\n"); + return ret; +} + +int ObLogEntryParser::dump_ilog_entry(ObIndexEntry& entry) +{ + int ret = OB_SUCCESS; + if (!entry.check_integrity()) { + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "entry.check_integrity() fail", K(entry), K(ret)); + } else { + fprintf(stdout, "INDEX_LOG: %s||\n", to_cstring(entry)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_freeze_log(const char* buf, const int64_t buf_len, ObStorageLogType& log_type, + ObFreezeType& freeze_type, ObPartitionKey& pkey, int64_t& frozen_version, ObSavedStorageInfo& info) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + log_type = storage::OB_LOG_UNKNOWN; + freeze_type = INVALID_FREEZE; + int64_t local_log_type = storage::OB_LOG_UNKNOWN; + int64_t local_freeze_type = INVALID_FREEZE; + if (OB_ISNULL(buf) || buf_len <= 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "invalid argument", KP(buf), K(buf_len), K(ret)); + } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &local_log_type))) { + CLOG_LOG(ERROR, "deserialize log_type error", K(buf_len), K(pos), K(ret)); + } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &local_freeze_type))) { + CLOG_LOG(ERROR, "deserialize freeze_type error", K(buf_len), K(pos), K(ret)); + } else if (OB_FAIL(pkey.deserialize(buf, buf_len, pos))) { + CLOG_LOG(ERROR, "deserialize partition key error", K(buf_len), K(pos), K(ret)); + } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &frozen_version))) { + CLOG_LOG(ERROR, "deserialize frozen version error", K(buf_len), K(pos), K(ret)); + } else if (OB_FAIL(info.deserialize(buf, buf_len, pos))) { + CLOG_LOG(ERROR, "deserialize saved storage info error", K(buf_len), K(pos), K(ret)); + } else { + log_type = static_cast(local_log_type); + freeze_type = static_cast(local_freeze_type); + } + return ret; +} + +int ObLogEntryParserImpl::dump_obj(const common::ObObj& obj, uint64_t column_id) +{ + int ret = OB_SUCCESS; + if (dump_hex_) { + int64_t buf_len = PRINT_BUF_SIZE - 1; // reserver one character for '\0' + int64_t pos = 0; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else if (OB_FAIL(obj.print_smart(print_buf_, buf_len, pos))) { + CLOG_LOG(ERROR, "failed to print smart obj", K(column_id), K(obj), K(ret)); + } else if (pos >= PRINT_BUF_SIZE - 1) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "UNEXPECTED pos", K(pos), K(buf_len), K(ret)); + } else { + print_buf_[pos] = '\0'; + fprintf(stdout, " %ld:%s", column_id, print_buf_); + } + } else { + fprintf(stdout, " %ld:%s", column_id, to_cstring(obj)); + } + return ret; +} + +int ObLogEntryParser::skip_block_(const ObLogBlockMetaV2& meta) +{ + + int ret = OB_SUCCESS; + if (!meta.check_meta_checksum()) { + // fatal error, data corrupt + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "check block meta checksum error", K(ret), K(meta)); + } else { + ObBlockType type = meta.get_block_type(); + switch (type) { + case OB_DATA_BLOCK: + advance_(meta.get_serialize_size()); + break; + case OB_HEADER_BLOCK: // not used any more + // file header is CLOG_DIO_ALIGN_SIZE bytes, + // skip it + advance_(meta.get_total_len()); + break; + case OB_INFO_BLOCK: + if (is_ofs_) { + advance_(meta.get_serialize_size() + meta.get_data_len()); + } else { + ret = OB_ITER_END; // info block is the end for local file + } + break; + case OB_TRAILER_BLOCK: + ret = OB_ITER_END; // trailer block is the end + break; + default: + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "unknown block meta type", K(ret), K(meta), K(file_id_), K(cur_offset_)); + } + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_clog_entry(ObLogEntry& entry) +{ + int ret = OB_SUCCESS; + if (!entry.check_integrity()) { + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "entry.check_integrity() fail", K(entry), K(ret)); + } else { + // only handle redo log + if (OB_LOG_SUBMIT == entry.get_header().get_log_type()) { + int64_t trans_inc = -1; + int64_t submit_log_type_tmp = -1; + const char* buf = entry.get_buf(); + int64_t buf_len = entry.get_header().get_data_len(); + int64_t pos = 0; + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &submit_log_type_tmp))) { + CLOG_LOG(ERROR, "failed to decode submit_log_type", K(entry), K(ret)); + } else { + int submit_log_type = static_cast(submit_log_type_tmp); + const uint64_t real_tenant_id = entry.get_header().get_partition_key().get_tenant_id(); + if (((OB_LOG_SP_TRANS_REDO & submit_log_type) || (OB_LOG_TRANS_REDO & submit_log_type) || + OB_LOG_SP_ELR_TRANS_COMMIT == submit_log_type || OB_LOG_SP_TRANS_COMMIT == submit_log_type) && + submit_log_type < OB_LOG_TRANS_MAX) { + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &trans_inc))) { + CLOG_LOG(ERROR, "failed to decode trans_inc", K(ret), KP(buf), K(pos), K(buf_len)); + } else { + switch (submit_log_type) { + case OB_LOG_SP_TRANS_REDO: { + if (OB_FAIL(format_dump_sp_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "format_dump_sp_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_COMMIT: + case OB_LOG_SP_ELR_TRANS_COMMIT: { + if (OB_FAIL(format_dump_sp_trans_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "format_dump_sp_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO: { + if (OB_FAIL(format_dump_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "format_dump_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE: { + if (OB_FAIL(format_dump_trans_redo_prepare_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "format_dump_trans_redo_prepare_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT: { + if (OB_FAIL(format_dump_trans_redo_prepare_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "format_dump_trans_redo_prepare_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR: { + if (OB_FAIL( + format_dump_trans_redo_prepare_commit_clear_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG( + WARN, "format_dump_trans_redo_prepare_commit_clear_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + default: { + CLOG_LOG(ERROR, "unknown log type", K(submit_log_type)); + break; + } + } + } + } + } + } + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransRedoLogHelper helper; + ObSpTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + cur_trans_id_ = log.get_trans_id(); + format_dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransCommitLogHelper helper; + ObSpTransCommitLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + cur_trans_id_ = log.get_trans_id(); + format_dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + cur_trans_id_ = log.get_trans_id(); + format_dump_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_trans_redo_prepare_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + int64_t pos = 0; + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + cur_trans_id_ = r_log.get_trans_id(); + format_dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_trans_redo_prepare_commit_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + cur_trans_id_ = r_log.get_trans_id(); + format_dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_trans_redo_prepare_commit_clear_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + cur_trans_id_ = r_log.get_trans_id(); + format_dump_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_memtable_mutator(const char* buf, int64_t len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObMemtableMutatorIterator mmi; + ObMemtableMutatorMeta meta; + int64_t meta_pos = 0; + if (OB_FAIL(meta.deserialize(buf, len, meta_pos))) { + CLOG_LOG(ERROR, "meta.deserialize fail", K(ret)); + } else if (ObTransRowFlag::is_normal_row(meta.get_flags())) { + if (OB_FAIL(mmi.deserialize(buf, len, pos))) { + CLOG_LOG(ERROR, "mmi.deserialize fail", K(ret)); + } + while (OB_SUCCESS == ret) { + ObMemtableMutatorRow row; + if (OB_FAIL(mmi.get_next_row(row)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "mmi.get_next_row fail", K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + if (T_DML_LOCK == row.dml_type_) { + // do nothing + } else { + fprintf(stdout, "%s\t", to_cstring(cur_trans_id_)); + fprintf(stdout, "%s\t", get_row_dml_type_str(row.dml_type_)); + fprintf(stdout, "%lu\t", row.table_id_); + int64_t pos = row.rowkey_.to_format_string(print_buf_, PRINT_BUF_SIZE - 1); + print_buf_[pos] = '\0'; + fprintf(stdout, "%s\t", print_buf_); + + ObCellReader old_cci; + if (NULL != row.old_row_.data_ && row.old_row_.size_ > 0) { + if (OB_FAIL(old_cci.init(row.old_row_.data_, row.old_row_.size_, SPARSE))) { + CLOG_LOG(ERROR, "old cci.init fail", K(ret)); + } + int64_t index = 0; + while (OB_SUCC(ret) && OB_SUCC(old_cci.next_cell())) { + uint64_t column_id = OB_INVALID_ID; + const ObObj* value = NULL; + if (OB_FAIL(old_cci.get_cell(column_id, value))) { + CLOG_LOG(ERROR, "failed to get cell", K(ret)); + } else if (NULL == value) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "got value is NULL", K(ret)); + } else if (ObExtendType == value->get_type() && ObActionFlag::OP_END_FLAG == value->get_ext()) { + ret = OB_ITER_END; + } else { + if (index > 0) { + fprintf(stdout, " "); // separate columns + } + if (OB_SUCC(ret) && OB_FAIL(format_dump_obj(*value, column_id))) { + CLOG_LOG(ERROR, "failed to dump obj", K(*value), K(ret)); + } else { + ++index; + } + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + } else if (NULL == row.old_row_.data_ && 0 == row.old_row_.size_) { + // no data of old row + } else { + CLOG_LOG(ERROR, "old cci.init fail", K(row), K(ret)); + } + + if (OB_SUCC(ret) && T_DML_DELETE != row.dml_type_) { + ObCellReader new_cci; + if (OB_FAIL(new_cci.init(row.new_row_.data_, row.new_row_.size_, SPARSE))) { + CLOG_LOG(ERROR, "new cci.init fail", K(ret)); + } else { + int64_t index = 0; + while (OB_SUCC(ret) && OB_SUCC(new_cci.next_cell())) { + uint64_t column_id = OB_INVALID_ID; + const ObObj* value = NULL; + if (OB_FAIL(new_cci.get_cell(column_id, value))) { + } else if (NULL == value) { + ret = OB_ERR_UNEXPECTED; + } else if (ObExtendType == value->get_type() && ObActionFlag::OP_END_FLAG == value->get_ext()) { + ret = OB_ITER_END; + } else { + if (OB_SUCC(ret) && OB_FAIL(format_dump_obj(*value, column_id))) { + CLOG_LOG(ERROR, "failed to dump obj", K(*value), K(ret)); + } else { + ++index; + } + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + } + } + fprintf(stdout, "\n"); + } + } + } + } else { + fprintf(stdout, "BIG_ROW: %s\n", to_cstring(mmi.get_meta())); + } + + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + return ret; +} + +int ObLogEntryParserImpl::format_dump_obj(const common::ObObj& obj, uint64_t column_id) +{ + int ret = OB_SUCCESS; + int64_t buf_len = PRINT_BUF_SIZE - 1; // reserver one character for '\0' + int64_t pos = 0; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "log entry parser is not inited", K(is_inited_), K(ret)); + } else if (OB_FAIL(obj.print_format(print_buf_, buf_len, pos))) { + CLOG_LOG(ERROR, "failed to print smart obj", K(column_id), K(obj), K(ret)); + } else if (pos >= PRINT_BUF_SIZE - 1) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "UNEXPECTED pos", K(pos), K(buf_len), K(ret)); + } else { + print_buf_[pos] = '\0'; + fprintf(stdout, "%lu:%s", column_id, print_buf_); + } + return ret; +} + +int ObLogEntryParser::stat_block_(const ObLogBlockMetaV2& meta) +{ + + int ret = OB_SUCCESS; + if (!meta.check_meta_checksum()) { + // fatal error, data corrupt + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "check block meta checksum error", K(ret), K(meta)); + } else { + ObBlockType type = meta.get_block_type(); + switch (type) { + case OB_DATA_BLOCK: { + int64_t size = meta.get_serialize_size(); + advance_(size); + log_stat_.data_block_header_size_ += size; + break; + } + case OB_HEADER_BLOCK: + // not used any more + // file header is CLOG_DIO_ALIGN_SIZE bytes, + // skip it + advance_(meta.get_total_len()); + break; + case OB_INFO_BLOCK: { + ret = OB_ITER_END; // info block is the end + break; + } + case OB_TRAILER_BLOCK: + ret = OB_ITER_END; + break; + // iter will never encounter a trailer for InfoBlock lies ahead + default: + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "unknown block meta type", K(ret), K(meta), K(file_id_), K(cur_offset_)); + } + } + return ret; +} + +int ObLogEntryParser::stat_clog_entry(const ObLogEntry& entry, const int64_t pos, const bool is_compressed) +{ + int ret = OB_SUCCESS; + if (!entry.check_integrity()) { + ret = OB_INVALID_DATA; + CLOG_LOG(ERROR, "entry.check_integrity() fail", K(entry), K(ret)); + } else if (is_compressed && OB_FAIL(log_stat_.compressed_tenant_ids_.set_refactored( + entry.get_header().get_partition_key().get_tenant_id(), 1 /* overwrite*/))) { + CLOG_LOG(ERROR, "failed to set_refactored", K(entry), K(ret)); + } else { + log_stat_.log_header_size_ += entry.get_header().get_serialize_size(); + log_stat_.log_size_ += entry.get_header().get_data_len(); + log_stat_.primary_table_id_ = entry.get_header().get_partition_key().get_table_id(); + if (OB_LOG_SUBMIT == entry.get_header().get_log_type()) { + if (is_compressed) { + log_stat_.compressed_log_cnt_++; + log_stat_.compressed_log_size_ += pos; + log_stat_.original_log_size_ += entry.get_total_len(); + } else { + log_stat_.non_compressed_log_cnt_++; + } + + ++log_stat_.total_log_count_; + int64_t trans_inc = -1; + int64_t submit_log_type_tmp = -1; + const char* buf = entry.get_buf(); + int64_t buf_len = entry.get_header().get_data_len(); + int64_t pos = 0; + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &submit_log_type_tmp))) { + CLOG_LOG(ERROR, "failed to decode submit_log_type", K(entry), K(ret)); + } else { + int submit_log_type = static_cast(submit_log_type_tmp); + const uint64_t real_tenant_id = entry.get_header().get_partition_key().get_tenant_id(); + if ((OB_LOG_SP_TRANS_COMMIT & submit_log_type) || (OB_LOG_SP_ELR_TRANS_COMMIT & submit_log_type)) { + ++log_stat_.sp_trans_count_; + } else if (OB_LOG_TRANS_PREPARE & submit_log_type) { + ++log_stat_.dist_trans_count_; + } + + if (((OB_LOG_SP_TRANS_REDO & submit_log_type) || (OB_LOG_TRANS_REDO & submit_log_type) || + OB_LOG_SP_ELR_TRANS_COMMIT == submit_log_type || OB_LOG_SP_TRANS_COMMIT == submit_log_type) && + submit_log_type < OB_LOG_TRANS_MAX) { + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &trans_inc))) { + CLOG_LOG(ERROR, "failed to decode trans_inc", K(ret), KP(buf), K(pos), K(buf_len)); + } else { + log_stat_.trans_log_size_ += buf_len - pos; + switch (submit_log_type) { + case OB_LOG_TRANS_REDO: { + if (OB_FAIL(stat_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_REDO: { + if (OB_FAIL(stat_sp_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_sp_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE: { + if (OB_FAIL(stat_trans_redo_prepare_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_trans_redo_prepare_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_COMMIT: + case OB_LOG_SP_ELR_TRANS_COMMIT: { + if (OB_FAIL(stat_sp_trans_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_sp_trans_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT: { + if (OB_FAIL(stat_trans_redo_prepare_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_redo_prepare_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR: { + if (OB_FAIL(stat_trans_redo_prepare_commit_clear_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_redo_prepare_commit_clear_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + default: { + CLOG_LOG(ERROR, "unknown log type", K(submit_log_type)); + break; + } + } + } + } + } + } + } + return ret; +} + +int ObLogEntryParser::stat_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransRedoLogHelper helper; + ObSpTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + log_stat_.mutator_size_ += log.get_mutator().get_position(); + stat_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObSpTransCommitLogHelper helper; + ObSpTransCommitLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + log_stat_.mutator_size_ += log.get_mutator().get_position(); + stat_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog log(helper); + int64_t pos = 0; + if (OB_FAIL(log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(log.decrypt_table_key())) { + log_stat_.mutator_size_ += log.get_mutator().get_position(); + stat_memtable_mutator(log.get_mutator().get_data(), log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + int64_t pos = 0; + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialzie log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + log_stat_.mutator_size_ += r_log.get_mutator().get_position(); + stat_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + log_stat_.mutator_size_ += r_log.get_mutator().get_position(); + stat_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_trans_redo_prepare_commit_clear_log( + const char* data, int64_t len, const uint64_t real_tenant_id) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObTransRedoLogHelper helper; + ObTransRedoLog r_log(helper); + if (OB_FAIL(r_log.init_for_deserialize())) { + CLOG_LOG(WARN, "redo log init for deserialize fail", K(ret)); + } else if (OB_FAIL(r_log.deserialize(data, len, pos))) { + CLOG_LOG(WARN, "failed to deserialize log", K(ret)); + } else if (OB_FAIL(r_log.replace_encrypt_info_tenant_id(real_tenant_id))) { + CLOG_LOG(WARN, "failed to replace encrypt info tenant id", K(ret)); + } else if (OB_SUCC(r_log.decrypt_table_key())) { + log_stat_.mutator_size_ += r_log.get_mutator().get_position(); + stat_memtable_mutator(r_log.get_mutator().get_data(), r_log.get_mutator().get_position()); + } else if (ret == OB_EAGAIN) { + // impossible branch + } else { + CLOG_LOG(WARN, "redo log deserialize error", K(ret), KP(data), K(len), K(pos)); + } + return ret; +} + +int ObLogEntryParser::stat_memtable_mutator(const char* buf, int64_t len) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + ObMemtableMutatorIterator mmi; + ObMemtableMutatorMeta meta; + int64_t meta_pos = 0; + if (OB_FAIL(meta.deserialize(buf, len, meta_pos))) { + CLOG_LOG(ERROR, "meta.deserialize fail", K(ret)); + } else if (ObTransRowFlag::is_normal_row(meta.get_flags())) { + if (OB_FAIL(mmi.deserialize(buf, len, pos))) { + CLOG_LOG(ERROR, "mmi.deserialize fail", K(ret)); + } + + while (OB_SUCC(ret)) { + ObMemtableMutatorRow row; + if (OB_FAIL(mmi.get_next_row(row)) && OB_ITER_END != ret) { + CLOG_LOG(ERROR, "mmi.get_next_row fail", K(ret)); + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + ++log_stat_.total_row_count_; + if (row.table_id_ == log_stat_.primary_table_id_) { + ++log_stat_.primary_row_count_; + log_stat_.new_primary_row_size_ += row.new_row_.size_; + } + log_stat_.old_row_size_ += row.old_row_.size_; + log_stat_.new_row_size_ += row.new_row_.size_; + } + } + } else { /*do nothing*/ + } + return ret; +} + +int ObLogEntryParserImpl::check_filter(const ObLogEntry& entry, bool& need_print) +{ + int ret = OB_SUCCESS; + need_print = true; + const ObLogEntryHeader& header = entry.get_header(); + const ObPartitionKey& pkey = header.get_partition_key(); + const uint64_t log_id = header.get_log_id(); + if ((filter_.is_table_id_valid() && filter_.get_table_id() != pkey.get_table_id()) || + (filter_.is_partition_id_valid() && filter_.get_partition_id() != pkey.get_partition_id()) || + (filter_.is_log_id_valid() && filter_.get_log_id() != log_id)) { + need_print = false; + } + if (need_print) { + CLOG_LOG(INFO, "find target log entry", K(header)); + } + return ret; +} + +int ObLogEntryParserImpl::dump_trans_log(const ObStorageLogType log_type, const int64_t trans_inc, + const uint64_t real_tenant_id, const char* buf, const int64_t buf_len, int64_t& pos) +{ + int ret = OB_SUCCESS; + fprintf(stdout, "\t|||Trans: log_type:%s, trans_inc:%ld", get_submit_log_type(log_type), trans_inc); + switch (log_type) { + case OB_LOG_TRANS_REDO: { + if (OB_FAIL(dump_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_REDO: { + if (OB_FAIL(dump_sp_trans_redo_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_sp_trans_redo_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_PREPARE: { + if (OB_FAIL(dump_trans_prepare_log(buf + pos, buf_len - pos))) { + CLOG_LOG(ERROR, "dump_trans_prepare_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE: { + if (OB_FAIL(dump_trans_redo_prepare_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(ERROR, "dump_trans_redo_prepare_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_COMMIT: { + if (OB_FAIL(dump_trans_commit_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_COMMIT: + case OB_LOG_SP_ELR_TRANS_COMMIT: { + if (OB_FAIL(dump_sp_trans_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_sp_trans_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_CLEAR: { + if (OB_FAIL(dump_trans_clear_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_clear_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_ABORT: { + if (OB_FAIL(dump_trans_abort_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_sp_trans_abort_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_SP_TRANS_ABORT: { + if (OB_FAIL(dump_sp_trans_abort_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_sp_trans_abort_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_PREPARE_WITH_COMMIT: { + if (OB_FAIL(dump_trans_prepare_commit_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_prepare_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT: { + if (OB_FAIL(dump_trans_redo_prepare_commit_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_redo_prepare_commit_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_PREPARE_WITH_COMMIT_WITH_CLEAR: { + if (OB_FAIL(dump_trans_prepare_commit_clear_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_prepare_commit_clear_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR: { + if (OB_FAIL(dump_trans_redo_prepare_commit_clear_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_redo_prepare_commit_clear_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_MUTATOR: { + if (OB_FAIL(dump_trans_mutator_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_mutator_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_TRANS_STATE: { + if (OB_FAIL(dump_trans_state_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_state_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_MUTATOR_WITH_STATE: { + if (OB_FAIL(dump_trans_mutator_state_log(buf + pos, buf_len - pos, real_tenant_id))) { + CLOG_LOG(WARN, "dump_trans_mutator_state_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + case OB_LOG_MUTATOR_ABORT: { + if (OB_FAIL(dump_trans_mutator_abort_log(buf + pos, buf_len - pos))) { + CLOG_LOG(WARN, "dump_trans_mutator_abort_log fail", K(ret), K(buf), K(pos), K(buf_len)); + } + break; + } + default: { + CLOG_LOG(ERROR, "unknown log type", K(log_type)); + break; + } + } + return ret; +} + +} // end of namespace clog +} // end of namespace oceanbase diff --git a/tools/ob_admin/clog_tool/ob_log_entry_parser.h b/tools/ob_admin/clog_tool/ob_log_entry_parser.h new file mode 100644 index 0000000000..5f07426d01 --- /dev/null +++ b/tools/ob_admin/clog_tool/ob_log_entry_parser.h @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_TOOL_LOG_ENTRY_PARSER +#define OCEANBASE_TOOL_LOG_ENTRY_PARSER + +#include "ob_func_utils.h" +#include "ob_log_entry_filter.h" +#include "clog/ob_log_block.h" +#include "clog/ob_raw_entry_iterator.h" +#include "lib/net/ob_addr.h" +#include "share/ob_srv_rpc_proxy.h" +#include "rpc/obrpc/ob_net_client.h" + +namespace oceanbase { + +namespace share { +class ObKmsClient; +} +namespace clog { +struct ObLogStat { + ObLogStat() + : primary_table_id_(OB_INVALID_ID), + data_block_header_size_(0), + log_header_size_(0), + log_size_(0), + trans_log_size_(0), + mutator_size_(0), + padding_size_(0), + new_row_size_(0), + old_row_size_(0), + new_primary_row_size_(0), + primary_row_count_(0), + total_row_count_(0), + total_log_count_(0), + dist_trans_count_(0), + sp_trans_count_(0), + non_compressed_log_cnt_(0), + compressed_log_cnt_(0), + compressed_log_size_(0), + original_log_size_(0), + compressed_tenant_ids_() + {} + ~ObLogStat() + {} + int init(); + uint64_t primary_table_id_; + int64_t data_block_header_size_; + int64_t log_header_size_; + int64_t log_size_; + int64_t trans_log_size_; + int64_t mutator_size_; + int64_t padding_size_; + int64_t new_row_size_; + int64_t old_row_size_; + int64_t new_primary_row_size_; + int64_t primary_row_count_; + int64_t total_row_count_; + int64_t total_log_count_; + int64_t dist_trans_count_; + int64_t sp_trans_count_; + // compressed info + int64_t non_compressed_log_cnt_; // number of uncompressed entries` + int64_t compressed_log_cnt_; // number of compressed entries + int64_t compressed_log_size_; // data size of compressed entries + int64_t original_log_size_; // data size of compressed entries before compression + hash::ObHashSet compressed_tenant_ids_; + TO_STRING_KV(K_(data_block_header_size), K_(log_header_size), K_(log_size), K_(trans_log_size), K_(mutator_size), + K_(padding_size), K_(new_row_size), K_(old_row_size), K_(new_primary_row_size), K_(primary_row_count), + K_(total_row_count), K_(total_log_count), K_(dist_trans_count), K_(sp_trans_count), K_(non_compressed_log_cnt), + K_(compressed_log_cnt), K_(compressed_log_size), K_(original_log_size), K_(compressed_tenant_ids)); + +private: + DISALLOW_COPY_AND_ASSIGN(ObLogStat); +}; + +class ObInfoEntryDumpFunctor { +public: + ObInfoEntryDumpFunctor(const uint64_t file_id) : file_id_(file_id) + {} + virtual ~ObInfoEntryDumpFunctor() + {} + bool operator()(const common::ObPartitionKey& partition_key, const uint64_t min_log_id); + +private: + uint64_t file_id_; +}; + +enum ObLogFileType { + OB_UNKNOWN_FILE_TYPE, + OB_CLOG_FILE_TYPE, + OB_ILOG_FILE_TYPE, + OB_MAX_FILE_TYPE, +}; + +class ObLogEntryParserImpl { +public: + ObLogEntryParserImpl() + : is_inited_(false), + dump_hex_(false), + file_id_(-1), + cur_offset_(OB_INVALID_OFFSET), + print_buf_(NULL), + allocator_(ObModIds::OB_LOG_TOOL) + {} + virtual ~ObLogEntryParserImpl() + {} + int init(const int64_t file_id, const ObLogEntryFilter& filter, const common::ObString& host, const int32_t port, + const char* config_file); + bool is_inited() const + { + return is_inited_; + }; + int dump_clog_entry(ObLogEntry& entry, int64_t pos); + +protected: + // parse mutator data + int dump_memtable_mutator(const char* buf, int64_t len); + // parse trans log entry + int dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_sp_trans_abort_log(const char* data, int64_t len); + int dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_prepare_log(const char* data, int64_t len); + int dump_trans_commit_log(const char* data, int64_t len); + int dump_trans_abort_log(const char* data, int64_t len); + int dump_trans_clear_log(const char* data, int64_t len); + int dump_trans_prepare_commit_log(const char* data, int64_t len); + int dump_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_prepare_commit_clear_log(const char* data, int64_t len); + int dump_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_mutator_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_mutator_state_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int dump_trans_state_log(const char* data, int64_t len); + int dump_trans_mutator_abort_log(const char* data, int64_t len); + int dump_part_split_src_log(const char* data, int64_t len); + int dump_part_split_dest_log(const char* data, int64_t len); + int dump_trans_checkpoint_log(const char* data, int64_t len); + int dump_new_offline_partition_log(const char* data, int64_t len); + int dump_add_partition_to_pg_log(const char* data, int64_t len); + int dump_remove_partition_from_pg_log(const char* data, int64_t len); + int dump_trans_log(const ObStorageLogType log_type, const int64_t trans_inc, const uint64_t real_tenant_id, + const char* buf, const int64_t buf_len, int64_t& pos); + int dump_partition_schema_version_change_log(const char* data, int64_t len); + + // parse freeze log + int dump_freeze_log(const char* buf, const int64_t buf_len, ObStorageLogType& log_type, ObFreezeType& freeze_type, + ObPartitionKey& pkey, int64_t& frozen_version, ObSavedStorageInfo& info); + int dump_obj(const common::ObObj& obj, uint64_t column_id); + + int format_dump_clog_entry(ObLogEntry& entry); + int format_dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int format_dump_memtable_mutator(const char* buf, int64_t len); + int format_dump_obj(const common::ObObj& obj, uint64_t column_id); + int check_filter(const ObLogEntry& entry, bool& need_print); + +private: + int dump_clog_entry_(ObLogEntry& entry, int64_t pos); + +protected: + static const int64_t MAGIC_NUM_LEN = 2L; + static const int64_t SKIP_STEP = 4 * 1024L; + static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024; + bool is_inited_; + bool dump_hex_; + uint64_t file_id_; + int64_t cur_offset_; + ObTransID cur_trans_id_; + char* print_buf_; + ObLogEntryFilter filter_; + ObArenaAllocator allocator_; + common::ObAddr host_addr_; + obrpc::ObNetClient client_; + obrpc::ObSrvRpcProxy rpc_proxy_; +}; + +class ObLogEntryParser : public ObLogEntryParserImpl { +public: + ObLogEntryParser() + : ObLogEntryParserImpl(), log_file_type_(OB_UNKNOWN_FILE_TYPE), buf_cur_(NULL), buf_end_(NULL), is_ofs_(false) + {} + virtual ~ObLogEntryParser() + {} + + int init(uint64_t file_id, char* buf, int64_t buf_len, const ObLogEntryFilter& filter, const common::ObString& host, + const int32_t port, const char* config_file, const bool is_ofs); + int dump_all_entry(bool is_hex); + int format_dump_entry(); + int stat_log(); + const ObLogStat& get_log_stat() const + { + return log_stat_; + } + +protected: + int get_type_(ObCLogItemType& item_type); + int dump_block_(const ObLogBlockMetaV2& meta); + inline void advance_(const int64_t step); + int advance_to_next_align_(); + + int parse_next_entry(); + int format_dump_next_entry(); + int stat_next_entry(); + + // for format_dump + int skip_block_(const ObLogBlockMetaV2& meta); + int dump_ilog_entry(ObIndexEntry& entry); + + // for stat clog file + int stat_block_(const ObLogBlockMetaV2& meta); + int stat_clog_entry(const ObLogEntry& entry, const int64_t pos, const bool is_compressed); + int stat_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id); + int stat_memtable_mutator(const char* buf, int64_t len); + +protected: + static const int64_t MAGIC_NUM_LEN = 2L; + static const int64_t SKIP_STEP = 4 * 1024L; + static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024; + + ObLogFileType log_file_type_; + char* buf_cur_; + char* buf_end_; + bool is_ofs_; + ObLogStat log_stat_; + ObReadBuf compress_rbuf_; + + DISALLOW_COPY_AND_ASSIGN(ObLogEntryParser); +}; +} // namespace clog +} // end namespace oceanbase + +#endif // OCEANBASE_CLOG_OB_RAW_ENTRY_ITERATOR_ diff --git a/tools/ob_admin/main.cpp b/tools/ob_admin/main.cpp new file mode 100644 index 0000000000..944bdf8874 --- /dev/null +++ b/tools/ob_admin/main.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#include +#include +#include "share/ob_define.h" +#include "ob_admin_executor.h" +#include "clog_tool/ob_admin_clog_v2_executor.h" + +using namespace oceanbase::common; +using namespace oceanbase::tools; + +void print_usage() +{ + fprintf(stderr, "\nUsage: ob_admin clog_tool\n"); +} + +int main(int argc, char* argv[]) +{ + int ret = 0; + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_file_name("ob_admin.log", true, false); + const char* log_level = getenv("OB_ADMIN_LOG_LEVEL"); + if (NULL != log_level) { + OB_LOGGER.set_log_level(log_level); + } + std::ostringstream ss; + copy(argv, argv + argc, std::ostream_iterator(ss, " ")); + _OB_LOG(INFO, "cmd: [%s]", ss.str().c_str()); + + ObAdminExecutor* executor = NULL; + if (argc < 2) { + print_usage(); + } else { + if (0 == strcmp("clog_tool", argv[1])) { + executor = new ObAdminClogV2Executor(); + } else { + print_usage(); + } + + if (NULL != executor) { + if (OB_FAIL(executor->execute(argc, argv))) { + COMMON_LOG(WARN, "Fail to executor cmd, ", K(ret)); + } + delete executor; + } + } + return ret; +} diff --git a/tools/ob_admin/ob_admin_executor.cpp b/tools/ob_admin/ob_admin_executor.cpp new file mode 100644 index 0000000000..eebc920d0a --- /dev/null +++ b/tools/ob_admin/ob_admin_executor.cpp @@ -0,0 +1,57 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_admin_executor.h" + +namespace oceanbase { +using namespace common; +namespace tools { +int ObAdminExecutor::parse_options(int argc, char* argv[]) +{ + int ret = OB_SUCCESS; + int option_index = 0; + struct option long_options[] = {{"host", 1, NULL, 'h'}, + {"port", 1, NULL, 'p'}, + {"config_file", 1, NULL, 'f'}, + {"tenant_id", 1, NULL, 't'}, + {"wallet", 1, NULL, 'w'}, + {NULL, 0, NULL, 0}}; + int c; + while (-1 != (c = getopt_long(argc, argv, "h:p:f:t:w:", long_options, &option_index))) { + switch (c) { + case 'h': + DB_host_.assign_ptr(optarg, strlen(optarg)); + break; + case 'p': + DB_port_ = static_cast(strtol(optarg, NULL, 10)); + break; + case 'f': + config_file_ = optarg; + break; + case 't': + tenant_id_ = static_cast(strtol(optarg, NULL, 10)); + break; + case 'w': + wallet_file_ = optarg; + break; + case '?': + case ':': + ret = OB_ERR_UNEXPECTED; + break; + default: + break; + } + } + return ret; +} +} // namespace tools +} // namespace oceanbase diff --git a/tools/ob_admin/ob_admin_executor.h b/tools/ob_admin/ob_admin_executor.h new file mode 100644 index 0000000000..ad3ce929d7 --- /dev/null +++ b/tools/ob_admin/ob_admin_executor.h @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ADMIN_EXECUTOR_H_ +#define OB_ADMIN_EXECUTOR_H_ +#include +#include +#include +#include "share/ob_define.h" + +namespace oceanbase { +namespace tools { +class ObAdminExecutor { +public: + ObAdminExecutor() : DB_port_(-1), tenant_id_(0), config_file_(NULL), wallet_file_(NULL) + {} + virtual ~ObAdminExecutor() + {} + virtual int execute(int argc, char* argv[]) = 0; + +protected: + int parse_options(int argc, char* argv[]); + +protected: + common::ObString DB_host_; + int32_t DB_port_; + uint64_t tenant_id_; + const char* config_file_; + const char* wallet_file_; +}; +} // namespace tools +} // namespace oceanbase + +#endif /* OB_ADMIN_EXECUTOR_H_ */ -- GitLab