From aa3637dac7b1ed1ece07510404a6b5d744106230 Mon Sep 17 00:00:00 2001 From: tj0 Date: Wed, 1 Dec 2021 11:06:39 +0800 Subject: [PATCH] Implement MySQL Function TO_BASE64() and FROM_BASE64() --- .../oblib/src/lib/encode/ob_base64_encode.cpp | 316 +++++++++++++++++- deps/oblib/src/lib/encode/ob_base64_encode.h | 21 +- deps/oblib/src/lib/ob_name_def.h | 11 + deps/oblib/src/lib/rowid/ob_urowid.cpp | 3 + src/sql/CMakeLists.txt | 2 + src/sql/engine/expr/ob_expr_benchmark.cpp | 23 +- src/sql/engine/expr/ob_expr_benchmark.h | 19 +- .../engine/expr/ob_expr_eval_functions.cpp | 6 +- src/sql/engine/expr/ob_expr_from_base64.cpp | 174 ++++++++++ src/sql/engine/expr/ob_expr_from_base64.h | 44 +++ .../engine/expr/ob_expr_operator_factory.cpp | 5 + src/sql/engine/expr/ob_expr_to_base64.cpp | 171 ++++++++++ src/sql/engine/expr/ob_expr_to_base64.h | 53 +++ .../ob_expr_validate_password_strength.cpp | 14 +- .../expr/ob_expr_validate_password_strength.h | 14 +- src/sql/parser/ob_item_type.h | 2 + 16 files changed, 833 insertions(+), 45 deletions(-) create mode 100644 src/sql/engine/expr/ob_expr_from_base64.cpp create mode 100644 src/sql/engine/expr/ob_expr_from_base64.h create mode 100644 src/sql/engine/expr/ob_expr_to_base64.cpp create mode 100644 src/sql/engine/expr/ob_expr_to_base64.h diff --git a/deps/oblib/src/lib/encode/ob_base64_encode.cpp b/deps/oblib/src/lib/encode/ob_base64_encode.cpp index f463046581..32a583dd67 100644 --- a/deps/oblib/src/lib/encode/ob_base64_encode.cpp +++ b/deps/oblib/src/lib/encode/ob_base64_encode.cpp @@ -23,6 +23,264 @@ char ObBase64Encoder::BASE64_CHARS[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; +int ObBase64Encoder::FROM_BASE64_TABLE[] = { + /*00*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -2, + -2, + -2, + -2, + -2, + -1, + -1, + /*10*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*20*/ -2, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 62, + -1, + -1, + -1, + 63, /* !"#$%&'()*+,-./ */ + /*30*/ 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + -1, + -1, + -1, + -1, + -1, + -1, /* 0123456789:;<=>? */ + /*40*/ -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, /* @ABCDEFGHIJKLMNO */ + /*50*/ 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + -1, + -1, + -1, + -1, + -1, /* PQRSTUVWXYZ[\]^_ */ + /*60*/ -1, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, /* `abcdefghijklmno */ + /*70*/ 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + -1, + -1, + -1, + -1, + -1, /* pqrstuvwxyz{|}~ */ + /*80*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*90*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*A0*/ -2, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*B0*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*C0*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*D0*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*E0*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + /*F0*/ -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1}; + template struct InitBase64Values { typedef InitBase64Values next_v_; @@ -47,8 +305,8 @@ struct InitBase64Values { static int init_ret = InitBase64Values<0>::init(); -int ObBase64Encoder::encode( - const uint8_t* input, const int64_t input_len, char* output, const int64_t output_len, int64_t& pos) +int ObBase64Encoder::encode(const uint8_t *input, const int64_t input_len, char *output, const int64_t output_len, + int64_t &pos, const int16_t wrap) { int ret = OB_SUCCESS; if (OB_ISNULL(input) || OB_ISNULL(output) || OB_UNLIKELY(input_len < 0 || output_len < 0) || pos < 0) { @@ -77,6 +335,11 @@ int ObBase64Encoder::encode( for (int j = 0; OB_SUCC(ret) && j < 4; j++) { output[pos++] = BASE64_CHARS[output_idxes[j]]; } + if (wrap > 0) { + if (((i + 1) % (wrap / 4) == 0) && ((i + 1) * 3 != input_len)) { + output[pos++] = '\n'; + } + } iter_input++; } // for end @@ -115,8 +378,8 @@ int ObBase64Encoder::encode( return ret; } -int ObBase64Encoder::decode( - const char* input, const int64_t input_len, uint8_t* output, const int64_t output_len, int64_t& pos) +int ObBase64Encoder::decode(const char *input, const int64_t input_len, uint8_t *output, const int64_t output_len, + int64_t &pos, bool skip_spaces) { int ret = OB_SUCCESS; if (OB_ISNULL(input) || OB_ISNULL(output) || OB_UNLIKELY(input_len < 0 || output_len < 0 || pos < 0)) { @@ -138,11 +401,18 @@ int ObBase64Encoder::decode( ret = OB_BUF_NOT_ENOUGH; _OB_LOG(WARN, "buffer not enough, pos=%ld, output_len=%ld, input_len=%ld", pos, output_len, input_len); } - const char* iter_input = input; + const char *iter_input = input; + int64_t skipped_spaces = 0; for (; OB_SUCC(ret) && '=' != *iter_input && iter_input < input + input_len; iter_input++) { if (OB_UNLIKELY(!is_base64_char(*iter_input))) { - ret = OB_INVALID_ROWID; - _OB_LOG(WARN, "invalid base64 char, cur_idx=%ld, char=%c", iter_input - input, *iter_input); + if (skip_spaces) { + if (my_base64_decoder_skip_spaces(*iter_input)) { + ++skipped_spaces; + } + } else { + ret = OB_INVALID_ARGUMENT; + _OB_LOG(WARN, "invalid base64 char, cur_idx=%ld, char=%c", iter_input - input, *iter_input); + } } else { uint8_array_4[i++] = (uint8_t)(*iter_input); if (4 == i) { @@ -161,15 +431,27 @@ int ObBase64Encoder::decode( } } // for end int64_t cur_idx = iter_input - input; - for (const char* iter = iter_input; iter < input + input_len; iter++) { - // all the rest chars must be '=' - if (OB_UNLIKELY('=' != *iter)) { - ret = OB_INVALID_ROWID; + for (const char *iter = iter_input; iter < input + input_len; iter++) { + if (skip_spaces) { + if (my_base64_decoder_skip_spaces(*iter)) { + ++skipped_spaces; + } + } else { + // all the rest chars must be '=' + if (OB_UNLIKELY('=' != *iter)) { + ret = OB_INVALID_ARGUMENT; + } } } // end for - if (OB_UNLIKELY(cur_idx + 3 <= input_len)) { + if (skip_spaces) { + int64_t valid_len = input_len - skipped_spaces; + if (valid_len % 4 != 0 || valid_len < 4 || cur_idx + 3 <= valid_len) { + ret = OB_INVALID_ARGUMENT; + } + } + if (OB_UNLIKELY((cur_idx + 3 <= input_len) && !skip_spaces)) { // only last char or last two chars can be '=' - ret = OB_INVALID_ROWID; + ret = OB_INVALID_ARGUMENT; } else if (i > 0) { for (int k = 0; k < i; k++) { uint8_array_4[k] = BASE64_VALUES[uint8_array_4[k]]; @@ -179,7 +461,11 @@ int ObBase64Encoder::decode( if (OB_UNLIKELY(pos + i - 1 >= output_len)) { ret = OB_BUF_NOT_ENOUGH; - _OB_LOG(WARN, "buffer not enought, pos=%ld, output_len = %ld, i = %ld", pos, output_len, i); + if (skip_spaces && (pos + i - 1 >= output_len)) { + ret = OB_INVALID_ARGUMENT; + } else { + _OB_LOG(WARN, "buffer not enough, pos=%ld, output_len = %ld, i = %ld", pos, output_len, i); + } } else { for (int k = 0; k < i - 1; k++) { output[pos++] = (uint8_t)(uint8_array_3[k]); @@ -190,4 +476,4 @@ int ObBase64Encoder::decode( return ret; } } // namespace common -} // end namespace oceanbase \ No newline at end of file +} // end namespace oceanbase diff --git a/deps/oblib/src/lib/encode/ob_base64_encode.h b/deps/oblib/src/lib/encode/ob_base64_encode.h index 5cc3d48a1a..b3c449a30c 100644 --- a/deps/oblib/src/lib/encode/ob_base64_encode.h +++ b/deps/oblib/src/lib/encode/ob_base64_encode.h @@ -23,6 +23,7 @@ private: template friend class InitBase64Values; static char BASE64_CHARS[]; + static int FROM_BASE64_TABLE[]; static uint8_t BASE64_VALUES[256]; @@ -31,6 +32,16 @@ private: return std::isalnum(c) || c == '+' || c == '/'; } + static inline bool my_base64_decoder_skip_spaces(char c) + { + if (FROM_BASE64_TABLE[(uint8_t)c] != -2) { + return false; + } + + return true; + } + static const int64_t SOFT_NEW_LINE_STR_POS = 19; + public: static int64_t needed_encoded_length(const int64_t buf_size) { @@ -42,12 +53,12 @@ public: return (buf_size / 4) * 3; } - static int encode( - const uint8_t* input, const int64_t input_len, char* output, const int64_t output_len, int64_t& pos); + static int encode(const uint8_t *input, const int64_t input_len, char *output, const int64_t output_len, int64_t &pos, + const int16_t wrap = 0); - static int decode( - const char* input, const int64_t input_len, uint8_t* output, const int64_t output_len, int64_t& pos); + static int decode(const char *input, const int64_t input_len, uint8_t *output, const int64_t output_len, int64_t &pos, + bool skip_spaces = false); }; } // end namespace common } // end namespace oceanbase -#endif // !OB_BASE64_ENCODE_H_ \ No newline at end of file +#endif // !OB_BASE64_ENCODE_H_ diff --git a/deps/oblib/src/lib/ob_name_def.h b/deps/oblib/src/lib/ob_name_def.h index 862d2de24e..72ef069704 100644 --- a/deps/oblib/src/lib/ob_name_def.h +++ b/deps/oblib/src/lib/ob_name_def.h @@ -885,5 +885,16 @@ #define N_MULTI_VALUE_VECTORS "multi_value_vectors" #define N_MULTI_INSERT_COL_CONV_FUNCS "multi_insert_col_conv_funcs" #define N_BENCHMARK "benchmark" +#define N_UNISTR "unistr" +#define N_ASCIISTR "asciistr" +#define N_ROWID_TO_CHAR "rowidtochar" +#define N_ROWID_TO_NCHAR "rowidtonchar" +#define N_CHAR_TO_ROWID "chartorowid" +#define N_OUTPUT_PACK "output_pack" +#define N_BENCHMARK "benchmark" +#define N_WEIGHT_STRING "weight_string" +#define N_DML_EVENT "dml_event" +#define N_TO_BASE64 "to_base64" +#define N_FROM_BASE64 "from_base64" #endif // OCEANBASE_LIB_OB_NAME_DEF_H_ diff --git a/deps/oblib/src/lib/rowid/ob_urowid.cpp b/deps/oblib/src/lib/rowid/ob_urowid.cpp index bd10e1c839..cc18a561f0 100644 --- a/deps/oblib/src/lib/rowid/ob_urowid.cpp +++ b/deps/oblib/src/lib/rowid/ob_urowid.cpp @@ -329,6 +329,9 @@ int ObURowIDData::decode_base64_str( ret = OB_INVALID_ROWID; COMMON_LOG(WARN, "invalid urowid format", K(ret)); } else if (OB_FAIL(ObBase64Encoder::decode(input + 1, input_len - 1, output, output_len, pos))) { + if (ret == OB_INVALID_ARGUMENT) { + ret = OB_INVALID_ROWID; + } COMMON_LOG(WARN, "failed to decode base64 str", K(ret)); } return ret; diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index b43eb69eae..6913b889a7 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -470,6 +470,8 @@ ob_set_subtarget(ob_sql engine engine/expr/ob_expr_validate_password_strength.h engine/expr/ob_expr_benchmark.cpp engine/expr/ob_expr_benchmark.h + engine/expr/ob_expr_to_base64.cpp + engine/expr/ob_expr_from_base64.cpp engine/join/ob_basic_nested_loop_join.cpp engine/join/ob_block_based_nested_loop_join.cpp engine/join/ob_hash_join.cpp diff --git a/src/sql/engine/expr/ob_expr_benchmark.cpp b/src/sql/engine/expr/ob_expr_benchmark.cpp index 1f5bbce12c..e916fd64f3 100644 --- a/src/sql/engine/expr/ob_expr_benchmark.cpp +++ b/src/sql/engine/expr/ob_expr_benchmark.cpp @@ -1,13 +1,14 @@ -// (C) Copyright 2013-2016 Alibaba Inc. All Rights Reserved. -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License -// version 2 as published by the Free Software Foundation. -// Version: $Id$ -// Authors: -// peihan.dph@alibaba-inc.com -// Normalizer: -// peihan.dph@alibaba-inc.com -// This file is for implementation of func expr_benchmark +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ #define USING_LOG_PREFIX SQL_ENG @@ -113,4 +114,4 @@ void ObExprBenchmark::clear_all_flags(common::ObIArray &exprs, ObEvalC } } // end namespace sql -} // end namespace oceanbase \ No newline at end of file +} // end namespace oceanbase diff --git a/src/sql/engine/expr/ob_expr_benchmark.h b/src/sql/engine/expr/ob_expr_benchmark.h index 97e7a3141e..82e5273a31 100644 --- a/src/sql/engine/expr/ob_expr_benchmark.h +++ b/src/sql/engine/expr/ob_expr_benchmark.h @@ -1,10 +1,15 @@ -// Copyright 2010-2016 Alibaba Inc. All Rights Reserved. -// Author: -// peihan.dph@alibaba-inc.com -// Normalizer: -// peihan.dph@alibaba-inc.com -// -// This file defines implementation for benchmark operator +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + #ifndef OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_BENCHMARK_ #define OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_BENCHMARK_ diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index dce0669c01..d4ea53b415 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -190,6 +190,8 @@ #include "ob_expr_validate_password_strength.h" #include "ob_expr_benchmark.h" #include "ob_expr_uuid_short.h" +#include "ob_expr_to_base64.h" +#include "ob_expr_from_base64.h" namespace oceanbase { using namespace common; @@ -723,7 +725,9 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { ObExprInet6Ntoa::calc_inet6_ntoa, /* 457 */ ObExprWeightString::eval_weight_string, /* 458 */ ObExprConvertTZ::eval_convert_tz, /* 459 */ - ObExprCrc32::calc_crc32_expr /* 460 */ + ObExprCrc32::calc_crc32_expr, /* 460 */ + ObExprToBase64::eval_to_base64, /* 461 */ + ObExprFromBase64::eval_from_base64 /* 462 */ }; REG_SER_FUNC_ARRAY(OB_SFA_SQL_EXPR_EVAL, g_expr_eval_functions, ARRAYSIZEOF(g_expr_eval_functions)); diff --git a/src/sql/engine/expr/ob_expr_from_base64.cpp b/src/sql/engine/expr/ob_expr_from_base64.cpp new file mode 100644 index 0000000000..1243b263a9 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_from_base64.cpp @@ -0,0 +1,174 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG + +#include "ob_expr_from_base64.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/engine/ob_exec_context.h" +#include "sql/engine/expr/ob_expr_util.h" +#include "lib/encode/ob_base64_encode.h" +#include "lib/oblog/ob_log.h" + +using namespace oceanbase::common; +namespace oceanbase { +namespace sql { + +ObExprFromBase64::ObExprFromBase64(ObIAllocator &alloc) + : ObFuncExprOperator(alloc, T_FUN_SYS_FROM_BASE64, N_FROM_BASE64, 1, NOT_ROW_DIMENSION) +{} + +ObExprFromBase64::~ObExprFromBase64() +{} + +int ObExprFromBase64::calc(ObObj &result, const ObObj &obj, ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(allocator)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null allocator", K(ret), K(allocator)); + } else { + const ObString &in_raw = obj.get_string(); + ObLength in_raw_len = in_raw.length(); + if (OB_UNLIKELY(in_raw_len == 0)) { + result.set_string(obj.get_type(), nullptr, 0); + } else { + const char *buf = in_raw.ptr(); + int64_t buf_len = base64_needed_decoded_length(in_raw_len); + int64_t pos = 0; + char *output_buf = static_cast(allocator->alloc(buf_len)); + if (OB_ISNULL(output_buf)) { + LOG_WARN("output_buf is null", K(ret), K(buf_len), K(in_raw_len)); + result.set_null(); + } else if (OB_FAIL(ObBase64Encoder::decode( + buf, in_raw_len, reinterpret_cast(output_buf), buf_len, pos, true))) { + if (OB_UNLIKELY(ret == OB_INVALID_ARGUMENT)) { + ret = OB_SUCCESS; + result.set_null(); + } else { + LOG_WARN("failed to decode base64", K(ret)); + } + } else { + result.set_string(obj.get_type(), output_buf, pos); + result.set_collation_level(obj.get_collation_level()); + result.set_collation_type(obj.get_collation_type()); + } + } + } + return ret; +} + +int ObExprFromBase64::calc_result1(ObObj &result, const ObObj &obj, ObExprCtx &expr_ctx) const +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(obj.is_null())) { + result.set_null(); + } else if (OB_FAIL(calc(result, obj, expr_ctx.calc_buf_))) { + LOG_WARN("fail to calc function from_base64", K(ret), K(obj)); + } else { + result.set_collation_level(obj.get_collation_level()); + result.set_collation_type(get_result_type().get_collation_type()); + } + + return ret; +} + +int ObExprFromBase64::calc_result_type1(ObExprResType &type, ObExprResType &str, ObExprTypeCtx &type_ctx) const +{ + UNUSED(type_ctx); + int ret = OB_SUCCESS; + + str.set_calc_type(ObVarcharType); + str.set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + + int64_t mbmaxlen = 0; + int64_t max_result_length = 0; + if (OB_FAIL(common::ObCharset::get_mbmaxlen_by_coll(str.get_collation_type(), mbmaxlen))) { + LOG_WARN("fail to get mbmaxlen", K(type.get_collation_type()), K(ret)); + } else { + max_result_length = base64_needed_decoded_length(str.get_length()) * mbmaxlen; + if (max_result_length > MAX_BLOB_WIDTH) { + max_result_length = MAX_BLOB_WIDTH; + } + int64_t max_l = max_result_length / mbmaxlen; + int64_t max_deduce_length = max_l * mbmaxlen; + if (max_deduce_length < OB_MAX_MONITOR_INFO_LENGTH) { + type.set_varbinary(); + type.set_length(max_deduce_length); + type.set_collation_level(CS_LEVEL_IMPLICIT); + } else { + type.set_blob(); + // TODO : Fixme the blob type do not need to set_length. + // Maybe need wait ObDDLResolver::check_text_length fix the judge of length. + type.set_length(max_deduce_length); + } + } + + return ret; +} + +int ObExprFromBase64::eval_from_base64(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *arg = nullptr; + ObExecContext* exec_ctx = &ctx.exec_ctx_; + + if (OB_FAIL(expr.args_[0]->eval(ctx, arg))) { + LOG_WARN("eval arg failed", K(ret)); + } else if (OB_UNLIKELY(arg->is_null())) { + res.set_null(); + } else { + const ObString &in_raw = arg->get_string(); + ObLength in_raw_len = in_raw.length(); + if (OB_UNLIKELY(in_raw_len == 0)) { + res.set_string(nullptr, 0); + } else { + const char *buf = in_raw.ptr(); + char *output_buf = nullptr; + int64_t buf_len = base64_needed_decoded_length(in_raw_len); + int64_t pos = 0; + output_buf = static_cast(exec_ctx->get_allocator().alloc(buf_len)); + if (OB_ISNULL(output_buf)) { + LOG_WARN("output_buf is null", K(ret), K(buf_len), K(in_raw_len)); + res.set_null(); + } else if (OB_FAIL(ObBase64Encoder::decode( + buf, in_raw_len, reinterpret_cast(output_buf), buf_len, pos, true))) { + if (OB_UNLIKELY(ret == OB_INVALID_ARGUMENT)) { + ret = OB_SUCCESS; + res.set_null(); + } else { + LOG_WARN("failed to decode base64", K(ret)); + } + } else { + res.set_string(output_buf, pos); + if (OB_FAIL(ObExprUtil::set_expr_ascii_result(expr, ctx, res, ObString(pos, output_buf)))) { + LOG_WARN("set ASCII result failed", K(ret)); + } + } + } + } + return ret; +} + +int ObExprFromBase64::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const +{ + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + int ret = OB_SUCCESS; + rt_expr.eval_func_ = ObExprFromBase64::eval_from_base64; + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/engine/expr/ob_expr_from_base64.h b/src/sql/engine/expr/ob_expr_from_base64.h new file mode 100644 index 0000000000..612a4e031d --- /dev/null +++ b/src/sql/engine/expr/ob_expr_from_base64.h @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_ENGINE_OB_EXPR_FROM_BASE64_ +#define OCEANBASE_SQL_ENGINE_OB_EXPR_FROM_BASE64_ + +#include "sql/engine/expr/ob_expr_operator.h" +#include "share/object/ob_obj_cast.h" + +namespace oceanbase { +namespace sql { +class ObExprFromBase64 : public ObFuncExprOperator { +public: + explicit ObExprFromBase64(common::ObIAllocator &alloc); + virtual ~ObExprFromBase64(); + virtual int calc_result1(common::ObObj &result, const common::ObObj &obj, common::ObExprCtx &expr_ctx) const; + virtual int calc_result_type1(ObExprResType &type, ObExprResType &str, common::ObExprTypeCtx &type_ctx) const; + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const; + static int eval_from_base64(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); + +private: + DISALLOW_COPY_AND_ASSIGN(ObExprFromBase64); + + static int calc(common::ObObj &result, const common::ObObj &obj1, common::ObIAllocator *allocator); + static const int64_t NCHAR_PER_BASE64 = 4; + static const int64_t NCHAR_PER_BASE64_GROUP = 3; + static inline ObLength base64_needed_decoded_length(ObLength length_of_encoded_data) + { + return (ObLength)ceil(length_of_encoded_data * NCHAR_PER_BASE64_GROUP / NCHAR_PER_BASE64); + } +}; +} // namespace sql +} // namespace oceanbase + +#endif // OCEANBASE_SQL_ENGINE_OB_EXPR_FROM_BASE64_ diff --git a/src/sql/engine/expr/ob_expr_operator_factory.cpp b/src/sql/engine/expr/ob_expr_operator_factory.cpp index 4706a66fd1..9418f4fa73 100644 --- a/src/sql/engine/expr/ob_expr_operator_factory.cpp +++ b/src/sql/engine/expr/ob_expr_operator_factory.cpp @@ -274,6 +274,9 @@ #include "sql/engine/expr/ob_expr_validate_password_strength.h" #include "sql/engine/expr/ob_expr_benchmark.h" #include "sql/engine/expr/ob_expr_uuid_short.h" +#include "sql/engine/expr/ob_expr_convert_tz.h" +#include "sql/engine/expr/ob_expr_to_base64.h" +#include "sql/engine/expr/ob_expr_from_base64.h" using namespace oceanbase::common; namespace oceanbase { @@ -690,6 +693,8 @@ void ObExprOperatorFactory::register_expr_operators() REG_OP(ObExprWeightString); REG_OP(ObExprBenchmark); REG_OP(ObExprDay); + REG_OP(ObExprToBase64); + REG_OP(ObExprFromBase64); // register oracle system function REG_OP_ORCL(ObExprSysConnectByPath); REG_OP_ORCL(ObExprTimestampNvl); diff --git a/src/sql/engine/expr/ob_expr_to_base64.cpp b/src/sql/engine/expr/ob_expr_to_base64.cpp new file mode 100644 index 0000000000..1df949291e --- /dev/null +++ b/src/sql/engine/expr/ob_expr_to_base64.cpp @@ -0,0 +1,171 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG + +#include "ob_expr_to_base64.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/engine/ob_exec_context.h" +#include "sql/engine/expr/ob_expr_util.h" +#include "lib/encode/ob_base64_encode.h" +#include "lib/oblog/ob_log.h" + +using namespace oceanbase::common; +namespace oceanbase { +namespace sql { + +ObExprToBase64::ObExprToBase64(ObIAllocator &alloc) + : ObFuncExprOperator(alloc, T_FUN_SYS_TO_BASE64, N_TO_BASE64, 1, NOT_ROW_DIMENSION) +{} + +ObExprToBase64::~ObExprToBase64() +{} + +int ObExprToBase64::calc(ObObj &result, const ObObj &obj, ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null allocator", K(ret), K(allocator)); + } else { + const ObString &in_raw = obj.get_string(); + ObLength in_raw_len = in_raw.length(); + const char *buf = in_raw.ptr(); + int64_t buf_len = base64_needed_encoded_length(in_raw_len); + if (OB_UNLIKELY(buf_len == 0)) { + result.set_string(obj.get_type(), nullptr, 0); + } else { + int64_t pos = 0; + char *output_buf = static_cast(allocator->alloc(buf_len)); + if (OB_ISNULL(output_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret), K(buf_len), K(in_raw.length())); + } else if (OB_FAIL(ObBase64Encoder::encode(reinterpret_cast(buf), + in_raw_len, + output_buf, + buf_len, + pos, + NUM_OF_CHAR_PER_LINE_QUOTED_PRINTABLE))) { + LOG_WARN("failed to encode base64", K(ret)); + } else { + result.set_string(obj.get_type(), output_buf, pos); + } + } + } + return ret; +} + +int ObExprToBase64::calc_result1(ObObj &result, const ObObj &obj, ObExprCtx &expr_ctx) const +{ + int ret = OB_SUCCESS; + + if (obj.is_null()) { + result.set_null(); + } else if (OB_FAIL(calc(result, obj, expr_ctx.calc_buf_))) { + LOG_WARN("fail to calc function to_base64", K(ret), K(obj)); + } else { + result.set_collation_level(obj.get_collation_level()); + result.set_collation_type(get_result_type().get_collation_type()); + } + + return ret; +} + +int ObExprToBase64::calc_result_type1(ObExprResType &type, ObExprResType &str, ObExprTypeCtx &type_ctx) const +{ + int ret = OB_SUCCESS; + + str.set_calc_type(ObVarcharType); + str.set_calc_collation_type(str.is_string_type() ? str.get_collation_type() : CS_TYPE_UTF8MB4_BIN); + + int64_t mbmaxlen = 0; + int64_t max_result_length = 0; + if (OB_FAIL(common::ObCharset::get_mbmaxlen_by_coll(str.get_collation_type(), mbmaxlen))) { + LOG_WARN("fail to get mbmaxlen", K(str.get_collation_type()), K(ret)); + } else { + max_result_length = (base64_needed_encoded_length(str.get_length()) - 1) * mbmaxlen; + if (max_result_length > MAX_BLOB_WIDTH) { + max_result_length = MAX_BLOB_WIDTH; + } + int64_t max_l = max_result_length / mbmaxlen; + int64_t max_deduce_length = max_l * mbmaxlen; + if (max_deduce_length < OB_MAX_MONITOR_INFO_LENGTH) { + type.set_varchar(); + type.set_length(max_deduce_length); + type.set_collation_type(get_default_collation_type(type.get_type(), *type_ctx.get_session())); + } else { + type.set_blob(); + // TODO : Fixme the blob type do not need to set_length. + // Maybe need wait ObDDLResolver::check_text_length fix the judge of length. + type.set_length(max_deduce_length); + type.set_collation_type(get_default_collation_type(type.get_type(), *type_ctx.get_session())); + } + type.set_collation_level(CS_LEVEL_COERCIBLE); + } + + return ret; +} + +int ObExprToBase64::eval_to_base64(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *arg = nullptr; + ObExecContext* exec_ctx = &ctx.exec_ctx_; + + if (OB_FAIL(expr.args_[0]->eval(ctx, arg))) { + LOG_WARN("eval arg failed", K(ret)); + } else if (OB_UNLIKELY(arg->is_null())) { + res.set_null(); + } else { + const ObString &in_raw = arg->get_string(); + ObLength in_raw_len = in_raw.length(); + const char *buf = in_raw.ptr(); + char *output_buf = nullptr; + int64_t buf_len = base64_needed_encoded_length(in_raw_len); + if (OB_UNLIKELY(buf_len == 0)) { + res.set_string(nullptr, 0); + } else { + int64_t pos = 0; + output_buf = static_cast(exec_ctx->get_allocator().alloc(buf_len)); + if (OB_ISNULL(output_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret), K(buf_len), K(in_raw.length())); + } else if (OB_FAIL(ObBase64Encoder::encode(reinterpret_cast(buf), + in_raw_len, + output_buf, + buf_len, + pos, + NUM_OF_CHAR_PER_LINE_QUOTED_PRINTABLE))) { + LOG_WARN("failed to encode base64", K(ret)); + } else { + res.set_string(output_buf, pos); + if (OB_FAIL(ObExprUtil::set_expr_ascii_result(expr, ctx, res, ObString(pos, output_buf)))) { + LOG_WARN("set ASCII result failed", K(ret)); + } + } + } + } + + return ret; +} + +int ObExprToBase64::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const +{ + UNUSED(expr_cg_ctx); + UNUSED(raw_expr); + int ret = OB_SUCCESS; + rt_expr.eval_func_ = ObExprToBase64::eval_to_base64; + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/engine/expr/ob_expr_to_base64.h b/src/sql/engine/expr/ob_expr_to_base64.h new file mode 100644 index 0000000000..7abe5f3d01 --- /dev/null +++ b/src/sql/engine/expr/ob_expr_to_base64.h @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_ENGINE_OB_EXPR_TO_BASE64_ +#define OCEANBASE_SQL_ENGINE_OB_EXPR_TO_BASE64_ + +#include "sql/engine/expr/ob_expr_operator.h" +#include "share/object/ob_obj_cast.h" + +namespace oceanbase { +namespace sql { +class ObExprToBase64 : public ObFuncExprOperator { +public: + explicit ObExprToBase64(common::ObIAllocator &alloc); + virtual ~ObExprToBase64(); + virtual int calc_result1(common::ObObj &result, const common::ObObj &obj, common::ObExprCtx &expr_ctx) const; + virtual int calc_result_type1(ObExprResType &type, ObExprResType &str, common::ObExprTypeCtx &type_ctx) const; + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const; + static int eval_to_base64(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); + +private: + DISALLOW_COPY_AND_ASSIGN(ObExprToBase64); + + static int calc(common::ObObj &result, const common::ObObj &obj1, common::ObIAllocator *allocator); + static const int64_t NCHAR_PER_BASE64 = 4; + static const int64_t NCHAR_PER_BASE64_GROUP = 3; + static const int64_t NCHAR_PER_STR_PAD = 2; + static const int64_t NUM_OF_CHAR_PER_LINE_QUOTED_PRINTABLE = 76; + static inline ObLength base64_needed_encoded_length(ObLength length_of_data) + { + ObLength nb_base64_chars; + if (length_of_data == 0) { + return 0; + } + nb_base64_chars = (length_of_data + NCHAR_PER_STR_PAD) / NCHAR_PER_BASE64_GROUP * NCHAR_PER_BASE64; + return nb_base64_chars + /* base64 char incl padding */ + (nb_base64_chars - 1) / 76 + /* newlines */ + 1; /* NUL termination of string */ + } +}; +} // namespace sql +} // namespace oceanbase + +#endif // OCEANBASE_SQL_ENGINE_OB_EXPR_TO_BASE64_ diff --git a/src/sql/engine/expr/ob_expr_validate_password_strength.cpp b/src/sql/engine/expr/ob_expr_validate_password_strength.cpp index 19f8f9e84a..67f97830a6 100644 --- a/src/sql/engine/expr/ob_expr_validate_password_strength.cpp +++ b/src/sql/engine/expr/ob_expr_validate_password_strength.cpp @@ -1,6 +1,14 @@ -// Copyright (c) 2021 Alibaba Inc. All Rights Reserved. -// Author: -// renqing +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ #define USING_LOG_PREFIX SQL_ENG diff --git a/src/sql/engine/expr/ob_expr_validate_password_strength.h b/src/sql/engine/expr/ob_expr_validate_password_strength.h index 07c358e523..d37f12a717 100644 --- a/src/sql/engine/expr/ob_expr_validate_password_strength.h +++ b/src/sql/engine/expr/ob_expr_validate_password_strength.h @@ -1,6 +1,14 @@ -// Copyright (c) 2021 Alibaba Inc. All Rights Reserved. -// Author: -// renqing +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ #ifndef SRC_SQL_ENGINE_EXPR_OB_EXPR_VALIDATE_PASSWORD_STRENGTH_H_ #define SRC_SQL_ENGINE_EXPR_OB_EXPR_VALIDATE_PASSWORD_STRENGTH_H_ diff --git a/src/sql/parser/ob_item_type.h b/src/sql/parser/ob_item_type.h index 5f0e21d360..383f3c06cb 100644 --- a/src/sql/parser/ob_item_type.h +++ b/src/sql/parser/ob_item_type.h @@ -443,6 +443,8 @@ typedef enum ObItemType { T_FUN_SYS_WEIGHT_STRING = 729, T_FUN_SYS_CRC32 = 730, T_FUN_SYS_DAY = 731, + T_FUN_SYS_TO_BASE64 = 732, + T_FUN_SYS_FROM_BASE64 = 733, ///< @note add new mysql only function type before this line T_MYSQL_ONLY_SYS_MAX_OP = 800, -- GitLab