fix unhex when sql_mode is strict

src/share/object/ob_obj_cast.cpp

@@ -517,7 +517,6 @@ int ObHexUtils::unhex(const ObString& text, ObCastCtx& cast_ctx, ObObj& result)
     ret = OB_ALLOCATE_MEMORY_FAILED;
     LOG_ERROR("alloc memory failed", K(alloc_length), K(ret));
   } else {
-    bool all_valid_char = true;
     int32_t i = 0;
     char c1 = 0;
     char c2 = 0;
@@ -532,21 +531,18 @@ int ObHexUtils::unhex(const ObString& text, ObCastCtx& cast_ctx, ObObj& result)
         i = 1;
       }
     }
-    while (OB_SUCC(ret) && all_valid_char && i < text.length()) {
+    while (OB_SUCC(ret) && i < text.length()) {
       if (isxdigit(c1) && isxdigit(c2)) {
         buf[i / 2] = (char)((get_xdigit(c1) << 4) | get_xdigit(c2));
         c1 = text[++i];
         c2 = text[++i];
-      } else if (lib::is_oracle_mode()) {
+      } else {
         ret = OB_ERR_INVALID_HEX_NUMBER;
         LOG_WARN("invalid hex number", K(ret), K(c1), K(c2), K(text));
-      } else {
-        all_valid_char = false;
-        result.set_null();
       }
     }
 
-    if (OB_SUCC(ret) && all_valid_char) {
+    if (OB_SUCC(ret)) {
       str_result.assign_ptr(buf, tmp_length);
       result.set_varchar(str_result);
     }

src/sql/engine/expr/ob_datum_cast.cpp

@@ -477,7 +477,6 @@ int ObDatumHexUtils::unhex(const ObExpr& expr, const ObString& in_str, ObEvalCtx
     ret = OB_ALLOCATE_MEMORY_FAILED;
     LOG_WARN("alloc memory failed", K(alloc_length), K(ret));
   } else {
-    bool all_valid_char = true;
     int32_t i = 0;
     char c1 = 0;
     char c2 = 0;
@@ -492,27 +491,23 @@ int ObDatumHexUtils::unhex(const ObExpr& expr, const ObString& in_str, ObEvalCtx
         i = 1;
       }
     }
-    while (OB_SUCC(ret) && all_valid_char && i < in_str.length()) {
+    while (OB_SUCC(ret) && i < in_str.length()) {
       if (isxdigit(c1) && isxdigit(c2)) {
         buf[i / 2] = (char)((get_xdigit(c1) << 4) | get_xdigit(c2));
         c1 = in_str[++i];
         c2 = in_str[++i];
-      } else if (lib::is_oracle_mode()) {
+      } else {
         ret = OB_ERR_INVALID_HEX_NUMBER;
         LOG_WARN("invalid hex number", K(ret), K(c1), K(c2), K(in_str));
-      } else {
-        all_valid_char = false;
-        res_datum.set_null();
       }
     }
-    if (OB_SUCC(ret) && all_valid_char) {
+    if (OB_SUCC(ret)) {
       ObString str_res(tmp_length, buf);
       // There will be no zero fill in the unhex() function, so it is directly assigned here
       res_datum.pack_ = tmp_length;
       res_datum.ptr_ = buf;
     }
   }
-
   return ret;
 }
 

src/sql/engine/expr/ob_expr_unhex.cpp

@@ -17,6 +17,7 @@
 #include "share/object/ob_obj_cast.h"
 #include "sql/engine/expr/ob_datum_cast.h"
 #include "sql/session/ob_sql_session_info.h"
+#include "sql/engine/ob_exec_context.h"
 
 using namespace oceanbase::common;
 
@@ -29,7 +30,7 @@ ObExprUnhex::ObExprUnhex(ObIAllocator& alloc) : ObStringExprOperator(alloc, T_FU
 ObExprUnhex::~ObExprUnhex()
 {}
 
-int ObExprUnhex::calc(ObObj& result, const ObObj& text, common::ObCastCtx& cast_ctx)
+int ObExprUnhex::calc(ObObj& result, const ObObj& text, common::ObCastCtx& cast_ctx, ObExprCtx& expr_ctx)
 {
   int ret = OB_SUCCESS;
   if (text.is_null()) {
@@ -43,7 +44,28 @@ int ObExprUnhex::calc(ObObj& result, const ObObj& text, common::ObCastCtx& cast_
     if (OB_FAIL(ret)) {
       LOG_WARN("invalid input format. need varchar.", K(ret), K(text));
     } else if (OB_FAIL(ObHexUtils::unhex(str, cast_ctx, result))) {
-      LOG_WARN("Failed to unhex", K(ret), K(str));
+      // when ret is OB_ERR_INVALID_HEX_NUMBER and sql_mode is not strict return null
+      if (OB_ERR_INVALID_HEX_NUMBER == ret) {
+        ObCastMode default_cast_mode = CM_NONE;
+        const ObSQLSessionInfo* session = expr_ctx.my_session_;
+        if (OB_ISNULL(session)) {
+          ret = OB_ERR_UNEXPECTED;
+          LOG_WARN("session is NULL", K(ret));
+        } else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(session->get_stmt_type(), session, default_cast_mode))) {
+          LOG_WARN("failed to get default cast mode", K(ret));
+        } else if (CM_IS_WARN_ON_FAIL(default_cast_mode)) {
+          ret = OB_SUCCESS;
+          result.set_null();
+        } else {
+          ret = OB_ERR_INVALID_HEX_NUMBER;
+          LOG_WARN("Failed to unhex", K(ret), K(str));
+        }
+      } else {
+        // when ret is other error code
+        LOG_WARN("Failed to unhex", K(ret), K(str));
+      }
+    } else {
+      // when ret is success do nothing
     }
   }
   return ret;
@@ -59,7 +81,7 @@ int ObExprUnhex::calc_result1(ObObj& result, const ObObj& text, ObExprCtx& expr_
     LOG_WARN("varchar buffer not init", K(ret));
   } else {
     EXPR_DEFINE_CAST_CTX(expr_ctx, CM_NONE);
-    if (OB_FAIL(calc(result, text, cast_ctx))) {
+    if (OB_FAIL(calc(result, text, cast_ctx, expr_ctx))) {
       LOG_WARN("fail to calc", K(ret), K(text));
     } else if (!result.is_null()) {
       result.set_meta_type(result_type_.get_obj_meta());
@@ -88,9 +110,29 @@ int ObExprUnhex::eval_unhex(const ObExpr& expr, ObEvalCtx& ctx, ObDatum& res_dat
   } else if (param->is_null()) {
     res_datum.set_null();
   } else if (OB_FAIL(ObDatumHexUtils::unhex(expr, param->get_string(), ctx, res_datum))) {
-    LOG_WARN("fail to eval unhex", K(ret), K(expr), K(*param));
+    // when ret is OB_ERR_INVALID_HEX_NUMBER and sql_mode is not strict return null
+    if (OB_ERR_INVALID_HEX_NUMBER == ret) {
+      ObCastMode default_cast_mode = CM_NONE;
+      const ObSQLSessionInfo* session = ctx.exec_ctx_.get_my_session();
+      if (OB_ISNULL(session)) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("session is NULL", K(ret));
+      } else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(session->get_stmt_type(), session, default_cast_mode))) {
+        LOG_WARN("failed to get default cast mode", K(ret));
+      } else if (CM_IS_WARN_ON_FAIL(default_cast_mode)) {
+        ret = OB_SUCCESS;
+        res_datum.set_null();
+      } else {
+        ret = OB_ERR_INVALID_HEX_NUMBER;
+        LOG_WARN("fail to eval unhex", K(ret), K(expr), K(*param));
+      }
+    } else {
+      // ret is other error code
+      LOG_WARN("fail to eval unhex", K(ret), K(expr), K(*param));
+    }
+  } else {
+    // ret is success
   }
-
   return ret;
 }
 

src/sql/engine/expr/ob_expr_unhex.h

@@ -22,7 +22,8 @@ public:
   explicit ObExprUnhex(common::ObIAllocator& alloc);
   virtual ~ObExprUnhex();
   virtual int calc_result_type1(ObExprResType& type, ObExprResType& text, common::ObExprTypeCtx& type_ctx) const;
-  static int calc(common::ObObj& result, const common::ObObj& text, common::ObCastCtx& cast_ctx);
+  static int calc(
+      common::ObObj& result, const common::ObObj& text, common::ObCastCtx& cast_ctx, common::ObExprCtx& expr_ctx);
   virtual int calc_result1(common::ObObj& result, const common::ObObj& text, common::ObExprCtx& expr_ctx) const;
   virtual int cg_expr(ObExprCGCtx& op_cg_ctx, const ObRawExpr& raw_expr, ObExpr& rt_expr) const override;
   static int eval_unhex(const ObExpr& expr, ObEvalCtx& ctx, ObDatum& res_datum);