diff --git a/Makefile.am b/Makefile.am index e5f5a8ce298b105d08d28c078aaf3dcb5d53daf1..178b56652c33d3eba26f87141e09076590f959b4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ -EXTRA_DIST=build.sh svn_dist_version script/deploy unittest/Makefile.in unittest/obproxy/Makefile.in deps/easy +EXTRA_DIST=build.sh svn_dist_version script/deploy unittest/Makefile.in unittest/obproxy/Makefile.in deps/easy hotfuncs.txt bin_SCRIPTS=script/deploy/obproxyd.sh if WITH_TEST_CASE diff --git a/build.sh b/build.sh index 39a56da3c60675d01974cb856c51c3e86534ec8d..f1a82755cc9deccac824569dab33c73cacbf94e5 100755 --- a/build.sh +++ b/build.sh @@ -69,17 +69,22 @@ function do_config() case "x$1" in xdebug) # configure for developers - ./configure --with-gcc-version=5.2.0 --with-coverage=no --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no + ./configure --with-gcc-version=9.3.0 --with-coverage=no --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no echo -e "\033[31m ===build debug version=== \033[0m" ;; xgcov) # configure for release - ./configure --with-gcc-version=5.2.0 --with-coverage=yes --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no + ./configure --with-gcc-version=9.3.0 --with-coverage=yes --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no echo -e "\033[31m ===build gcov version=== \033[0m" ;; + xperf) + # configure for release + ./configure --with-gcc-version=9.3.0 --with-coverage=no --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no --with-release --with-perf + echo -e "\033[31m ===build perf version=== \033[0m" + ;; *) # configure for release - ./configure --with-gcc-version=5.2.0 --with-coverage=no --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no --with-release + ./configure --with-gcc-version=9.3.0 --with-coverage=no --enable-buildtime=no --enable-strip-ut=no --enable-silent-rules --enable-dlink-observer=no --with-release echo -e "\033[31m ===build release version=== \033[0m" ;; esac diff --git a/configure.ac b/configure.ac index 1d3f2a7ef6d18a78232b9b5a075e9ade7797ade1..368789772fa45b32e14ccf357445fc8b51f466a4 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_INIT([OceanBase], - [3.2.0], + [3.2.3], [wgs13579@gmail.com], [obproxy-ce], [http://oceanbase.taobao.org/]) - obapi_version="3.2.0" + obapi_version="3.2.3" AC_SUBST(obapi_version) AC_DISABLE_STATIC @@ -26,23 +26,23 @@ if test "$withval" = "yes"; then case "$host_cpu" in *aarch64* ) - AM_CXXFLAGS="-g -O2 -D_OB_VERSION=1000 -D_NO_EXCEPTION -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-deprecated -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -finline-functions -fno-strict-aliasing -mtune=generic -Wno-psabi -Wno-sign-compare" - AM_CFLAGS="-g -O2 -D_OB_VERSION=1000 -DCOMPATIBLE -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -DNDEBUG -finline-functions -fno-strict-aliasing -Wall -mtune=generic -Wno-psabi -Wno-sign-compare" + AM_CXXFLAGS="-g -O2 -D_OB_VERSION=1000 -D_NO_EXCEPTION -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-deprecated -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -finline-functions -fno-strict-aliasing -mtune=generic -Wno-psabi -Wno-sign-compare -Wno-class-memaccess -Wno-deprecated-copy -Wno-ignored-qualifiers -Wno-aligned-new -Wno-format-truncation -Wno-literal-suffix -Wno-format-overflow -Wno-stringop-truncation -Wno-memset-elt-size -Wno-cast-function-type -Wno-address-of-packed-member -fuse-ld=lld -ffunction-sections -Wl,--no-warn-symbol-ordering,--symbol-ordering-file,${ac_abs_confdir}/hotfuncs.txt" + AM_CFLAGS="-g -O2 -D_OB_VERSION=1000 -DCOMPATIBLE -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -DNDEBUG -finline-functions -fno-strict-aliasing -Wall -mtune=generic -Wno-psabi -Wno-sign-compare -fuse-ld=lld -ffunction-sections -Wl,--no-warn-symbol-ordering,--symbol-ordering-file,${ac_abs_confdir}/hotfuncs.txt" ;; * ) - AM_CXXFLAGS="-g -O2 -D_OB_VERSION=1000 -D_NO_EXCEPTION -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-deprecated -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -finline-functions -fno-strict-aliasing -mtune=core2 -Wno-psabi -Wno-sign-compare" - AM_CFLAGS="-g -O2 -D_OB_VERSION=1000 -DCOMPATIBLE -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -DNDEBUG -finline-functions -fno-strict-aliasing -Wall -mtune=core2 -Wno-psabi -Wno-sign-compare" + AM_CXXFLAGS="-g -O2 -D_OB_VERSION=1000 -D_NO_EXCEPTION -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DNDEBUG -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-deprecated -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -finline-functions -fno-strict-aliasing -mtune=core2 -Wno-psabi -Wno-sign-compare -Wno-class-memaccess -Wno-deprecated-copy -Wno-ignored-qualifiers -Wno-aligned-new -Wno-format-truncation -Wno-literal-suffix -Wno-format-overflow -Wno-stringop-truncation -Wno-memset-elt-size -Wno-cast-function-type -Wno-address-of-packed-member -fuse-ld=lld -ffunction-sections -Wl,--no-warn-symbol-ordering,--symbol-ordering-file,${ac_abs_confdir}/hotfuncs.txt" + AM_CFLAGS="-g -O2 -D_OB_VERSION=1000 -DCOMPATIBLE -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -DNDEBUG -finline-functions -fno-strict-aliasing -Wall -mtune=core2 -Wno-psabi -Wno-sign-compare -fuse-ld=lld -ffunction-sections -Wl,--no-warn-symbol-ordering,--symbol-ordering-file,${ac_abs_confdir}/hotfuncs.txt" ;; esac fi ], - [ AM_CXXFLAGS="-D__STDC_LIMIT_MACROS -D_OB_VERSION=1000 -D__STDC_CONSTANT_MACROS -D_NO_EXCEPTION -g -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -Wno-deprecated -fno-strict-aliasing -fno-omit-frame-pointer -mtune=generic -Wno-psabi -Wno-sign-compare" + [ AM_CXXFLAGS="-D__STDC_LIMIT_MACROS -D_OB_VERSION=1000 -D__STDC_CONSTANT_MACROS -D_NO_EXCEPTION -g -Wall -Wextra -Wno-unused-parameter -Wformat -Wno-conversion -Wno-invalid-offsetof -Wno-unused-result -Wno-format-security -Wno-deprecated -fno-strict-aliasing -fno-omit-frame-pointer -mtune=generic -Wno-psabi -Wno-sign-compare -Wno-class-memaccess -Wno-deprecated-copy -Wno-ignored-qualifiers -Wno-aligned-new -Wno-format-truncation -Wno-literal-suffix -Wno-format-overflow -Wno-stringop-truncation -Wno-memset-elt-size -Wno-cast-function-type -Wno-address-of-packed-member" AM_CFLAGS="-D_OB_VERSION=1000 -DCOMPATIBLE -D__USE_LARGEFILE64 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -fno-omit-frame-pointer -g -Wall -Wno-psabi -Wno-sign-compare" ] ) #check gcc version, add -Wno-ignored-qualifiers flag for gcc version greater than 4.3.0 - GCC_VERSION=`$CC -dumpversion` + GCC_VERSION=`$CC -dumpfullversion -dumpversion` if test $? -eq 0; then major=`echo $GCC_VERSION | cut -d. -f1` minor=`echo $GCC_VERSION | cut -d. -f2` @@ -106,9 +106,12 @@ [with perf (default is NO)]), [ if test "$withval" = "yes"; then - test_perf=yes - AM_CXXFLAGS="${AM_CXXFLAGS} -D__NEED_PERF__" - AM_LDFLAGS="${AM_LDFLAGS} -lprofiler" + # 下面参数为使用gperftools的选项，目前未用tcmalloc + # test_perf=yes + # AM_CXXFLAGS="${AM_CXXFLAGS} -D__NEED_PERF__" + # AM_LDFLAGS="${AM_LDFLAGS} -lprofiler" + AM_CXXFLAGS="${AM_CXXFLAGS} -fno-omit-frame-pointer" + AM_CFLAGS="${AM_CFLAGS} -fno-omit-frame-pointer" fi ], [with_perf=no] @@ -306,7 +309,7 @@ AM_LDFLAGS="${AM_LDFLAGS} -Wl,--no-as-needed -Wl,--allow-multiple-definition" BIN_LDFLAGS="-lc -lrt" - if test "X5.2.0" = "X$GCC_VERSION"; then + if test "X9.3.0" = "X$GCC_VERSION"; then AM_LDFLAGS="${AM_LDFLAGS} -static-libstdc++ -static-libgcc" BIN_LDFLAGS="${BIN_LDFLAGS} ${TOOLS_DIR}/lib64/libstdc++.a" postdeps_CXX=`echo " $postdeps_CXX " | sed 's, -lstdc++ ,,g'` diff --git a/deps/3rd/obproxy.el7.aarch64.deps b/deps/3rd/obproxy.el7.aarch64.deps index 7da7a80a8af15ce8c9e59dc5cfbc98c3e50a73d2..e13aeab86d6949736192782c49a20da5cce9b52e 100644 --- a/deps/3rd/obproxy.el7.aarch64.deps +++ b/deps/3rd/obproxy.el7.aarch64.deps @@ -10,12 +10,14 @@ devdeps-mariadb-connector-c-3.1.12-3.el7.aarch64.rpm devdeps-gtest-1.8.0-3.el7.aarch64.rpm devdeps-prometheus-cpp-0.8.0-2.el7.aarch64.rpm devdeps-grpc-1.20.1-8.el7.aarch64.rpm +devdeps-sqlite-3.38.1-5.el7.aarch64.rpm [tools] -obdevtools-gcc-5.2.0-3.el7.aarch64.rpm +obdevtools-gcc9-9.3.0-3.el7.aarch64.rpm obdevtools-bintuils-2.30-3.el7.aarch64.rpm obdevtools-bison-2.4.1-3.el7.aarch64.rpm obdevtools-flex-2.5.35-3.el7.aarch64.rpm +obdevtools-llvm-11.0.1-40.el7.aarch64.rpm [release] oceanbase-ce-devel-3.1.3-10000102022030411.el7.aarch64.rpm diff --git a/deps/3rd/obproxy.el7.x86_64.deps b/deps/3rd/obproxy.el7.x86_64.deps index 4004e36df4aab3d0d2bc73382bf556b5f4eff7b1..05ec58e86789a0daed10c0629a159a57398d98e0 100644 --- a/deps/3rd/obproxy.el7.x86_64.deps +++ b/deps/3rd/obproxy.el7.x86_64.deps @@ -9,12 +9,14 @@ devdeps-mariadb-connector-c-3.1.12-3.el7.x86_64.rpm devdeps-prometheus-cpp-0.8.0-4.el7.x86_64.rpm devdeps-gtest-1.8.0-3.el7.x86_64.rpm devdeps-grpc-1.20.1-3.el7.x86_64.rpm +devdeps-sqlite-3.38.1-5.el7.x86_64.rpm [tools] -obdevtools-gcc-5.2.0-3.el7.x86_64.rpm +obdevtools-gcc9-9.3.0-3.el7.x86_64.rpm obdevtools-bintuils-2.30-3.el7.x86_64.rpm obdevtools-bison-2.4.1-3.el7.x86_64.rpm obdevtools-flex-2.5.35-3.el7.x86_64.rpm +obdevtools-llvm-11.0.1-40.el7.x86_64.rpm [release] oceanbase-ce-devel-3.1.3-10000102022030411.el7.x86_64.rpm diff --git a/deps/3rd/obproxy.el8.aarch64.deps b/deps/3rd/obproxy.el8.aarch64.deps index 28270b280c626d1eccd07e6f5bd99100a596795e..2eb3ff67ceb13320463892ce0326dc67e3a77b2b 100644 --- a/deps/3rd/obproxy.el8.aarch64.deps +++ b/deps/3rd/obproxy.el8.aarch64.deps @@ -10,12 +10,14 @@ devdeps-mariadb-connector-c-3.1.12-16.el8.aarch64.rpm devdeps-gtest-1.8.0-16.el8.aarch64.rpm devdeps-prometheus-cpp-0.8.0-2.el8.aarch64.rpm devdeps-grpc-1.20.1-8.el8.aarch64.rpm +devdeps-sqlite-3.38.1-5.el8.aarch64.rpm [tools] -obdevtools-gcc-5.2.0-15.el8.aarch64.rpm +obdevtools-gcc9-9.3.0-3.el8.aarch64.rpm obdevtools-binutils-2.30-7.el8.aarch64.rpm obdevtools-bison-2.4.1-9.el8.aarch64.rpm obdevtools-flex-2.5.35-10.el8.aarch64.rpm +obdevtools-llvm-11.0.1-40.el8.aarch64.rpm [release] oceanbase-ce-devel-3.1.3-10000102022030411.el8.aarch64.rpm diff --git a/deps/3rd/obproxy.el8.x86_64.deps b/deps/3rd/obproxy.el8.x86_64.deps index 0a8cb46ad28c0c7a4ba1597f0889daa812dcf79c..94724899fbc8c081d99932acb69d78ddad77334b 100644 --- a/deps/3rd/obproxy.el8.x86_64.deps +++ b/deps/3rd/obproxy.el8.x86_64.deps @@ -9,12 +9,14 @@ devdeps-mariadb-connector-c-3.1.12-3.el8.x86_64.rpm devdeps-prometheus-cpp-0.8.0-4.el8.x86_64.rpm devdeps-gtest-1.8.0-3.el8.x86_64.rpm devdeps-grpc-1.20.1-3.el8.x86_64.rpm +devdeps-sqlite-3.38.1-5.el8.x86_64.rpm [tools] -obdevtools-gcc-5.2.0-3.el8.x86_64.rpm +obdevtools-gcc9-9.3.0-3.el8.x86_64.rpm obdevtools-bintuils-2.30-3.el8.x86_64.rpm obdevtools-bison-2.4.1-3.el8.x86_64.rpm obdevtools-flex-2.5.35-3.el8.x86_64.rpm +obdevtools-llvm-11.0.1-40.el8.x86_64.rpm [release] oceanbase-ce-devel-3.1.3-10000102022030411.el8.x86_64.rpm diff --git a/deps/3rd/obproxy.spec b/deps/3rd/obproxy.spec index 2d88e8dccd57664873168b646c3dc44d80a06609..2fccee9d654062cb3170316d71ba64701b7ea286 100644 --- a/deps/3rd/obproxy.spec +++ b/deps/3rd/obproxy.spec @@ -34,7 +34,7 @@ OceanBase Database Proxy %setup %build -./configure CXX=${CXX} CC=${CC} --with-gcc-version=5.2.0 RELEASEID=%{RELEASE} --prefix=%{_prefix} --with-test-case=no --with-release=yes --with-tblib-root=/opt/csr/common --with-easy-root=/usr --with-easy-lib-path=/usr/lib64 --with-svnfile --enable-shared=default --enable-silent-rules +./configure CXX=${CXX} CC=${CC} --with-gcc-version=9.3.0 RELEASEID=%{RELEASE} --prefix=%{_prefix} --with-test-case=no --with-release=yes --with-tblib-root=/opt/csr/common --with-easy-root=/usr --with-easy-lib-path=/usr/lib64 --with-svnfile --enable-shared=default --enable-silent-rules mkdir -p unittest CPU_CORES=`grep -c ^processor /proc/cpuinfo` MAKE_ARGS="-j $CPU_CORES" diff --git a/hotfuncs.txt b/hotfuncs.txt new file mode 100644 index 0000000000000000000000000000000000000000..ecb8845bbd6c442d765feb420d7d1c59fe5cd04d --- /dev/null +++ b/hotfuncs.txt @@ -0,0 +1,189 @@ +_ZN9oceanbase7obproxy5event9ObEThread7executeEv +_ZN9oceanbase7obproxy5event12mutex_unlockEPNS1_12ObProxyMutexEPNS1_9ObEThreadE +_ZN9oceanbase6common5ObPtrINS_7obproxy5event12ObProxyMutexEED1Ev +_ZN9oceanbase6common5ObPtrINS_7obproxy5event12ObProxyMutexEED2Ev +_ZN9oceanbase7obproxy5event9ObEThread13process_eventEPNS1_7ObEventEi +_ZN9oceanbase7obproxy3net12ObNetHandler14main_net_eventEiPNS0_5event7ObEventE +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection13read_from_netERNS0_5event9ObEThreadE +_ZN9oceanbase6common7ObLatch10try_wrlockEjPKj +_ZN9oceanbase7obproxy15ObStatProcessor17incr_raw_stat_sumEPNS0_17ObRecRawStatBlockEPNS0_5event9ObEThreadEll +_ZN9oceanbase6common7ObLatch6unlockEPKj +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection22read_signal_and_updateEi +_ZN9oceanbase7obproxy5proxy14ObMysqlVCTable10find_entryEPNS0_5event5ObVIOE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM12main_handlerEiPv +_ZN9oceanbase7obproxy7obutils23get_global_proxy_configEv +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection18reset_read_triggerEv +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection12write_to_netERNS0_5event9ObEThreadE +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection23write_signal_and_updateEi +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel12main_handlerEiPv +_ZN9oceanbase6common8ObLogger10get_loggerEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM26state_server_response_readEiPv +_ZNK9oceanbase7obproxy17ObSessionFieldMgr29get_common_sys_variable_valueERKNS_6common8ObStringERNS2_5ObObjE +_ZNK9oceanbase7obproxy5proxy9ObMysqlSM36is_extra_ok_packet_for_stats_enabledEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM35handle_first_normal_response_packetERNS1_20ObMysqlAnalyzeStatusEbRl +_ZNK9oceanbase7obproxy17ObSessionFieldMgr23get_common_sys_variableERKNS_6common8ObStringERPNS0_17ObSessionSysFieldE +_ZN9oceanbase7obproxy5proxy26ObMysqlTransactionAnalyzer14set_server_cmdENS_7obmysql10ObMySQLCmdENS1_19ObMysqlProtocolModeEbb +_ZN9oceanbase7obproxy5proxy26ObMysqlTransactionAnalyzer16analyze_responseERNS0_5event16ObIOBufferReaderERNS1_20ObMysqlAnalyzeResultEPNS1_11ObMysqlRespEb +_ZN9oceanbase7obproxy5proxy18ObProxyParserUtils18analyze_one_packetERNS0_5event16ObIOBufferReaderERNS1_20ObMysqlAnalyzeResultE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM25state_client_request_readEiPv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM34tunnel_handler_response_transferedEiPv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact16handle_pl_updateERNS2_12ObTransStateE +_ZNK9oceanbase6common6ObAddr8is_validEv +_ZNK9oceanbase7obproxy5proxy24ObProxyPartitionLocation10get_leaderEv +_ZNK9oceanbase7obproxy17ObSessionFieldMgr25get_variable_value_commonERKNS_6common8ObStringERNS2_5ObObjENS0_16ObSessionVarTypeE +_ZNK9oceanbase7obproxy17ObSessionFieldMgr27get_sys_variable_from_blockERKNS_6common8ObStringERPNS0_17ObSessionSysFieldEPKNS0_12ObFieldBlockIS6_LNS0_11HeapObjTypeE1EEE +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel16consumer_handlerEiRNS1_21ObMysqlTunnelConsumerE +_ZN9oceanbase7obproxy5event11ObMIOBuffer14dealloc_readerEPNS1_16ObIOBufferReaderE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM21tunnel_handler_clientEiRNS1_21ObMysqlTunnelConsumerE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession27handle_transaction_completeEPNS0_5event16ObIOBufferReaderERb +_ZN9oceanbase7obproxy5proxy20ObSysVarSetProcessor7releaseEPNS0_18ObDefaultSysVarSetE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM21analyze_mysql_requestERNS1_20ObMysqlAnalyzeStatusE +_ZN9oceanbase7obproxy5proxy19ObProxyMysqlRequest5reuseEb +_ZN9oceanbase7obproxy7obutils16ObSqlParseResult16clear_proxy_stmtEv +_ZN9oceanbase6common9ObSEArrayINS_7obproxy7obutils8SqlFieldELl5ENS0_19ModulePageAllocatorENS0_22ObArrayDefaultCallBackIS4_EENS0_22NotImplementItemEncodeIS4_EENS0_7ObArrayIS4_S5_S7_S9_EEE7destroyEv +_ZN9oceanbase7obproxy5proxy22ObMysqlRequestAnalyzer15analyze_requestERKNS1_19ObRequestAnalyzeCtxERNS1_18ObMysqlAuthRequestERNS1_19ObProxyMysqlRequestERNS_7obmysql10ObMySQLCmdERNS1_20ObMysqlAnalyzeStatusEb +_ZN9oceanbase7obproxy5proxy19ObProxyMysqlRequest13get_parse_sqlERKNS_6common8ObStringE +_ZN9oceanbase7obproxy5proxy22ObMysqlRequestAnalyzer18do_analyze_requestERKNS1_19ObRequestAnalyzeCtxENS_7obmysql10ObMySQLCmdERNS1_18ObMysqlAuthRequestERNS1_19ObProxyMysqlRequestEb +__dynamic_cast +_ZN9oceanbase7obproxy7obutils16ObProxySqlParser9parse_sqlERKNS_6common8ObStringE16ObProxyParseModeRNS1_16ObSqlParseResultEbNS3_15ObCollationTypeEbb +_ZN9oceanbase7obproxy20string_to_lower_caseEPci +obproxy_parse_utf8_sql +ob_proxy_parser_utf8_yyset_extra +ob_proxy_parser_utf8_yy_scan_buffer +ob_proxy_parser_utf8_yylex_init_extra +ob_proxy_parser_utf8_yy_switch_to_buffer +ob_proxy_parser_utf8_yyparse +ob_proxy_parser_utf8_yyget_extra +ob_proxy_parser_utf8_yyget_lval +ob_proxy_parser_utf8_yylex +yy_push_state +obproxy_parse_malloc +_ZN9oceanbase6common16ObArenaAllocator5allocEl +_ZN9oceanbase7obproxy7obutils16ObProxySqlParser19get_parse_allocatorERPNS_6common16ObArenaAllocatorE +_ZN9oceanbase7obproxy5proxy22ObMysqlRequestAnalyzer19handle_internal_cmdERNS1_19ObProxyMysqlRequestE +_ZN9oceanbase7obproxy5proxy19ObProxyMysqlRequest11add_requestEPNS0_5event16ObIOBufferReaderEl +_ZN9oceanbase7obproxy5event16ObIOBufferReader4copyEPcll +_ZN9oceanbase7obproxy5proxy19ObClientSessionInfo22revalidate_sys_var_setERNS1_20ObSysVarSetProcessorE +_ZN9oceanbase7obproxy5proxy20ObSysVarSetProcessor7acquireEv +_ZN9oceanbase6common7DRWLock11RDLockGuardC1ERS1_ +_ZN9oceanbase6common7DRWLock11RDLockGuardC2ERS1_ +_ZN9oceanbase7obproxy7obutils16ObSqlParseResult11load_resultERK19_ObProxyParseResultbbb +_ZN9oceanbase6common7ObLatch6rdlockEjl +_ZN9oceanbase6common7ObLatch14low_try_rdlockEjjRb +_ZN9oceanbase6common7ObLatch8low_lockEjljjMS1_FijjRbES4_ +_ZN9oceanbase6common20ObDiagnoseTenantInfo23get_local_diagnose_infoEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM25state_server_request_sendEiPv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM26setup_server_response_readEv +_ZN9oceanbase7obproxy5event11ObMIOBuffer21append_block_internalEPNS1_15ObIOBufferBlockE +_ZN9oceanbase7obproxy5proxy19ObProxyMysqlRequest7get_sqlEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM26setup_get_cluster_resourceEv +_ZN9oceanbase7obproxy3net15ObInactivityCop16check_inactivityEiPNS0_5event7ObEventE +_ZN9oceanbase7obproxy5event9ObEThread19dequeue_local_eventERNS_6common5QueueINS1_7ObEventENS5_10Link_link_EEE +_ZN9oceanbase7obproxy5event11ObMIOBuffer12append_blockEl +_ZN9oceanbase7obproxy5event15ObIOBufferBlock4freeEv +_ZNK10__cxxabiv120__si_class_type_info12__do_dyncastElNS_17__class_type_info10__sub_kindEPKS1_PKvS4_S6_RNS1_16__dyncast_resultE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM17handle_api_returnEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact15handle_responseERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession10do_io_readEPNS0_5event14ObContinuationElPNS3_11ObMIOBufferE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession10do_io_readEPNS0_5event14ObContinuationElPNS3_11ObMIOBufferE.localalias +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection22set_inactivity_timeoutEl +_ZN9oceanbase7obproxy5proxy9ObMysqlSM32call_transact_and_set_next_stateEPFvRNS1_15ObMysqlTransact12ObTransStateEE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM33callout_api_and_start_next_actionENS1_15ObMysqlTransact24ObStateMachineActionTypeE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM21setup_server_transferEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact23lookup_skip_open_serverERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM18setup_cmd_completeEv +_ZN9oceanbase7obproxy5proxy13ObLDCLocation5resetEv +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection10do_io_readEPNS0_5event14ObContinuationElPNS3_11ObMIOBufferE +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel12add_producerEPNS0_5event13ObVConnectionElPNS3_16ObIOBufferReaderEMNS1_9ObMysqlSMEFiiRNS1_21ObMysqlTunnelProducerEENS1_17ObMysqlTunnelTypeEPKcb +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel12add_consumerEPNS0_5event13ObVConnectionES5_MNS1_9ObMysqlSMEFiiRNS1_21ObMysqlTunnelConsumerEENS1_17ObMysqlTunnelTypeEPKcl +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact12ObTransState21reset_internal_bufferEv +_ZN9oceanbase7obproxy5proxy22get_global_table_cacheEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM25setup_server_request_sendEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact12ObTransState27get_trans_consistency_levelERNS1_19ObClientSessionInfoE +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection16set_read_triggerEv +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection25cancel_inactivity_timeoutEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM20server_transfer_initEPNS0_5event11ObMIOBufferERl +_ZN9oceanbase7obproxy5event11ObMIOBuffer13remove_appendEPNS1_16ObIOBufferReaderERl +_ZN9oceanbase7obproxy5proxy9ObMysqlSM16do_observer_openEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM25do_internal_observer_openEv +_ZN9oceanbase7obproxy5proxy20ObMysqlServerSession11do_io_writeEPNS0_5event14ObContinuationElPNS3_16ObIOBufferReaderE +_ZN9oceanbase7obproxy5proxy20ObMysqlServerSession10do_io_readEPNS0_5event14ObContinuationElPNS3_11ObMIOBufferE +_ZN9oceanbase7obproxy5proxy14ObMysqlVCTable9new_entryEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM21attach_server_sessionERNS1_20ObMysqlServerSessionE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM20handle_observer_openEv +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel10tunnel_runEPNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession11do_io_writeEPNS0_5event14ObContinuationElPNS3_16ObIOBufferReaderE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession11do_io_writeEPNS0_5event14ObContinuationElPNS3_16ObIOBufferReaderE.localalias +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel12producer_runERNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel16producer_handlerEiRNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel23producer_handler_packetEiRNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy26ObMysqlTransactionAnalyzer16analyze_responseERNS0_5event16ObIOBufferReaderEPNS1_11ObMysqlRespE +_ZN9oceanbase7obproxy5proxy16ObPacketAnalyzer24process_response_contentERbS3_ +_ZN9oceanbase7obproxy5proxy26ObMysqlTransactionAnalyzer22analyze_trans_responseERNS0_5event16ObIOBufferReaderEPNS1_11ObMysqlRespE +_ZN9oceanbase7obproxy5proxy26ObMysqlTransactionAnalyzer22analyze_trans_responseERKNS_6common8ObStringEPNS1_11ObMysqlRespE +_ZN9oceanbase7obproxy5proxy19ObMysqlRespAnalyzer18analyze_mysql_respERNS1_14ObBufferReaderERNS1_12ObRespResultEPNS1_11ObMysqlRespE +_ZN9oceanbase7obproxy5proxy19ObMysqlRespAnalyzer15handle_last_eofEj +_ZN9oceanbase7obproxy5proxy19ObMysqlRespAnalyzer13read_pkt_typeERNS1_14ObBufferReaderERNS1_12ObRespResultE +_ZN9oceanbase7obproxy5proxy19ObMysqlRespAnalyzer16analyze_resp_pktERNS1_12ObRespResultEPNS1_11ObMysqlRespE +_ZN9oceanbase7obmysql11ObMySQLUtil10get_lengthERPKcRm +_ZN9oceanbase7obproxy5proxy9ObMysqlSM21tunnel_handler_serverEiRNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession21attach_server_sessionEPNS1_20ObMysqlServerSessionE +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection11do_io_writeEPNS0_5event14ObContinuationElPNS3_16ObIOBufferReaderE +_ZN9oceanbase7obproxy3net20ObUnixNetVConnection8reenableEPNS0_5event5ObVIOE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM34tunnel_handler_server_cmd_completeERNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel19finish_all_internalERNS1_21ObMysqlTunnelProducerEb +_ZN9oceanbase7obproxy5proxy9ObMysqlSM14trim_ok_packetERNS0_5event16ObIOBufferReaderE +_ZNK9oceanbase7obproxy5proxy12ObRespResult16is_resp_finishedERbRNS1_21ObMysqlRespEndingTypeE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact14modify_requestERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact20build_server_requestERNS2_12ObTransStateERPNS0_5event16ObIOBufferReaderERl +_ZNK9oceanbase7obproxy5proxy9ObMysqlSM24use_compression_protocolEv +_ZN9oceanbase7obproxy5proxy14ObMysqlVCTable13cleanup_entryEPNS1_19ObMysqlVCTableEntryEb +_ZN9oceanbase7obproxy5proxy14ObMysqlVCTable12remove_entryEPNS1_19ObMysqlVCTableEntryE +_ZN9oceanbase7obproxy5proxy25ObProxySessionInfoHandler23analyze_extra_ok_packetERNS0_5event16ObIOBufferReaderERNS1_19ObClientSessionInfoERNS1_19ObServerSessionInfoEbRNS1_19ObRespAnalyzeResultE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM28do_congestion_control_lookupEv +_ZNK9oceanbase7obproxy5proxy19ObClientSessionInfo28need_reset_user_session_varsERKNS1_19ObServerSessionInfoE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM35do_oceanbase_internal_observer_openERPNS1_20ObMysqlServerSessionE +_ZN9oceanbase7obproxy5proxy19ObClientSessionInfo17get_full_usernameEv +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession19acquire_svr_sessionERK8sockaddrbRPNS1_20ObMysqlServerSessionE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM30handle_saved_session_variablesEv +_ZNK9oceanbase7obproxy5proxy19ObClientSessionInfo23need_reset_session_varsERKNS1_19ObServerSessionInfoE +_ZN9oceanbase7obproxy7obutils19ObVariableLenBufferILl32EE4initEl +_ZN9oceanbase7obproxy5proxy9ObMysqlSM28do_partition_location_lookupEv +_ZN9oceanbase6common13murmurhash64AEPKvim +_ZN9oceanbase7obproxy12ObRefHashMapINS0_5proxy15ObTableEntryKeyEPNS2_12ObTableEntryENS2_18ObGetTableEntryKeyELl16384ENS_6common19ModulePageAllocatorEE3getERKS3_ +_ZN9oceanbase7obproxy5proxy12ObTableCache22is_table_entry_expiredERKNS1_12ObTableEntryE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM31state_partition_location_lookupEiPv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM26process_partition_locationERNS1_18ObMysqlRouteResultE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM25update_cached_dummy_entryERNS1_18ObMysqlRouteResultE +_ZN9oceanbase7obproxy5proxy20ObMysqlClientSession16check_update_ldcEv +_ZNK9oceanbase7obproxy5proxy20ObMysqlClientSession20get_current_idc_nameEv +_ZN9oceanbase7obproxy5proxy9ObMysqlSM29is_cached_dummy_entry_expiredEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact12ObTransState21alloc_internal_bufferEl +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact14handle_requestERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy22is_supported_mysql_cmdENS_7obmysql10ObMySQLCmdE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact19is_internal_requestERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM24print_mysql_complete_logEPNS1_21ObMysqlTunnelProducerE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact16handle_pl_lookupERNS2_12ObTransStateE +_ZN9oceanbase7obproxy3net11ops_ip_copyER8sockaddrjt +_ZN9oceanbase7obproxy5proxy13ObMysqlTunnel5resetEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact32handle_congestion_control_lookupERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact20start_access_controlERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy9ObMysqlSM12update_statsEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact27handle_response_from_serverERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact20handle_server_failedERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact33handle_on_forward_server_responseERNS2_12ObTransStateE +_ZN9oceanbase7obproxy5proxy12ObMysqlSMApi26do_response_transform_openEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact28build_oceanbase_user_requestERNS2_12ObTransStateEPNS0_5event16ObIOBufferReaderERS7_Rl +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact15rewrite_stmt_idERNS2_12ObTransStateEPNS0_5event16ObIOBufferReaderE +_ZN9oceanbase7obproxy5proxy13ObLDCLocation16reset_item_arrayEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact24handle_oceanbase_requestERNS2_12ObTransStateE +_ZNK9oceanbase7obproxy17ObSessionFieldMgr17get_database_nameERNS_6common8ObStringE +_ZNK9oceanbase7obproxy5proxy19ObClientSessionInfo17get_database_nameERNS_6common8ObStringE +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact22extract_partition_infoERNS2_12ObTransStateE +_ZNK9oceanbase7obproxy17ObSessionFieldMgr19get_str_field_valueENS0_12ObVFieldTypeERNS_6common8ObStringE +_ZN9oceanbase7obproxy5proxy13ObLDCLocation20get_thread_allocatorERPNS_6common19ModulePageAllocatorE +_ZN9oceanbase7obproxy5proxy12ObMysqlSMApi5resetEv +_ZN9oceanbase7obproxy5proxy15ObMysqlTransact23can_direct_ok_for_loginERNS2_12ObTransStateE +_ZNK9oceanbase7obproxy5proxy12ObTableEntry9to_stringEPcl +_ZN9oceanbase7obproxy5proxy20ObSqlTableRefHashMap14clean_hash_mapEv +_ZN9oceanbase7obproxy5proxy21ObPartitionRefHashMap14clean_hash_mapEv diff --git a/rpm/obproxy-ce-VER.txt b/rpm/obproxy-ce-VER.txt index a4f52a5dbb5a6ad929239d21c3c77812a5862540..b347b11eac8ae6e5c3d9a0ed133b14e11280c9ff 100644 --- a/rpm/obproxy-ce-VER.txt +++ b/rpm/obproxy-ce-VER.txt @@ -1 +1 @@ -3.2.0 \ No newline at end of file +3.2.3 diff --git a/script/deploy/obproxyd.sh b/script/deploy/obproxyd.sh index bb61560b95c8d42fec63884ba576ad1768cdf0ea..f3b9156248e471a8dff6b6c3b2bf41a0af8c8a2a 100755 --- a/script/deploy/obproxyd.sh +++ b/script/deploy/obproxyd.sh @@ -4,12 +4,20 @@ # description: obproxyd is a daemon which guarantee obproxy is running OBPROXY_CMD= -OBPROXY_APP_NAME= -OBPROXY_APP_NAME_ARG='' -OBPROXY_IDC_NAME= +OBPROXY_APP_NAME=${APPNAME} +OBPROXY_APP_NAME_ARG= +OBPROXY_IDC_NAME=${OBPROXY_IDC_NAME} OBPROXY_IDC_NAME_ARG= OBPROXY_ROOT=$(dirname $(dirname $(readlink -f "$0"))) -OBPROXY_PORT=2883 +OBPROXY_PORT=${OBPROXY_PORT} +ENABLE_ROOT_SERVER=${ENABLE_ROOT_SERVER} +ENABLE_ROOT_SERVER_ARG= +ROOT_SERVER_CLUSTER_NAME=${ROOT_SERVER_CLUSTER_NAME} +ROOT_SERVER_CLUSTER_NAME_ARG= +ROOT_SERVER_LIST=${ROOT_SERVER_LIST} +ROOT_SERVER_LIST_ARG= +OBPROXY_CONFIG_SERVER_URL=${OBPROXY_CONFIG_SERVER_URL} +OBPROXY_CONFIG_SERVER_URL_ARG= STATUS_ALL_OBPROXY= @@ -66,8 +74,40 @@ function check_opt() ;; esac - OBPROXY_CONFIG_SERVER_URL=$OBPROXY_CONFIG_SERVER_URL - echo "obproxy config server url:$OBPROXY_CONFIG_SERVER_URL" + if [ x$ENABLE_ROOT_SERVER == xtrue ];then + ENABLE_ROOT_SERVER_ARG=" -r" + if [ -z $ROOT_SERVER_CLUSTER_NAME ] + then + echo "ROOT_SERVER_CLUSTER_NAME not set" + exit 1 + else + ROOT_SERVER_CLUSTER_NAME_ARG=" -s ${ROOT_SERVER_CLUSTER_NAME}" + fi + + if [ -z $ROOT_SERVER_LIST ] + then + echo "ROOT_SERVER_LIST not set" + exit 1 + else + ROOT_SERVER_LIST_ARG=" -t ${ROOT_SERVER_LIST}" + fi + else + echo "obproxy config server url:$OBPROXY_CONFIG_SERVER_URL" + OBPROXY_CONFIG_SERVER_URL_ARG=" -u $OBPROXY_CONFIG_SERVER_URL" + if + + # check PORT + if [ -z $OBPROXY_PORT ] + then + if [ ! -z $obproxy_port ] + then + OBPROXY_PORT=$obproxy_port + else + echo "env OBPROXY_PORT not set, use default 2883" + OBPROXY_PORT=2883 + fi + fi + info_msg "port is: $OBPROXY_PORT" # check APP_NAME if [ -z "$OBPROXY_APP_NAME" ]; @@ -85,6 +125,7 @@ function check_opt() else info_msg "idc name is: $OBPROXY_IDC_NAME" OBPROXY_IDC_NAME_ARG=" -i ${OBPROXY_IDC_NAME}" + OBPROXY_OPT_LOCAL="${OBPROXY_OPT_LOCAL},proxy_idc_name=$OBPROXY_IDC_NAME" fi if [ -z $WORK_THREAD_NUM ] @@ -94,11 +135,19 @@ function check_opt() fi echo "obproxy work_thread_num:$WORK_THREAD_NUM" + if [ ! -z $NEED_CONVERT_VIP_TO_NAME ] + then + echo "obproxy NEED_CONVERT_VIP_TO_NAME: $NEED_CONVERT_VIP_TO_NAME" + OBPROXY_OPT_LOCAL="${OBPROXY_OPT_LOCAL},need_convert_vip_to_tname=true" + fi + if [ ! -z $OBPROXY_EXTRA_OPT ] then echo "obproxy OBPROXY_EXTRA_OPT: $OBPROXY_EXTRA_OPT" - OBPROXY_OPT_LOCAL=",$OBPROXY_EXTRA_OPT" + OBPROXY_OPT_LOCAL="${OBPROXY_OPT_LOCAL},$OBPROXY_EXTRA_OPT" fi + + OBPROXY_OPT_LOCAL=",enable_cached_server=false,enable_get_rslist_remote=true,monitor_stat_dump_interval=1s,enable_qos=true,enable_standby=false,query_digest_time_threshold=2ms,monitor_cost_ms_unit=true,enable_strict_kernel_release=false,enable_proxy_scramble=true,work_thread_num=$WORK_THREAD_NUM,proxy_mem_limited='2G',log_dir_size_threshold=10G${OBPROXY_OPT_LOCAL}" } # change to the path where this script locates. @@ -134,7 +183,7 @@ function start() exit 0 fi - (cd $OBPROXY_ROOT; nohup ./bin/obproxyd.sh -c checkalive $OBPROXY_APP_NAME_ARG $OBPROXY_IDC_NAME_ARG > /dev/null 2>&1 &) + (cd $OBPROXY_ROOT; nohup ./bin/obproxyd.sh -c checkalive -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG $OBPROXY_IDC_NAME_ARG $ENABLE_ROOT_SERVER_ARG $ROOT_SERVER_LIST_ARG $ROOT_SERVER_CLUSTER_NAME_ARG $OBPROXY_CONFIG_SERVER_URL_ARG > /dev/null 2>&1 &) success_msg "obproxy started" } @@ -193,6 +242,7 @@ function status() function checkalive() { + is_first=true; while [ 1 ] do restart=0 @@ -200,7 +250,7 @@ function checkalive() if [ $? -ge 1 ] then is_obproxy_exists - if [ $? -eq 1 ] + if [ $? -ge 1 ] then is_obproxy_healthy if [ $? -ne 0 ] @@ -218,13 +268,24 @@ function checkalive() if [ $restart -eq 1 ] then echo "ObProxy will running ..." - if [ -z "$OBPROXY_IDC_NAME" ]; + if [ x$is_first == xtrue ]; then - ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -o enable_get_rslist_remote=true,monitor_stat_dump_interval='1s',enable_qos=true,enable_standby=false,query_digest_time_threshold='2ms',monitor_cost_ms_unit=true,enable_strict_kernel_release=false,obproxy_config_server_url=''$OBPROXY_CONFIG_SERVER_URL'',enable_proxy_scramble=true,work_thread_num=$WORK_THREAD_NUM,proxy_mem_limited='2G',log_dir_size_threshold=10G"$OBPROXY_OPT_LOCAL") + if [ x$ENABLE_ROOT_SERVER == xtrue ]; + then + ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -c $ROOT_SERVER_CLUSTER_NAME -r "$ROOT_SERVER_LIST" -o obproxy_config_server_url=''"$OBPROXY_OPT_LOCAL") + else + ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -o obproxy_config_server_url=''$OBPROXY_CONFIG_SERVER_URL''"$OBPROXY_OPT_LOCAL") + fi else - ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -o enable_get_rslist_remote=true,monitor_stat_dump_interval='1s',enable_qos=true,enable_standby=false,query_digest_time_threshold='2ms',monitor_cost_ms_unit=true,enable_strict_kernel_release=false,obproxy_config_server_url=''$OBPROXY_CONFIG_SERVER_URL'',enable_proxy_scramble=true,work_thread_num=$WORK_THREAD_NUM,proxy_mem_limited='2G',log_dir_size_threshold=10G,proxy_idc_name=''$OBPROXY_IDC_NAME''"$OBPROXY_OPT_LOCAL") + if [ x$ENABLE_ROOT_SERVER == xtrue ]; + then + ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -c $ROOT_SERVER_CLUSTER_NAME -r "$ROOT_SERVER_LIST" -o obproxy_config_server_url='') + else + ($OBPROXY_ROOT/bin/obproxy -p $OBPROXY_PORT $OBPROXY_APP_NAME_ARG -o obproxy_config_server_url=''$OBPROXY_CONFIG_SERVER_URL'') + fi fi fi + is_first=false; sleep 1 done } @@ -258,7 +319,7 @@ function kill_all_checkalive() } -while getopts c:n:i:ah opt +while getopts c:n:i:ahp:t:rs:u: opt do case $opt in c) @@ -273,6 +334,21 @@ do a) STATUS_ALL_OBPROXY='1' ;; + p) + OBPROXY_PORT=$OPTARG + ;; + t) + ROOT_SERVER_LIST=$OPTARG + ;; + r) + ENABLE_ROOT_SERVER='true' + ;; + s) + ROOT_SERVER_CLUSTER_NAME=$OPTARG + ;; + u) + OBPROXY_CONFIG_SERVER_URL=$OPTARG + ;; h) help exit 0 diff --git a/src/common/ob_accuracy.h b/src/common/ob_accuracy.h index ae382d3be7499e46852889d6a6e76ed8a0ca0c80..1c724a16026d8c8cdd0f220722208cfd3b735c6b 100644 --- a/src/common/ob_accuracy.h +++ b/src/common/ob_accuracy.h @@ -32,12 +32,15 @@ public: ~ObAccuracy() {} explicit ObAccuracy(ObLength length) { set_length(length); } ObAccuracy(ObPrecision precision, ObScale scale) { set_precision(precision); set_scale(scale); } + ObAccuracy(ObLength length, ObPrecision precision, ObScale scale) + { set_length(length); set_precision(precision); set_scale(scale); } ObAccuracy(const ObAccuracy &other) { accuracy_ = other.accuracy_; } OB_INLINE void set_accuracy(const ObAccuracy &accuracy) { accuracy_ = accuracy.accuracy_; } OB_INLINE void set_accuracy(const int64_t &accuracy) { accuracy_ = accuracy; } OB_INLINE void set_length(ObLength length) { length_ = length; } OB_INLINE void set_precision(ObPrecision precision) { precision_ = precision; } OB_INLINE void set_scale(ObScale scale) { scale_ = scale; } + // get union data OB_INLINE int64_t get_accuracy() const { return accuracy_; } // get detail data diff --git a/src/common/ob_field.h b/src/common/ob_field.h index 400d37a02a5b34df806768e88f4da775dbd411c0..2f91713cfd8e3b376097e2f6cde2de6ddd9de599 100644 --- a/src/common/ob_field.h +++ b/src/common/ob_field.h @@ -45,7 +45,24 @@ struct ObField { } - int64_t to_string(char *buffer, int64_t length) const; + int64_t to_string(char *buffer, int64_t length) const + { + int64_t pos = 0; + databuff_printf(buffer, length, pos, + "dname:%.*s, tname: %.*s, org_tname: %.*s, " + "cname: %.*s, org_cname: %.*s, type: %s, " + "charset: %hu, " + "decimal_scale: %hu, flags: %x", + dname_.length(), dname_.ptr(), + tname_.length(), tname_.ptr(), + org_tname_.length(), org_tname_.ptr(), + cname_.length(), cname_.ptr(), + org_cname_.length(), org_cname_.ptr(), + to_cstring(type_), + charsetnr_, accuracy_.get_scale(), flags_); + return pos; + } + int deep_copy(const ObField &other, ObIAllocator *allocator); static int get_field_mb_length(const ObObjType type, const ObAccuracy &accuracy, diff --git a/src/common/ob_obj_cast.cpp b/src/common/ob_obj_cast.cpp index 99eafd9cac539515beea8d532c0dd9298c592af0..083b019fcd46b0969c60058b0affcf0b0181199e 100644 --- a/src/common/ob_obj_cast.cpp +++ b/src/common/ob_obj_cast.cpp @@ -47,11 +47,21 @@ static int not_support(const ObObjType expect_type, const ObCastMode cast_mode) { UNUSED(params); - LOG_WARN("not supported obj type convert", - K(expect_type), K(in), K(out), K(cast_mode)); + LOG_WARN("not supported obj type convert" , K(expect_type), K(in), K(out), K(cast_mode)); return OB_NOT_SUPPORTED; } +static int not_expected(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj& in, + ObObj& out, + const ObCastMode cast_mode) +{ + UNUSED(params); + LOG_WARN("not expected obj type convert", K(expect_type), K(in), K(out), K(cast_mode)); + return OB_ERR_UNEXPECTED; +} + static int unknown_other(const ObObjType expect_type, ObObjCastParams ¶ms, const ObObj &in, @@ -171,6 +181,47 @@ static int check_convert_str_err(const char *str, return ret; } +static int convert_string_collation(const ObString &in, + const ObCollationType in_collation, + ObString &out, + const ObCollationType out_collation, + ObObjCastParams ¶ms) +{ + int ret = OB_SUCCESS; + + if (!ObCharset::is_valid_collation(in_collation) + || !ObCharset::is_valid_collation(out_collation) + || ObCharset::charset_type_by_coll(in_collation) == CHARSET_BINARY + || ObCharset::charset_type_by_coll(out_collation) == CHARSET_BINARY + || (ObCharset::charset_type_by_coll(in_collation) == ObCharset::charset_type_by_coll(out_collation))) { + out = in; + } else if (in.empty()) { + out.reset(); + } else { + char* buf = NULL; + const int32_t CharConvertFactorNum = 4; + int32_t buf_len = in.length() * CharConvertFactorNum; + uint32_t result_len = 0; + if (OB_ISNULL(buf = static_cast(params.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret)); + } else if (OB_FAIL(ObCharset::charset_convert(in_collation, + in.ptr(), + in.length(), + out_collation, + buf, + buf_len, + result_len))) { + LOG_WARN("charset convert failed", K(ret)); + } else { + out.assign_ptr(buf, result_len); + } + } + LOG_DEBUG("convert_string_collation", K(in.length()), K(in_collation), K(out.length()), K(out_collation)); + + return ret; +} + //////////////////////////////////////////////////////////////// OB_INLINE int get_cast_ret(const ObCastMode cast_mode, @@ -218,6 +269,8 @@ OB_INLINE int get_cast_ret(const ObCastMode cast_mode, #define SET_RES_DATE(res) SET_RES_OBJ(res, date, , , value, ObTimeConverter::ZERO_DATE) #define SET_RES_TIME(res) SET_RES_OBJ(res, time, , , value, ObTimeConverter::ZERO_TIME) #define SET_RES_YEAR(res) SET_RES_OBJ(res, year, , , value, ObTimeConverter::ZERO_YEAR) +#define SET_RES_OTIMESTAMP(res) SET_RES_OBJ(res, otimestamp_value, expect_type, COMMA, value, ObOTimestampData()) + #define SET_RES_ACCURACY(res_precision, res_scale, res_length) \ if (params.res_accuracy_ != NULL && OB_SUCCESS == ret) {\ @@ -225,6 +278,18 @@ OB_INLINE int get_cast_ret(const ObCastMode cast_mode, params.res_accuracy_->set_precision(res_precision);\ params.res_accuracy_->set_length(res_length);\ } + +#define SET_RES_ACCURACY_STRING(type, res_precision, res_length) \ + if (params.res_accuracy_ != NULL && OB_SUCCESS == ret) { \ + params.res_accuracy_->set_precision(res_precision); \ + params.res_accuracy_->set_length(res_length); \ + if (ob_is_text_tc(type)) { \ + params.res_accuracy_->set_scale(DEFAULT_SCALE_FOR_TEXT); \ + } else { \ + params.res_accuracy_->set_scale(DEFAULT_SCALE_FOR_STRING); \ + } \ + } + //////////////////////////////////////////////////////////////// // range check function templates. @@ -419,6 +484,354 @@ ObPrecision get_precision_for_integer(T value) return static_cast((iter - (bound_info + BOUND_INFO_START_POS)) * flag + 1); } +int ObHexUtils::unhex(const ObString &text, ObCastCtx &cast_ctx, ObObj &result) +{ + int ret = OB_SUCCESS; + ObString str_result; + char *buf = NULL; + const bool need_fill_zero = (1 == text.length() % 2); + const int32_t tmp_length = text.length() / 2 + need_fill_zero; + int32_t alloc_length = (0 == tmp_length ? 1 : tmp_length); + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret), K(text)); + } else if (OB_ISNULL(buf = static_cast(cast_ctx.allocator_v2_->alloc(alloc_length)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(alloc_length), K(ret)); + } else { + int32_t i = 0; + char c1 = 0; + char c2 = 0; + if (text.length() > 0) { + if (need_fill_zero) { + c1 = '0'; + c2 = text[0]; + i = 0; + } else { + c1 = text[0]; + c2 = text[1]; + i = 1; + } + } + while (OB_SUCC(ret) && i < text.length()) { + if (isxdigit(c1) && isxdigit(c2)) { + buf[i / 2] = (char)((get_xdigit(c1) << 4) | get_xdigit(c2)); + c1 = text[++i]; + c2 = text[++i]; + } else { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(c1), K(c2), K(text)); + } + } + + if (OB_SUCC(ret)) { + str_result.assign_ptr(buf, tmp_length); + result.set_varchar(str_result); + } + } + return ret; +} + +int ObHexUtils::hex(const ObString &text, ObCastCtx &cast_ctx, ObObj &result) +{ + int ret = OB_SUCCESS; + ObString str_result; + char* buf = NULL; + const int32_t alloc_length = text.empty() ? 1 : text.length() * 2; + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret), K(text)); + } else if (OB_ISNULL(buf = static_cast(cast_ctx.allocator_v2_->alloc(alloc_length)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret), K(alloc_length)); + } else { + static const char* HEXCHARS = "0123456789ABCDEF"; + int32_t pos = 0; + for (int32_t i = 0; i < text.length(); ++i) { + buf[pos++] = HEXCHARS[text[i] >> 4 & 0xF]; + buf[pos++] = HEXCHARS[text[i] & 0xF]; + } + str_result.assign_ptr(buf, pos); + result.set_varchar(str_result); + LOG_DEBUG("succ to hex", K(text), "length", text.length(), K(str_result)); + } + return ret; +} + +int ObHexUtils::hex_for_mysql(const uint64_t uint_val, common::ObCastCtx &cast_ctx, common::ObObj &result) +{ + + int ret = OB_SUCCESS; + char* buf = NULL; + const int32_t MAX_INT64_LEN = 20; + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret), K(uint_val)); + } else if (OB_ISNULL(buf = static_cast(cast_ctx.allocator_v2_->alloc(MAX_INT64_LEN)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret)); + } else { + int pos = snprintf(buf, MAX_INT64_LEN, "%lX", uint_val); + if (OB_UNLIKELY(pos <= 0) || OB_UNLIKELY(pos >= MAX_INT64_LEN)) { + ret = OB_SIZE_OVERFLOW; + LOG_ERROR("size is overflow", K(ret), K(uint_val)); + } else { + ObString str_result(pos, buf); + result.set_varchar(str_result); + } + } + return ret; +} + +// https://docs.oracle.com/en/database/oracle/oracle-database/12.2/sqlrf/RAWTOHEX.html +// As a SQL built-in function, RAWTOHEX accepts an argument of any scalar data type other than LONG, +// LONG RAW, CLOB, NCLOB, BLOB, or BFILE. If the argument is of a data type other than RAW, +// then this function converts the argument value, which is represented using some number of data bytes, +// into a RAW value with the same number of data bytes. The data itself is not modified in any way, +// but the data type is recast to a RAW data type. +int ObHexUtils::rawtohex(const ObObj &text, ObCastCtx &cast_ctx, ObObj &result) +{ + int ret = OB_SUCCESS; + + if (text.is_null()) { + result.set_null(); + } else { + ObString str; + ObObj num_obj; + char* splice_num_str = NULL; // for splice Desc and degits_ of number. + ObOTimestampData time_value; + switch (text.get_type()) { + // TODO::this should same as oracle, and support dump func + case ObTinyIntType: + case ObSmallIntType: + case ObInt32Type: + case ObIntType: { + int64_t int_value = text.get_int(); + number::ObNumber nmb; + if (OB_FAIL(nmb.from(int_value, cast_ctx))) { + LOG_WARN("fail to int_number", K(ret), K(int_value), "type", text.get_type()); + } else { + num_obj.set_number(ObNumberType, nmb); + int32_t alloc_len = + static_cast(sizeof(num_obj.get_number_desc()) + num_obj.get_number_byte_length()); + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret)); + } else if (OB_ISNULL(splice_num_str = static_cast(cast_ctx.allocator_v2_->alloc(alloc_len)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret), K(alloc_len)); + } else { + MEMCPY(splice_num_str, &(num_obj.get_number_desc()), sizeof(num_obj.get_number_desc())); + MEMCPY(splice_num_str + sizeof(num_obj.get_number_desc()), + num_obj.get_data_ptr(), + num_obj.get_number_byte_length()); + str.assign_ptr(static_cast(splice_num_str), alloc_len); + } + LOG_DEBUG("succ to int_number", K(ret), K(int_value), "type", num_obj.get_type(), K(nmb), K(str)); + } + break; + } + case ObNumberFloatType: + case ObNumberType: { + int32_t alloc_len = static_cast(sizeof(text.get_number_desc()) + text.get_number_byte_length()); + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret)); + } else if (OB_ISNULL(splice_num_str = static_cast(cast_ctx.allocator_v2_->alloc(alloc_len)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret), K(alloc_len)); + } else { + MEMCPY(splice_num_str, &(text.get_number_desc()), sizeof(text.get_number_desc())); + MEMCPY(splice_num_str + sizeof(text.get_number_desc()), text.get_data_ptr(), text.get_number_byte_length()); + str.assign_ptr(static_cast(splice_num_str), alloc_len); + } + break; + } + case ObDateTimeType: { + str.assign_ptr(static_cast(text.get_data_ptr()), static_cast(sizeof(int64_t))); + break; + } + case ObNVarchar2Type: + case ObNCharType: + case ObVarcharType: + case ObCharType: + case ObLongTextType: + case ObRawType: { + // https://www.techonthenet.com/oracle/functions/rawtohex.php + // NOTE:: when convert string to raw, Oracle use utl_raw.cast_to_raw(), while PL/SQL use hextoraw() + // here we use utl_raw.cast_to_raw(), as we can not distinguish in which SQL + str = text.get_varbinary(); + break; + } + case ObTimestampTZType: + case ObTimestampLTZType: + case ObTimestampNanoType: { + time_value = text.get_otimestamp_value(); + str.assign_ptr(reinterpret_cast(&time_value), static_cast(text.get_otimestamp_store_size())); + break; + } + default: { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(text), "type", text.get_type()); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(hex(str, cast_ctx, result))) { + LOG_WARN("fail to convert to hex", K(ret), K(str)); + } else { + result.set_default_collation_type(); + LOG_DEBUG("succ to rawtohex", "type", text.get_type(), K(text), K(result), K(lbt())); + } + } + } + + return ret; +} + +int ObHexUtils::hextoraw(const ObObj &text, ObCastCtx &cast_ctx, ObObj &result) +{ + int ret = OB_SUCCESS; + if (text.is_null()) { + result.set_null(); + } else if (text.is_numeric_type()) { + number::ObNumber nmb_val; + if (OB_FAIL(get_uint(text, cast_ctx, nmb_val))) { + LOG_WARN("fail to get uint64", K(ret), K(text)); + } else if (OB_FAIL(uint_to_raw(nmb_val, cast_ctx, result))) { + LOG_WARN("fail to convert to hex", K(ret), K(nmb_val)); + } + } else if (text.is_raw()) { + // fast path + if (OB_FAIL(copy_raw(text, cast_ctx, result))) { + LOG_WARN("fail to convert to hex", K(ret), K(text)); + } + } else if (text.is_character_type() || text.is_varbinary_or_binary()) { + ObString utf8_string; + if (OB_FAIL(convert_string_collation(text.get_string(), + text.get_collation_type(), + utf8_string, + ObCharset::get_system_collation(), + cast_ctx))) { + LOG_WARN("convert_string_collation", K(ret)); + } else if (OB_FAIL(unhex(utf8_string, cast_ctx, result))) { + LOG_WARN("fail to convert to hex", K(ret), K(text)); + } else { + result.set_raw(result.get_raw()); + } + } else { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(text)); + } + + return ret; +} + +int ObHexUtils::get_uint(const ObObj &obj, ObCastCtx &cast_ctx, number::ObNumber &out) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ob_is_accurate_numeric_type(obj.get_type()))) { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(obj)); + } else if (obj.is_number() || obj.is_unumber()) { + const number::ObNumber& value = obj.get_number(); + if (OB_FAIL(out.from(value, cast_ctx))) { + LOG_WARN("deep copy failed", K(ret), K(obj)); + } else if (OB_UNLIKELY(!out.is_integer()) || OB_UNLIKELY(out.is_negative())) { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(out)); + } else if (OB_FAIL(out.round(0))) { + LOG_WARN("round failed", K(ret), K(out)); + } + } else { + if (OB_UNLIKELY(obj.get_int() < 0)) { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(obj)); + } else if (OB_FAIL(out.from(obj.get_int(), cast_ctx))) { + LOG_WARN("deep copy failed", K(ret), K(obj)); + } + } + return ret; +} + +int ObHexUtils::uint_to_raw(const number::ObNumber &uint_num, ObCastCtx &cast_ctx, ObObj &result) +{ + int ret = OB_SUCCESS; + const int64_t oracle_max_avail_len = 40; + char uint_buf[number::ObNumber::MAX_TOTAL_SCALE] = {0}; + int64_t uint_pos = 0; + ObString uint_str; + if (OB_FAIL(uint_num.format(uint_buf, number::ObNumber::MAX_TOTAL_SCALE, uint_pos, 0))) { + LOG_WARN("fail to format ", K(ret), K(uint_num)); + } else if (uint_pos > oracle_max_avail_len) { + ret = OB_ERR_INVALID_HEX_NUMBER; + LOG_WARN("invalid hex number", K(ret), K(uint_pos), K(oracle_max_avail_len), K(uint_num)); + } else { + uint_str.assign_ptr(uint_buf, static_cast(uint_pos)); + if (OB_FAIL(unhex(uint_str, cast_ctx, result))) { + LOG_WARN("fail to str_to_raw", K(ret), K(result)); + } else { + result.set_raw(result.get_raw()); + } + } + return ret; +} + +int ObHexUtils::copy_raw(const common::ObObj &obj, common::ObCastCtx &cast_ctx, common::ObObj &result) +{ + int ret = OB_SUCCESS; + char* buf = NULL; + const ObString& value = obj.get_raw(); + const int32_t alloc_length = value.empty() ? 1 : value.length(); + if (OB_ISNULL(cast_ctx.allocator_v2_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("allocator in cast ctx is NULL", K(ret)); + } else if (OB_ISNULL(buf = static_cast(cast_ctx.allocator_v2_->alloc(alloc_length)))) { + result.set_null(); + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("alloc memory failed", K(ret), K(alloc_length)); + } else { + MEMCPY(buf, value.ptr(), value.length()); + result.set_raw(buf, value.length()); + } + return ret; +} + +static int check_convert_string(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out) +{ + int ret = OB_SUCCESS; + if (lib::is_oracle_mode() && ob_is_blob(expect_type, params.expect_obj_collation_) && !in.is_blob() && !in.is_raw()) { + if (in.is_varchar_or_char()) { + if (OB_FAIL(ObHexUtils::hextoraw(in, params, out))) { + LOG_WARN("fail to hextoraw for blob", K(ret), K(in)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("Invalid use of blob type", K(ret), K(in), K(expect_type)); + } + } else { + out = in; + } + return ret; +} + +static int check_convert_string(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObString &in_string, + ObObj &out) +{ + ObObj tmp_obj; + tmp_obj.set_varchar(in_string); + return check_convert_string(expect_type, params, tmp_obj, out); +} + //////////////////////////////////////////////////////////////// // Int -> XXX @@ -546,7 +959,7 @@ static int int_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; if (in.get_int() < 0) { CAST_FAIL(OB_INVALID_DATE_FORMAT); @@ -1219,7 +1632,7 @@ static int double_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, K(ret), K(in), K(expect_type)); } else { int64_t value = 0; - const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.dtc_params_.tz_info_ : NULL; char buf[MAX_DOUBLE_PRINT_SIZE]; MEMSET(buf, 0, MAX_DOUBLE_PRINT_SIZE); int64_t length = ob_gcvt_opt(in.get_double(), OB_GCVT_ARG_DOUBLE, (int)sizeof(buf) - 1, buf, NULL, lib::is_oracle_mode()); @@ -1469,7 +1882,7 @@ static int number_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; int64_t int_part = 0; int64_t dec_part = 0; @@ -1608,7 +2021,7 @@ static int datetime_int(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; if (OB_FAIL(ObTimeConverter::datetime_to_int(in.get_datetime(), tz_info, value))) { } else if (expect_type < ObIntType && CAST_FAIL(int_range_check(expect_type, value, value))) { @@ -1632,7 +2045,7 @@ static int datetime_uint(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; int64_t int64 = 0; if (OB_FAIL(ObTimeConverter::datetime_to_int(in.get_datetime(), tz_info, int64))) { } else { @@ -1658,7 +2071,7 @@ static int datetime_float(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; double value = 0.0; if (OB_FAIL(ObTimeConverter::datetime_to_double(in.get_datetime(), tz_info, value))) { } else { @@ -1682,7 +2095,7 @@ static int datetime_double(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; double value = 0.0; if (OB_FAIL(ObTimeConverter::datetime_to_double(in.get_datetime(), tz_info, value))) { } else { @@ -1706,7 +2119,7 @@ static int datetime_number(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; char buf[OB_CAST_TO_VARCHAR_MAX_LENGTH]; MEMSET(buf, 0, OB_CAST_TO_VARCHAR_MAX_LENGTH); int64_t len = 0; @@ -1735,9 +2148,9 @@ static int datetime_datetime(const ObObjType expect_type, ObObjCastParams ¶m } else { int64_t value = in.get_datetime(); if (ObDateTimeType == in.get_type() && ObTimestampType == expect_type) { - ret = ObTimeConverter::datetime_to_timestamp(in.get_datetime(), params.tz_info_, value); + ret = ObTimeConverter::datetime_to_timestamp(in.get_datetime(), params.dtc_params_.tz_info_, value); } else if (ObTimestampType == in.get_type() && ObDateTimeType == expect_type) { - ret = ObTimeConverter::timestamp_to_datetime(in.get_datetime(), params.tz_info_, value); + ret = ObTimeConverter::timestamp_to_datetime(in.get_datetime(), params.dtc_params_.tz_info_, value); } if (OB_FAIL(ret)) { @@ -1760,7 +2173,7 @@ static int datetime_date(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; int32_t value = 0; if (OB_FAIL(ObTimeConverter::datetime_to_date(in.get_datetime(), tz_info, value))) { } else { @@ -1782,7 +2195,7 @@ static int datetime_time(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; if (OB_FAIL(ObTimeConverter::datetime_to_time(in.get_datetime(), tz_info, value))) { } else { @@ -1804,7 +2217,7 @@ static int datetime_year(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; uint8_t value = 0; if (CAST_FAIL(ObTimeConverter::datetime_to_year(in.get_datetime(), tz_info, value))) { } else { @@ -1828,7 +2241,7 @@ static int datetime_string(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == in.get_type()) ? params.dtc_params_.tz_info_ : NULL; char buf[OB_CAST_TO_VARCHAR_MAX_LENGTH]; MEMSET(buf, 0, OB_CAST_TO_VARCHAR_MAX_LENGTH); int64_t len = 0; @@ -1848,6 +2261,46 @@ static int datetime_string(const ObObjType expect_type, ObObjCastParams ¶ms, return ret; } +static int datetime_otimestamp(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out, + const ObCastMode cast_mode) +{ + UNUSED(cast_mode); + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(ObDateTimeTC != in.get_type_class()) + || OB_UNLIKELY(ObOTimestampTC != ob_obj_type_class(expect_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); + } else { + int64_t dt_value = 0; + if (ObTimestampType == in.get_type()) { + int64_t utc_value = in.get_timestamp(); + if (OB_FAIL(ObTimeConverter::timestamp_to_datetime(utc_value, params.dtc_params_.tz_info_, dt_value))) { + LOG_WARN("fail to convert timestamp to datetime", K(ret)); + } + } else { + dt_value = in.get_datetime(); + } + if (OB_SUCC(ret)) { + int64_t odate_value = 0; + ObOTimestampData value; + ObTimeConverter::datetime_to_odate(dt_value, odate_value); + if (OB_FAIL(ObTimeConverter::odate_to_otimestamp(odate_value, params.dtc_params_.tz_info_, expect_type, value))) { + LOG_WARN("fail to odate to otimestamp", K(ret), K(in), K(expect_type)); + } else { + SET_RES_OTIMESTAMP(out); + } + } + } + SET_RES_ACCURACY(DEFAULT_PRECISION_FOR_TEMPORAL, in.get_scale(), DEFAULT_LENGTH_FOR_TEMPORAL); + + return ret; +} + + //////////////////////////////////////////////////////////// // Date -> XXX @@ -1962,7 +2415,7 @@ static int date_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; if (OB_FAIL(ObTimeConverter::date_to_datetime(in.get_date(), tz_info, value))) { } else { @@ -2155,7 +2608,7 @@ static int time_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; - const ObTimeZoneInfo *tz_info = params.tz_info_; + const ObTimeZoneInfo *tz_info = params.dtc_params_.tz_info_; int64_t value = 0; if (OB_UNLIKELY(ObTimeTC != in.get_type_class() || ObDateTimeTC != ob_obj_type_class(expect_type))) { @@ -2529,7 +2982,7 @@ static int string_datetime(const ObObjType expect_type, ObObjCastParams ¶ms, LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); } else { - const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.tz_info_ : NULL; + const ObTimeZoneInfo *tz_info = (ObTimestampType == expect_type) ? params.dtc_params_.tz_info_ : NULL; int64_t value = 0; if (CAST_FAIL(ObTimeConverter::str_to_datetime(in.get_string(), tz_info, value, &res_scale))) { } else { @@ -2616,6 +3069,8 @@ static int string_string(const ObObjType expect_type, ObObjCastParams ¶ms, if (0 != str.length() && CS_TYPE_BINARY != in.get_collation_type() && CS_TYPE_BINARY != params.dest_collation_ + && CS_TYPE_INVALID != in.get_collation_type() + && CS_TYPE_INVALID != params.dest_collation_ && (ObCharset::charset_type_by_coll(in.get_collation_type()) != ObCharset::charset_type_by_coll(params.dest_collation_))) { char *buf = NULL; @@ -2660,245 +3115,728 @@ static int string_string(const ObObjType expect_type, ObObjCastParams ¶ms, return ret; } +static int string_otimestamp(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out, + const ObCastMode cast_mode) +{ + UNUSED(cast_mode); + int ret = OB_SUCCESS; + ObScale res_scale = -1; + ObString utf8_string; + + if (OB_UNLIKELY(ObStringTC != in.get_type_class() && ObTextTC != in.get_type_class()) + || OB_UNLIKELY(ObOTimestampTC != ob_obj_type_class(expect_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); + } else if (lib::is_oracle_mode() && in.is_blob()) { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("invalid use of blob type", K(ret), K(in), K(expect_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Cast to blob type"); + } else if (OB_FAIL(convert_string_collation(in.get_string(), + in.get_collation_type(), + utf8_string, + ObCharset::get_system_collation(), + params))) { + LOG_WARN("convert_string_collation", K(ret)); + } else { + ObOTimestampData value; + ObTimeConvertCtx cvrt_ctx(params.dtc_params_.tz_info_, true); + cvrt_ctx.oracle_nls_format_ = params.dtc_params_.get_nls_format(expect_type); + if (CAST_FAIL(ObTimeConverter::str_to_otimestamp(utf8_string, cvrt_ctx, expect_type, value, res_scale))) { + } else { + SET_RES_OTIMESTAMP(out); + } + } + + SET_RES_ACCURACY(DEFAULT_PRECISION_FOR_TEMPORAL, res_scale, DEFAULT_LENGTH_FOR_TEMPORAL); + return ret; +} + +//////////////////////////////////////////////////////////// +// OTimestamp -> XXX + +static int otimestamp_datetime(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out, + const ObCastMode cast_mode) +{ + UNUSED(cast_mode); + int ret = OB_SUCCESS; + + int64_t usec = 0; + if (OB_UNLIKELY(ObOTimestampTC != in.get_type_class()) + || OB_UNLIKELY(ObDateTimeTC != ob_obj_type_class(expect_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); + } else if (OB_FAIL(ObTimeConverter::otimestamp_to_odate(in.get_type(), + in.get_otimestamp_value(), + params.dtc_params_.tz_info_, + usec))) { + LOG_WARN("fail to timestamp_tz_to_timestamp", K(ret), K(in), K(expect_type)); + } else { + ObTimeConverter::trunc_datetime(OB_MAX_DATE_PRECISION, usec); + out.set_datetime(expect_type, usec); + out.set_scale(OB_MAX_DATE_PRECISION); + } + SET_RES_ACCURACY(DEFAULT_PRECISION_FOR_TEMPORAL, in.get_scale(), DEFAULT_LENGTH_FOR_TEMPORAL); + + return ret; +} + +static int otimestamp_string(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out, + const ObCastMode cast_mode) +{ + UNUSED(cast_mode); + int ret = OB_SUCCESS; + ObLength res_length = -1; + + if (OB_UNLIKELY(ObOTimestampTC != in.get_type_class()) + || OB_UNLIKELY(ObStringTC != ob_obj_type_class(expect_type) && ObTextTC != ob_obj_type_class(expect_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); + } else { + char buf[OB_CAST_TO_VARCHAR_MAX_LENGTH] = {0}; + int64_t len = 0; + if (OB_FAIL(ObTimeConverter::otimestamp_to_str(in.get_otimestamp_value(), + params.dtc_params_, + in.get_scale(), + in.get_type(), + buf, + OB_CAST_TO_VARCHAR_MAX_LENGTH, + len))) { + LOG_WARN("failed to convert otimestamp to string", K(ret)); + } else { + ObString tmp_str; + ObObj tmp_out; + if (OB_FAIL(convert_string_collation(ObString(len, buf), + ObCharset::get_system_collation(), + tmp_str, + params.dest_collation_, + params))) { + LOG_WARN("fail to convert string collation", K(ret)); + } else if (OB_FAIL(check_convert_string(expect_type, params, tmp_str, tmp_out))) { + LOG_WARN("fail to check_convert_string", K(ret), K(in), K(expect_type)); + } else if (OB_FAIL(copy_string(params, expect_type, tmp_out.get_string(), out))) { + LOG_WARN("failed to copy_string", K(ret), K(expect_type), K(len)); + } else { + out.set_type(expect_type); + res_length = static_cast(out.get_string_len()); + } + } + } + SET_RES_ACCURACY_STRING(expect_type, DEFAULT_PRECISION_FOR_STRING, res_length); + + return ret; +} + +static int otimestamp_otimestamp(const ObObjType expect_type, + ObObjCastParams ¶ms, + const ObObj &in, + ObObj &out, + const ObCastMode cast_mode) +{ + UNUSED(cast_mode); + int ret = OB_SUCCESS; + ObOTimestampData value; + + if (OB_UNLIKELY(ObOTimestampTC != in.get_type_class()) + || OB_UNLIKELY(ObOTimestampTC != ob_obj_type_class(expect_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("invalid input type", K(ret), K(in), K(expect_type)); + } else if (ObTimestampNanoType == in.get_type()) { + if (OB_FAIL(ObTimeConverter::odate_to_otimestamp(in.get_otimestamp_value().time_us_, + params.dtc_params_.tz_info_, + expect_type, + value))) { + LOG_WARN("fail to odate_to_otimestamp", K(ret), K(expect_type)); + } else { + value.time_ctx_.tail_nsec_ = in.get_otimestamp_value().time_ctx_.tail_nsec_; + } + } else if (ObTimestampNanoType == expect_type) { + if (OB_FAIL(ObTimeConverter::otimestamp_to_odate(in.get_type(), + in.get_otimestamp_value(), + params.dtc_params_.tz_info_, + *(int64_t*)&value.time_us_))) { + LOG_WARN("fail to otimestamp_to_odate", K(ret), K(expect_type)); + } else { + value.time_ctx_.tail_nsec_ = in.get_otimestamp_value().time_ctx_.tail_nsec_; + } + } else { + if (OB_FAIL(ObTimeConverter::otimestamp_to_otimestamp(in.get_type(), + in.get_otimestamp_value(), + params.dtc_params_.tz_info_, + expect_type, + value))) { + LOG_WARN("fail to otimestamp_to_otimestamp", K(ret), K(expect_type)); + } + } + + if (OB_SUCC(ret)) { + SET_RES_OTIMESTAMP(out); + } + SET_RES_ACCURACY(DEFAULT_PRECISION_FOR_TEMPORAL, in.get_scale(), DEFAULT_LENGTH_FOR_TEMPORAL); + + return ret; +} + + ObObjCastFunc OB_OBJ_CAST[ObMaxTC][ObMaxTC] = { { /*null -> XXX*/ - identity,/*null*/ - identity,/*int*/ - identity,/*uint*/ - identity,/*float*/ - identity,/*double*/ - identity,/*number*/ - identity,/*datetime*/ - identity,/*date*/ - identity,/*time*/ - identity,/*year*/ - identity,/*string*/ - identity,/*extend*/ - identity,/*unknown*/ - identity,/*text*/ + identity, /*null*/ + identity, /*int*/ + identity, /*uint*/ + identity, /*float*/ + identity, /*double*/ + identity, /*number*/ + identity, /*datetime*/ + identity, /*date*/ + identity, /*time*/ + identity, /*year*/ + identity, /*string*/ + identity, /*extend*/ + identity, /*unknown*/ + identity, /*text*/ + not_support, /*bit*/ + not_support, /*enumset*/ + not_support, /*enumsetInner*/ + identity, /*otimestamp*/ + not_expected, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*int -> XXX*/ - not_support,/*null*/ - int_int,/*int*/ - int_uint,/*uint*/ - int_float,/*float*/ - int_double,/*double*/ - int_number,/*number*/ - int_datetime,/*datetime*/ - int_date,/*date*/ - int_time,/*time*/ - int_year,/*year*/ - int_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - int_string,/*text*/ + not_support, /*null*/ + int_int, /*int*/ + int_uint, /*uint*/ + int_float, /*float*/ + int_double, /*double*/ + int_number, /*number*/ + int_datetime, /*datetime*/ + int_date, /*date*/ + int_time, /*time*/ + int_year, /*year*/ + int_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + int_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*uint -> XXX*/ - not_support,/*null*/ - uint_int,/*int*/ - uint_uint,/*uint*/ - uint_float,/*float*/ - uint_double,/*double*/ - uint_number,/*number*/ - uint_datetime,/*datetime*/ - uint_date,/*date*/ - uint_time,/*time*/ - uint_year,/*year*/ - uint_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - uint_string,/*text*/ + not_support, /*null*/ + uint_int, /*int*/ + uint_uint, /*uint*/ + uint_float, /*float*/ + uint_double, /*double*/ + uint_number, /*number*/ + uint_datetime, /*datetime*/ + uint_date, /*date*/ + uint_time, /*time*/ + uint_year, /*year*/ + uint_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + uint_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*float -> XXX*/ - not_support,/*null*/ - float_int,/*int*/ - float_uint,/*uint*/ - float_float,/*float*/ - float_double,/*double*/ - float_number,/*number*/ - float_datetime,/*datetime*/ - float_date,/*date*/ - float_time,/*time*/ - not_support,/*year*/ - float_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - float_string,/*text*/ + not_support, /*null*/ + float_int, /*int*/ + float_uint, /*uint*/ + float_float, /*float*/ + float_double, /*double*/ + float_number, /*number*/ + float_datetime, /*datetime*/ + float_date, /*date*/ + float_time, /*time*/ + not_support, /*year*/ + float_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + float_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*double -> XXX*/ - not_support,/*null*/ - double_int,/*int*/ - double_uint,/*uint*/ - double_float,/*float*/ - double_double,/*double*/ - double_number,/*number*/ - double_datetime,/*datetime*/ - double_date,/*date*/ - double_time,/*time*/ - not_support,/*year*/ - double_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - double_string,/*text*/ + not_support, /*null*/ + double_int, /*int*/ + double_uint, /*uint*/ + double_float, /*float*/ + double_double, /*double*/ + double_number, /*number*/ + double_datetime, /*datetime*/ + double_date, /*date*/ + double_time, /*time*/ + not_support, /*year*/ + double_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + double_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*number -> XXX*/ - not_support,/*null*/ - number_int,/*int*/ - number_uint,/*uint*/ - number_float,/*float*/ - number_double,/*double*/ - number_number,/*number*/ - number_datetime,/*datetime*/ - number_date,/*date*/ - number_time,/*time*/ - number_year,/*year*/ - number_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - number_string,/*text*/ + not_support, /*null*/ + number_int, /*int*/ + number_uint, /*uint*/ + number_float, /*float*/ + number_double, /*double*/ + number_number, /*number*/ + number_datetime, /*datetime*/ + number_date, /*date*/ + number_time, /*time*/ + number_year, /*year*/ + number_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + number_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*datetime -> XXX*/ - not_support,/*null*/ - datetime_int,/*int*/ - datetime_uint,/*uint*/ - datetime_float,/*float*/ - datetime_double,/*double*/ - datetime_number,/*number*/ - datetime_datetime,/*datetime*/ - datetime_date,/*date*/ - datetime_time,/*time*/ - datetime_year,/*year*/ - datetime_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - datetime_string,/*text*/ + not_support, /*null*/ + datetime_int, /*int*/ + datetime_uint, /*uint*/ + datetime_float, /*float*/ + datetime_double, /*double*/ + datetime_number, /*number*/ + datetime_datetime, /*datetime*/ + datetime_date, /*date*/ + datetime_time, /*time*/ + datetime_year, /*year*/ + datetime_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + datetime_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + datetime_otimestamp, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*date -> XXX*/ - not_support,/*null*/ - date_int,/*int*/ - date_uint,/*uint*/ - date_float,/*float*/ - date_double,/*double*/ - date_number,/*number*/ - date_datetime,/*datetime*/ - identity,/*date*/ - date_time,/*time*/ - date_year,/*year*/ - date_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - date_string,/*text*/ + not_support, /*null*/ + date_int, /*int*/ + date_uint, /*uint*/ + date_float, /*float*/ + date_double, /*double*/ + date_number, /*number*/ + date_datetime, /*datetime*/ + identity, /*date*/ + date_time, /*time*/ + date_year, /*year*/ + date_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + date_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*time -> XXX*/ - not_support,/*null*/ - time_int,/*int*/ - time_uint,/*uint*/ - time_float,/*float*/ - time_double,/*double*/ - time_number,/*number*/ - time_datetime,/*datetime*/ - not_support,/*date*/ - identity,/*time*/ - not_support,/*year*/ - time_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - time_string,/*text*/ + not_support, /*null*/ + time_int, /*int*/ + time_uint, /*uint*/ + time_float, /*float*/ + time_double, /*double*/ + time_number, /*number*/ + time_datetime, /*datetime*/ + not_support, /*date*/ + identity, /*time*/ + not_support, /*year*/ + time_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + time_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*year -> XXX*/ - not_support,/*null*/ - year_int,/*int*/ - year_uint,/*uint*/ - year_float,/*float*/ - year_double,/*double*/ - year_number,/*number*/ - not_support,/*datetime*/ - not_support,/*date*/ - not_support,/*time*/ - identity,/*year*/ - year_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - year_string,/*text*/ + not_support, /*null*/ + year_int, /*int*/ + year_uint, /*uint*/ + year_float, /*float*/ + year_double, /*double*/ + year_number, /*number*/ + not_support, /*datetime*/ + not_support, /*date*/ + not_support, /*time*/ + identity, /*year*/ + year_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + year_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*string -> XXX*/ - not_support,/*null*/ - string_int,/*int*/ - string_uint,/*uint*/ - string_float,/*float*/ - string_double,/*double*/ - string_number,/*number*/ - string_datetime,/*datetime*/ - string_date,/*date*/ - string_time,/*time*/ - string_year,/*year*/ - string_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - string_string,/*text*/ + not_support, /*null*/ + string_int, /*int*/ + string_uint, /*uint*/ + string_float, /*float*/ + string_double, /*double*/ + string_number, /*number*/ + string_datetime, /*datetime*/ + string_date, /*date*/ + string_time, /*time*/ + string_year, /*year*/ + string_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + string_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + string_otimestamp, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*extend -> XXX*/ - not_support,/*null*/ - not_support,/*int*/ - not_support,/*uint*/ - not_support,/*float*/ - not_support,/*double*/ - not_support,/*number*/ - not_support,/*datetime*/ - not_support,/*date*/ - not_support,/*time*/ - not_support,/*year*/ - not_support,/*string*/ - identity,/*extend*/ - not_support,/*unknown*/ - not_support,/*text*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_support, /*date*/ + not_support, /*time*/ + not_support, /*year*/ + not_support, /*string*/ + identity, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_support, /*enumset*/ + not_support, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*unknown -> XXX*/ - unknown_other,/*null*/ - unknown_other,/*int*/ - unknown_other,/*uint*/ - unknown_other,/*float*/ - unknown_other,/*double*/ - unknown_other,/*number*/ - unknown_other,/*datetime*/ - unknown_other,/*date*/ - unknown_other,/*time*/ - unknown_other,/*year*/ - unknown_other,/*string*/ - unknown_other,/*extend*/ - identity,/*unknown*/ - not_support,/*text*/ + unknown_other, /*null*/ + unknown_other, /*int*/ + unknown_other, /*uint*/ + unknown_other, /*float*/ + unknown_other, /*double*/ + unknown_other, /*number*/ + unknown_other, /*datetime*/ + unknown_other, /*date*/ + unknown_other, /*time*/ + unknown_other, /*year*/ + unknown_other, /*string*/ + unknown_other, /*extend*/ + identity, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_support, /*enumset*/ + not_support, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ }, { /*text -> XXX*/ - not_support,/*null*/ - string_int,/*int*/ - string_uint,/*uint*/ - string_float,/*float*/ - string_double,/*double*/ - string_number,/*number*/ - string_datetime,/*datetime*/ - string_date,/*date*/ - string_time,/*time*/ - string_year,/*year*/ - string_string,/*string*/ - not_support,/*extend*/ - not_support,/*unknown*/ - string_string,/*text*/ + not_support, /*null*/ + string_int, /*int*/ + string_uint, /*uint*/ + string_float, /*float*/ + string_double, /*double*/ + string_number, /*number*/ + string_datetime, /*datetime*/ + string_date, /*date*/ + string_time, /*time*/ + string_year, /*year*/ + string_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + string_string, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + string_otimestamp, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ + }, + { + /*bit -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_support, /*date*/ + not_support, /*time*/ + not_support, /*year*/ + not_support, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ + }, + { + /*enumset -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_expected, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_support, /*year*/ + not_expected, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_expected, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_expected, /*lob*/ + }, + { + /*enumset_inner -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_support, /*date*/ + not_support, /*time*/ + not_support, /*year*/ + not_support, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_expected, /*interval*/ + not_expected, /*rowid*/ + not_support, /*lob*/ + }, + { + /*otimestamp -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + otimestamp_datetime, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_expected, /*year*/ + otimestamp_string, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + otimestamp_otimestamp, /*otimestamp*/ + not_support, /*raw*/ + not_support, /*interval*/ + not_support, /*rowid*/ + not_support, /*lob*/ + }, + { + /*raw -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_expected, /*year*/ + not_support, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_support, /*interval*/ + not_support, /*rowid*/ + not_support, /*lob*/ + }, + { + /*interval -> XXX*/ + not_expected, /*null*/ + not_expected, /*int*/ + not_expected, /*uint*/ + not_expected, /*float*/ + not_expected, /*double*/ + not_expected, /*number*/ + not_expected, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_expected, /*year*/ + not_support, /*string*/ + not_expected, /*extend*/ + not_expected, /*unknown*/ + not_expected, /*text*/ + not_expected, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_expected, /*otimestamp*/ + not_expected, /*raw*/ + not_support, /*interval*/ + not_support, /*rowid*/ + not_support, /*lob*/ + }, + { + /*rowid -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_expected, /*year*/ + not_support, /*string*/ + not_expected, /*extend*/ + not_expected, /*unknown*/ + not_expected, /*text*/ + not_expected, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_support, /*interval*/ + not_support, /*rowid*/ + not_support, /*lob*/ + }, + { + /*lob -> XXX*/ + not_support, /*null*/ + not_support, /*int*/ + not_support, /*uint*/ + not_support, /*float*/ + not_support, /*double*/ + not_support, /*number*/ + not_support, /*datetime*/ + not_expected, /*date*/ + not_expected, /*time*/ + not_expected, /*year*/ + not_support, /*string*/ + not_support, /*extend*/ + not_support, /*unknown*/ + not_support, /*text*/ + not_support, /*bit*/ + not_expected, /*enumset*/ + not_expected, /*enumset_inner*/ + not_support, /*otimestamp*/ + not_support, /*raw*/ + not_support, /*interval*/ + not_support, /*rowid*/ + not_support, /*lob*/ }, }; @@ -2963,6 +3901,8 @@ int number_range_check(ObObjCastParams ¶ms, const ObAccuracy &accuracy, res_obj = &obj; if (OB_UNLIKELY(precision < scale)) { ret = OB_ERR_M_BIGGER_THAN_D; + } else if (precision < 0 && scale < 0) { + /* do nothing */ } else if (number::ObNumber::MAX_PRECISION >= precision && number::ObNumber::MAX_SCALE >= scale && precision >= 0 && @@ -3025,6 +3965,37 @@ int datetime_scale_check(ObObjCastParams ¶ms, const ObAccuracy &accuracy, return ret; } +int otimestamp_scale_check(ObObjCastParams ¶ms, const ObAccuracy &accuracy, const ObObj &obj, + ObObj &buf_obj, const ObObj *&res_obj, const ObCastMode cast_mode) +{ + UNUSED(params); + UNUSED(cast_mode); + + int ret = OB_SUCCESS; + res_obj = NULL; + ObScale scale = accuracy.get_scale(); + + if (OB_UNLIKELY(scale > MAX_SCALE_FOR_ORACLE_TEMPORAL)) { + ret = OB_ERR_TOO_BIG_PRECISION; + LOG_WARN("fail to scale check", K(ret), K(MAX_SCALE_FOR_ORACLE_TEMPORAL)); + } else if (0 <= scale && scale < MAX_SCALE_FOR_ORACLE_TEMPORAL) { + ObOTimestampData ot_data = ObTimeConverter::round_otimestamp(scale, obj.get_otimestamp_value()); + if (ObTimeConverter::is_valid_otimestamp(ot_data.time_us_, static_cast(ot_data.time_ctx_.tail_nsec_))) { + buf_obj.set_otimestamp_value(obj.get_type(), ot_data); + buf_obj.set_scale(scale); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid otimestamp, set it null", K(ret), K(ot_data), K(scale)); + buf_obj.set_null(); + } + res_obj = &buf_obj; + } else { + res_obj = &obj; + } + + return ret; +} + int time_scale_check(ObObjCastParams ¶ms, const ObAccuracy &accuracy, const ObObj &obj, ObObj &buf_obj, const ObObj *&res_obj, const ObCastMode cast_mode) { @@ -3146,25 +4117,51 @@ int obj_collation_check(const bool is_strict_mode, const ObCollationType cs_type return ret; } -int obj_accuracy_check(ObCastCtx &cast_ctx, const ObAccuracy &accuracy, const ObCollationType cs_type, - const ObObj &obj, ObObj &buf_obj, const ObObj *&res_obj) +int obj_accuracy_check(ObCastCtx &cast_ctx, + const ObAccuracy &accuracy, + const ObCollationType cs_type, + const ObObj &obj, + ObObj &buf_obj, + const ObObj *&res_obj) { int ret = OB_SUCCESS; - if (ObFloatTC == obj.get_type_class()) { - ret = float_range_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else if (ObDoubleTC == obj.get_type_class()) { - ret = double_check_precision(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else if (ObNumberTC == obj.get_type_class()) { - ret = number_range_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else if (ObDateTimeTC == obj.get_type_class()) { - ret = datetime_scale_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else if (ObTimeTC == obj.get_type_class()) { - ret = time_scale_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else if (ObStringTC == obj.get_type_class()) { - ret = string_length_check(cast_ctx, accuracy, cs_type, obj, buf_obj, res_obj, cast_ctx.cast_mode_); - } else { - //LOG_WARN("unexpected type class to check", K(obj)); + + LOG_DEBUG("obj_accuracy_check before", K(obj), K(accuracy), K(cs_type)); + + switch (obj.get_type_class()) { + case ObFloatTC: { + ret = float_range_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObDoubleTC: { + ret = double_check_precision(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObNumberTC: { + ret = number_range_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObDateTimeTC: { + ret = datetime_scale_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObOTimestampTC: { + ret = otimestamp_scale_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObTimeTC: { + ret = time_scale_check(cast_ctx, accuracy, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + case ObStringTC: { + ret = string_length_check(cast_ctx, accuracy, cs_type, obj, buf_obj, res_obj, cast_ctx.cast_mode_); + break; + } + default: { + break; + } } + return ret; } @@ -3235,8 +4232,11 @@ int ob_obj_to_ob_time_without_date(const ObObj &obj, const ObTimeZoneInfo *tz_in return ret; } -int ObObjCasterV2::to_type(const ObObjType expect_type, ObCastCtx &cast_ctx, - const ObObj &in_obj, ObObj &buf_obj, const ObObj *&res_obj) +int ObObjCasterV2::to_type(const ObObjType expect_type, + ObCastCtx &cast_ctx, + const ObObj &in_obj, + ObObj &buf_obj, + const ObObj *&res_obj) { int ret = OB_SUCCESS; res_obj = NULL; @@ -3251,43 +4251,52 @@ int ObObjCasterV2::to_type(const ObObjType expect_type, ObCastCtx &cast_ctx, return ret; } -int ObObjCasterV2::to_type(const ObObjType expect_type, ObCastCtx &cast_ctx, - const ObObj &in_obj, ObObj &out_obj) +int ObObjCasterV2::to_type(const ObObjType expect_type, + ObCastCtx &cast_ctx, + const ObObj &in_obj, + ObObj &out_obj) { return to_type(expect_type, CS_TYPE_INVALID, cast_ctx, in_obj, out_obj); } -int ObObjCasterV2::to_type(const ObObjType expect_type, const ObCollationType expect_cs_type, - ObCastCtx &cast_ctx, const ObObj &in_obj, ObObj &out_obj) +int ObObjCasterV2::to_type(const ObObjType expect_type, + ObCollationType expect_cs_type, + ObCastCtx &cast_ctx, + const ObObj &in_obj, + ObObj &out_obj) { int ret = OB_SUCCESS; const ObObjTypeClass in_tc = in_obj.get_type_class(); const ObObjTypeClass out_tc = ob_obj_type_class(expect_type); cast_ctx.warning_ = OB_SUCCESS; - cast_ctx.dest_collation_ = expect_cs_type; + if (CS_TYPE_INVALID != expect_cs_type) { + cast_ctx.dest_collation_ = expect_cs_type; + } else { + cast_ctx.dest_collation_ = cast_ctx.dtc_params_.connection_collation_; + } if (OB_UNLIKELY(ob_is_invalid_obj_tc(in_tc) || ob_is_invalid_obj_tc(out_tc))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected type", K(ret), K(in_obj), K(expect_type)); } else if (OB_FAIL(OB_OBJ_CAST[in_tc][out_tc](expect_type, cast_ctx, in_obj, out_obj, cast_ctx.cast_mode_))) { - LOG_WARN("failed to cast obj", K(ret), K(in_obj), K(expect_type), K(cast_ctx.cast_mode_)); - } else { - if (ObStringTC == out_tc) { - if (CS_TYPE_INVALID != expect_cs_type) { - // expect_cs_type has higher priority than collation of in_obj or connection. + LOG_WARN("failed to cast obj", K(ret), K(in_obj), K(in_tc), K(out_tc), K(expect_type), K(cast_ctx.cast_mode_)); + } + + if (OB_SUCC(ret)) { + if (ObStringTC == out_tc || ObTextTC == out_tc || ObLobTC == out_tc) { + if (ObStringTC == in_tc || ObTextTC == in_tc || ObLobTC == out_tc) { + out_obj.set_collation_level(in_obj.get_collation_level()); + } else { + out_obj.set_collation_level(CS_LEVEL_COERCIBLE); + } + if (OB_LIKELY(expect_cs_type != CS_TYPE_INVALID)) { out_obj.set_collation_type(expect_cs_type); - } else if (ObStringTC != in_tc) { - // cast to varchar from other types, the result collation is collation_connection - if (OB_UNLIKELY(CS_TYPE_INVALID == cast_ctx.connection_collation_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected collation type", K(ret), K(cast_ctx.connection_collation_)); - } else if (out_obj.is_varchar_or_char()) { - out_obj.set_collation_type(cast_ctx.connection_collation_); - } } else { - // collation of out_obj has been set in string_string(), nothing to do here. + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected collation type", K(ret), K(in_obj), K(out_obj), K(expect_cs_type), K(common::lbt())); } } } + LOG_DEBUG("process to_type", K(ret), "in_type", in_obj.get_type(), K(in_obj), K(expect_type), K(out_obj), K(lbt())); return ret; } diff --git a/src/common/ob_obj_cast.h b/src/common/ob_obj_cast.h index a4bdd1a92aa47b5adcf70d36191dbf13d4f4b3bc..0e752d7d4b420c0910c2070a0391b25c979b33dd 100644 --- a/src/common/ob_obj_cast.h +++ b/src/common/ob_obj_cast.h @@ -41,6 +41,8 @@ namespace common // do range check. #define CM_NO_CAST_INT_UINT (1ULL << 3) // no cast between int and uint, otherwise // do cast between int and uint. +#define CM_ORACLE_MODE (1ULL << 61) + #define CM_INSERT_UPDATE_SCOPE (1ULL << 62) // affect calculate values() function. return the insert values // otherwise return NULL; #define CM_INTERNAL_CALL (1ULL << 63) // is internal call, otherwise @@ -59,6 +61,7 @@ typedef uint64_t ObCastMode; #define CM_UNSET_NO_CAST_INT_UINT(mode) (~CM_NO_CAST_INT_UINT & (mode)) #define CM_IS_INTERNAL_CALL(mode) (CM_INTERNAL_CALL & (mode)) #define CM_IS_EXTERNAL_CALL(mode) (!CM_IS_INTERNAL_CALL(mode)) + struct ObObjCastParams { // add params when necessary @@ -66,42 +69,58 @@ struct ObObjCastParams ObObjCastParams() : allocator_(NULL), allocator_v2_(NULL), - tz_info_(NULL), cur_time_(0), cast_mode_(CM_NONE), warning_(OB_SUCCESS), zf_info_(NULL), - connection_collation_(CS_TYPE_INVALID), - res_accuracy_(NULL) {} + dest_collation_(CS_TYPE_INVALID), + expect_obj_collation_(CS_TYPE_INVALID), + res_accuracy_(NULL), + dtc_params_() + { + set_compatible_cast_mode(); + } - ObObjCastParams(ObIAllocator *allocator_v2, const ObTimeZoneInfo *tz_info, - ObCastMode cast_mode, ObCollationType connection_collation, - ObAccuracy *res_accuracy = NULL) + ObObjCastParams(ObIAllocator* allocator_v2, const ObDataTypeCastParams* dtc_params, ObCastMode cast_mode, + ObCollationType dest_collation, ObAccuracy* res_accuracy = NULL) + //ObIAllocator *allocator_v2, ObCastMode cast_mode, ObCollationType dest_collation, ObAccuracy *res_accuracy = NULL) : allocator_(NULL), allocator_v2_(allocator_v2), - tz_info_(tz_info), cur_time_(0), cast_mode_(cast_mode), warning_(OB_SUCCESS), zf_info_(NULL), - connection_collation_(connection_collation), - res_accuracy_(res_accuracy) {} - - ObObjCastParams(ObIAllocator *allocator_v2, const ObTimeZoneInfo *tz_info, - int64_t cur_time, ObCastMode cast_mode, - ObCollationType connection_collation, - const ObZerofillInfo *zf_info = NULL, - ObAccuracy *res_accuracy = NULL) + dest_collation_(dest_collation), + expect_obj_collation_(dest_collation), + res_accuracy_(res_accuracy), + dtc_params_() + { + set_compatible_cast_mode(); + if (NULL != dtc_params) { + dtc_params_ = *dtc_params; + } + } + + ObObjCastParams(ObIAllocator* allocator_v2, const ObDataTypeCastParams* dtc_params, int64_t cur_time, + ObCastMode cast_mode, ObCollationType dest_collation, const ObZerofillInfo* zf_info = NULL, + ObAccuracy* res_accuracy = NULL) : allocator_(NULL), allocator_v2_(allocator_v2), - tz_info_(tz_info), cur_time_(cur_time), cast_mode_(cast_mode), warning_(OB_SUCCESS), zf_info_(zf_info), - connection_collation_(connection_collation), - res_accuracy_(res_accuracy) {} - + dest_collation_(dest_collation), + expect_obj_collation_(dest_collation), + res_accuracy_(res_accuracy), + dtc_params_() + { + set_compatible_cast_mode(); + if (NULL != dtc_params) { + dtc_params_ = *dtc_params; + } + } + void *alloc(const int64_t size) const { void *ret = NULL; @@ -112,17 +131,52 @@ struct ObObjCastParams } return ret; } + + void set_compatible_cast_mode() + { + if (lib::is_oracle_mode()) { + cast_mode_ &= ~CM_WARN_ON_FAIL; + cast_mode_ |= CM_ORACLE_MODE; + } else { + cast_mode_ &= ~CM_ORACLE_MODE; + } + return; + } + + TO_STRING_KV(K(cur_time_), KP(cast_mode_), K(warning_), K(dest_collation_), + K(expect_obj_collation_), K(res_accuracy_)); + IAllocator *allocator_; ObIAllocator *allocator_v2_; - const ObTimeZoneInfo *tz_info_; int64_t cur_time_; ObCastMode cast_mode_; int warning_; const ObZerofillInfo *zf_info_; - ObCollationType connection_collation_; - ObAccuracy *res_accuracy_; ObCollationType dest_collation_; + ObCollationType expect_obj_collation_; // for each column obj + ObAccuracy *res_accuracy_; + ObDataTypeCastParams dtc_params_; +}; + + +typedef ObObjCastParams ObCastCtx; + +class ObHexUtils { +public: + // text can be odd number, like 'aaa', treat as '0aaa' + static int unhex(const common::ObString &text, common::ObCastCtx &cast_ctx, common::ObObj &result); + static int hex(const common::ObString &text, common::ObCastCtx &cast_ctx, common::ObObj &result); + static int hex_for_mysql(const uint64_t uint_val, common::ObCastCtx &cast_ctx, common::ObObj &result); + static int rawtohex(const common::ObObj &text, common::ObCastCtx &cast_ctx, common::ObObj &result); + static int hextoraw(const common::ObObj &text, common::ObCastCtx &cast_ctx, common::ObObj &result); + static int get_uint(const common::ObObj &obj, common::ObCastCtx &cast_ctx, common::number::ObNumber &out); + static int copy_raw(const common::ObObj &obj, common::ObCastCtx &cast_ctx, common::ObObj &result); + +private: + static int uint_to_raw(const common::number::ObNumber &text, common::ObCastCtx &cast_ctx, common::ObObj &result); }; + + /** * cast functions to do the real work * cast the input object to the type specified and store the result in out_obj @@ -337,7 +391,7 @@ int ObObjCaster::expr_obj_cast(const ObObjTypeClass orig_ ObObjCastParams::TAllocator ta(allocator_); ObObjCastParams params; params.allocator_ = &ta; - params.tz_info_ = tz_info; + params.dtc_params_.tz_info_ = tz_info; params.cast_mode_ = CM_WARN_ON_FAIL; int warning = OB_SUCCESS; extern ObObjCastFunc OB_OBJ_CAST[ObMaxTC][ObMaxTC]; @@ -427,13 +481,9 @@ bool cast_supported(const ObObjTypeClass orig_td, const ObObjTypeClass expect_td int ob_obj_to_ob_time_with_date(const ObObj &obj, const ObTimeZoneInfo *tz_info, ObTime &ob_time); int ob_obj_to_ob_time_without_date(const ObObj &obj, const ObTimeZoneInfo *tz_info, ObTime &ob_time); -//============================== - -typedef ObObjCastParams ObCastCtx; - int obj_collation_check(const bool is_strict_mode, const ObCollationType cs_type, ObObj &obj); -int obj_accuracy_check(ObCastCtx &cast_ctx, const ObAccuracy &accuracy, const ObCollationType cs_type, - const ObObj &obj, ObObj &buf_obj, const ObObj *&res_obj); +int obj_accuracy_check(ObCastCtx &cast_ctx, const ObAccuracy &accuracy, + const ObCollationType cs_type, const ObObj &obj, ObObj &buf_obj, const ObObj *&res_obj); class ObObjCasterV2 { @@ -442,7 +492,7 @@ public: const ObObj &in_obj, ObObj &buf_obj, const ObObj *&out_obj); static int to_type(const ObObjType expect_type, ObCastCtx &cast_ctx, const ObObj &in_obj, ObObj &out_obj); - static int to_type(const ObObjType expect_type, const ObCollationType expect_cs_type, + static int to_type(const ObObjType expect_type, ObCollationType expect_cs_type, ObCastCtx &cast_ctx, const ObObj &in_obj, ObObj &out_obj); static int is_cast_monotonic(ObObjType t1, ObObjType t2, bool &is_monotonic); static int is_order_consistent(const ObObjMeta &from, diff --git a/src/common/ob_obj_compare.cpp b/src/common/ob_obj_compare.cpp index 3705d05b1e6ae2b8dcb19a28533d0ac104f94f66..af14ba45fb240b2860075c2e266bcd25d62870f0 100644 --- a/src/common/ob_obj_compare.cpp +++ b/src/common/ob_obj_compare.cpp @@ -20,6 +20,12 @@ namespace oceanbase namespace common { +#define OBJ_TYPE_CLASS_CHECK(obj, tc) \ + if (OB_UNLIKELY(obj.get_type_class() != tc)) { \ + LOG_ERROR("unexpected error. mismatch function for comparison", K(obj), K(tc)); \ + right_to_die_or_duty_to_live(); \ + } + #define DEFINE_CMP_OP_FUNC(tc, type, op, op_str) \ template <> inline \ int ObObjCmpFuncs::cmp_op_func(const ObObj &obj1, \ @@ -458,6 +464,164 @@ namespace common : CR_ERROR; \ } +// type storedtime +// data local +// timestamp nano local +// timestamptz utc + tzid +// timestampltz utc + tzid +//datetimetc VS otimestamptc +#define DEFINE_CMP_OP_FUNC_DT_OT(op, op_str) \ + template <> inline \ + int ObObjCmpFuncs::cmp_op_func(const ObObj &obj1, \ + const ObObj &obj2, \ + const ObCompareCtx &cmp_ctx) \ + { \ + UNUSED(cmp_ctx); \ + OBJ_TYPE_CLASS_CHECK(obj1, ObDateTimeTC);\ + OBJ_TYPE_CLASS_CHECK(obj2, ObOTimestampTC); \ + ObCmpRes ret = CR_FALSE; \ + ObOTimestampData v1; \ + v1.time_us_ = obj1.get_datetime();\ + ObOTimestampData v2 = obj2.get_otimestamp_value();\ + if (!obj2.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) {\ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else {\ + v1.time_us_ -= cmp_ctx.tz_off_;\ + }\ + }\ + return (CR_ERROR != ret ? static_cast(v1 op_str v2) : CR_ERROR); \ + } + +//datetimetc VS otimestamptc +#define DEFINE_CMP_FUNC_DT_OT() \ + template <> inline \ + int ObObjCmpFuncs::cmp_func(const ObObj &obj1, \ + const ObObj &obj2, \ + const ObCompareCtx &cmp_ctx) \ + { \ + OBJ_TYPE_CLASS_CHECK(obj1, ObDateTimeTC);\ + OBJ_TYPE_CLASS_CHECK(obj2, ObOTimestampTC);\ + ObCmpRes ret = CR_FALSE;\ + ObOTimestampData v1; \ + v1.time_us_ = obj1.get_datetime();\ + ObOTimestampData v2 = obj2.get_otimestamp_value(); \ + if (!obj2.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) {\ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else {\ + v1.time_us_ -= cmp_ctx.tz_off_;\ + }\ + }\ + return (CR_ERROR != ret \ + ? (v1 < v2 \ + ? CR_LT \ + : (v1 > v2 \ + ? CR_GT \ + : CR_EQ))\ + : CR_ERROR);\ + } + +// otimestamptc VS datetimetc +#define DEFINE_CMP_OP_FUNC_OT_DT(op, op_str) \ + template <> \ + inline int ObObjCmpFuncs::cmp_op_func( \ + const ObObj& obj1, const ObObj& obj2, const ObCompareCtx& cmp_ctx) \ + { \ + UNUSED(cmp_ctx); \ + OBJ_TYPE_CLASS_CHECK(obj1, ObOTimestampTC); \ + OBJ_TYPE_CLASS_CHECK(obj2, ObDateTimeTC); \ + ObCmpRes ret = CR_FALSE; \ + ObOTimestampData v1 = obj1.get_otimestamp_value(); \ + ObOTimestampData v2; \ + v2.time_us_ = obj2.get_datetime(); \ + if (!obj1.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) { \ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else { \ + v2.time_us_ -= cmp_ctx.tz_off_; \ + } \ + } \ + return (CR_ERROR != ret ? static_cast(v1 op_str v2) : CR_ERROR); \ + } + +// otimestamptc VS datetimetc +#define DEFINE_CMP_FUNC_OT_DT() \ + template <> \ + inline int ObObjCmpFuncs::cmp_func( \ + const ObObj& obj1, const ObObj& obj2, const ObCompareCtx& cmp_ctx) \ + { \ + OBJ_TYPE_CLASS_CHECK(obj1, ObOTimestampTC); \ + OBJ_TYPE_CLASS_CHECK(obj2, ObDateTimeTC); \ + ObCmpRes ret = CR_FALSE; \ + ObOTimestampData v1 = obj1.get_otimestamp_value(); \ + ObOTimestampData v2; \ + v2.time_us_ = obj2.get_datetime(); \ + if (!obj1.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) { \ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else { \ + v2.time_us_ -= cmp_ctx.tz_off_; \ + } \ + } \ + return (CR_ERROR != ret ? (v1 < v2 ? CR_LT : (v1 > v2 ? CR_GT : CR_EQ)) : CR_ERROR); \ + } + +// otimestamptc VS otimestamptc +#define DEFINE_CMP_OP_FUNC_OT_OT(op, op_str) \ + template <> \ + inline int ObObjCmpFuncs::cmp_op_func( \ + const ObObj& obj1, const ObObj& obj2, const ObCompareCtx& cmp_ctx) \ + { \ + UNUSED(cmp_ctx); \ + OBJ_TYPE_CLASS_CHECK(obj1, ObOTimestampTC); \ + OBJ_TYPE_CLASS_CHECK(obj2, ObOTimestampTC); \ + ObCmpRes ret = CR_FALSE; \ + ObOTimestampData v1 = obj1.get_otimestamp_value(); \ + ObOTimestampData v2 = obj2.get_otimestamp_value(); \ + if (obj1.is_timestamp_nano() != obj2.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) { \ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else { \ + if (obj1.is_timestamp_nano()) { \ + v1.time_us_ -= cmp_ctx.tz_off_; \ + } else { \ + v2.time_us_ -= cmp_ctx.tz_off_; \ + } \ + } \ + } \ + return (CR_ERROR != ret ? static_cast(v1 op_str v2) : CR_ERROR); \ + } + +// otimestamptc VS otimestamptc +#define DEFINE_CMP_FUNC_OT_OT() \ + template <> \ + inline int ObObjCmpFuncs::cmp_func( \ + const ObObj& obj1, const ObObj& obj2, const ObCompareCtx& cmp_ctx) \ + { \ + OBJ_TYPE_CLASS_CHECK(obj1, ObOTimestampTC); \ + OBJ_TYPE_CLASS_CHECK(obj2, ObOTimestampTC); \ + ObCmpRes ret = CR_FALSE; \ + ObOTimestampData v1 = obj1.get_otimestamp_value(); \ + ObOTimestampData v2 = obj2.get_otimestamp_value(); \ + if (obj1.is_timestamp_nano() != obj2.is_timestamp_nano()) { \ + if (OB_UNLIKELY(INVALID_TZ_OFF == cmp_ctx.tz_off_)) { \ + LOG_ERROR("invalid timezone offset", K(obj1), K(obj2)); \ + ret = CR_ERROR; \ + } else if (obj1.is_timestamp_nano()) { \ + v1.time_us_ -= cmp_ctx.tz_off_; \ + } else { \ + v2.time_us_ -= cmp_ctx.tz_off_; \ + } \ + } \ + return (CR_ERROR != ret ? (v1 < v2 ? CR_LT : (v1 > v2 ? CR_GT : CR_EQ)) : CR_ERROR); \ + } + //============================== #define DEFINE_CMP_FUNCS(tc, type) \ @@ -609,6 +773,15 @@ namespace common DEFINE_CMP_OP_FUNC_DT_DT(CO_NE, !=); \ DEFINE_CMP_FUNC_DT_DT(); \ +#define DEFINE_CMP_FUNCS_DATETIME_OTIMESTAMP() \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_EQ, ==); \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_LE, <=); \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_LT, < ); \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_GE, >=); \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_GT, > ); \ + DEFINE_CMP_OP_FUNC_DT_OT(CO_NE, !=); \ + DEFINE_CMP_FUNC_DT_OT(); \ + #define DEFINE_CMP_FUNCS_DATE_DATE() \ DEFINE_CMP_FUNCS(ObDateTC, date); @@ -618,6 +791,25 @@ namespace common #define DEFINE_CMP_FUNCS_YEAR_YEAR() \ DEFINE_CMP_FUNCS(ObYearTC, year); + +#define DEFINE_CMP_FUNCS_OTIMESTAMP_DATETIME() \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_EQ, ==); \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_LE, <=); \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_LT, <); \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_GE, >=); \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_GT, >); \ + DEFINE_CMP_OP_FUNC_OT_DT(CO_NE, !=); \ + DEFINE_CMP_FUNC_OT_DT(); + +#define DEFINE_CMP_FUNCS_OTIMESTAMP_OTIMESTAMP() \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_EQ, ==); \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_LE, <=); \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_LT, <); \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_GE, >=); \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_GT, >); \ + DEFINE_CMP_OP_FUNC_OT_OT(CO_NE, !=); \ + DEFINE_CMP_FUNC_OT_OT(); + #define DEFINE_CMP_FUNCS_STRING_STRING() \ DEFINE_CMP_OP_FUNC_STRING_STRING(CO_EQ, ==); \ DEFINE_CMP_OP_FUNC_STRING_STRING(CO_LE, <=); \ @@ -716,6 +908,11 @@ DEFINE_CMP_FUNCS_NUMBER_UINT(); DEFINE_CMP_FUNCS_NUMBER_NUMBER(); DEFINE_CMP_FUNCS_DATETIME_DATETIME(); +DEFINE_CMP_FUNCS_DATETIME_OTIMESTAMP(); + +DEFINE_CMP_FUNCS_OTIMESTAMP_DATETIME(); +DEFINE_CMP_FUNCS_OTIMESTAMP_OTIMESTAMP(); + DEFINE_CMP_FUNCS_DATE_DATE(); DEFINE_CMP_FUNCS_TIME_TIME(); DEFINE_CMP_FUNCS_YEAR_YEAR(); @@ -760,7 +957,15 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), - DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //text + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //text + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //bit + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //setenun + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //setenuninner + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //otimestamp + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //raw + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //interval + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //rowid + DEFINE_CMP_FUNCS_ENTRY(ObNullTC, ObMaxTC), //lob }, { // int DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -776,7 +981,15 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY_NULL, // string DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown - DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // uint DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -792,7 +1005,15 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY_NULL, // string DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown - DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // float DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -809,6 +1030,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // double DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -825,6 +1054,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // number DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -841,6 +1078,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL,//enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // datetime DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -857,6 +1102,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY(ObDateTimeTC, ObOTimestampTC),//otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // date DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -873,6 +1126,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // time DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -889,6 +1150,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // year DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -905,6 +1174,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // string DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -921,6 +1198,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY(ObStringTC, ObStringTC), //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // extend DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObNullTC), @@ -937,6 +1222,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC), DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC), //text + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC), + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//enumset + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//enumsetInner + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//otimestamp + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//raw + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//interval + DEFINE_CMP_FUNCS_ENTRY(ObExtendTC, ObMaxTC),//rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // unknown DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -953,6 +1246,14 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), DEFINE_CMP_FUNCS_ENTRY(ObUnknownTC, ObUnknownTC), DEFINE_CMP_FUNCS_ENTRY_NULL, //text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, { // text DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), @@ -969,6 +1270,214 @@ const obj_cmp_func ObObjCmpFuncs::cmp_funcs[ObMaxTC][ObMaxTC][CO_MAX] = DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), // extend DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown DEFINE_CMP_FUNCS_ENTRY(ObStringTC, ObStringTC), // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, //otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // enumsetinner + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // otimestamp + DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObNullTC), + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY(ObOTimestampTC, ObDateTimeTC), // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY(ObMaxTC, ObExtendTC), // extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetinner + DEFINE_CMP_FUNCS_ENTRY(ObOTimestampTC, ObOTimestampTC),// otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, //raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob + }, + { + // lob + DEFINE_CMP_FUNCS_ENTRY_NULL, //null + DEFINE_CMP_FUNCS_ENTRY_NULL, // int + DEFINE_CMP_FUNCS_ENTRY_NULL, // uint + DEFINE_CMP_FUNCS_ENTRY_NULL, // float + DEFINE_CMP_FUNCS_ENTRY_NULL, // double + DEFINE_CMP_FUNCS_ENTRY_NULL, // number + DEFINE_CMP_FUNCS_ENTRY_NULL, // datetime + DEFINE_CMP_FUNCS_ENTRY_NULL, // date + DEFINE_CMP_FUNCS_ENTRY_NULL, // time + DEFINE_CMP_FUNCS_ENTRY_NULL, // year + DEFINE_CMP_FUNCS_ENTRY_NULL, // string + DEFINE_CMP_FUNCS_ENTRY_NULL, //extend + DEFINE_CMP_FUNCS_ENTRY_NULL, // unknown + DEFINE_CMP_FUNCS_ENTRY_NULL, // text + DEFINE_CMP_FUNCS_ENTRY_NULL, // bit + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumset + DEFINE_CMP_FUNCS_ENTRY_NULL, // enumsetInner will not go here + DEFINE_CMP_FUNCS_ENTRY_NULL, // otimestamp + DEFINE_CMP_FUNCS_ENTRY_NULL, // raw + DEFINE_CMP_FUNCS_ENTRY_NULL, // interval + DEFINE_CMP_FUNCS_ENTRY_NULL, //rowid + DEFINE_CMP_FUNCS_ENTRY_NULL, //lob }, }; diff --git a/src/common/ob_obj_funcs.h b/src/common/ob_obj_funcs.h index 897add2eeeba71e4454584d269c71d94ee013698..68c5f3fd63de3fd18a905b28eeefab32193087d6 100644 --- a/src/common/ob_obj_funcs.h +++ b/src/common/ob_obj_funcs.h @@ -51,59 +51,74 @@ struct ObObjTypeFuncs }; // function templates for the above functions template - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObObjPrintParams ¶ms); + template - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObObjPrintParams ¶ms); + template - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms); + template - int obj_print_json(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); + int obj_print_json(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObObjPrintParams ¶ms); + template int64_t obj_crc64(const ObObj &obj, const int64_t current); + template void obj_batch_checksum(const ObObj &obj, ObBatchChecksum &bc); + template uint64_t obj_murmurhash(const ObObj &obj, const uint64_t hash); + template int obj_val_serialize(const ObObj &obj, char* buf, const int64_t buf_len, int64_t& pos); + template int obj_val_deserialize(ObObj &obj, const char* buf, const int64_t data_len, int64_t& pos); + template int64_t obj_val_get_serialize_size(const ObObj &obj); + //////////////////////////////////////////////////////////////// // ObNullType = 0, template <> - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; ret = databuff_printf(buffer, length, pos, "NULL"); return ret; } template <> - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; ret = databuff_printf(buffer, length, pos, "NULL"); return ret; } template <> - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; ret = databuff_printf(buffer, length, pos, "NULL"); return ret; } template <> - int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); J_OBJ_START(); BUF_PRINTO(ob_obj_type_str(obj.get_type())); J_COLON(); @@ -199,7 +214,7 @@ template <> static uint64_t calc_hash_value(const P ¶m, const uint64_t hash) { \ VTYPE v = param.get_##TYPE(); \ HTYPE v2 = v; \ - if (OB_UNLIKELY(ObDoubleType == OBJTYPE || ObFloatType == OBJTYPE) && 0.0 == v2) { \ + if (OB_UNLIKELY(ObDoubleType == OBJTYPE || ObFloatType == OBJTYPE) && 0.0 == (double)v2) { \ v2 = 0.0; \ } \ return T::hash(&v2, sizeof(v2), hash); \ @@ -209,30 +224,34 @@ template <> // general print functions generator #define DEF_PRINT_FUNCS(OBJTYPE, TYPE, SQL_FORMAT, STR_FORMAT) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return databuff_printf(buffer, length, pos, SQL_FORMAT, obj.get_##TYPE()); \ } \ \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return databuff_printf(buffer, length, pos, STR_FORMAT, obj.get_##TYPE()); \ } \ \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return databuff_printf(buffer, length, pos, SQL_FORMAT, obj.get_##TYPE()); \ } \ \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ J_OBJ_START(); \ PRINT_META(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ @@ -312,9 +331,10 @@ DEF_NUMERIC_FUNCS(ObUDoubleType, udouble, double, "%2lf", "'%2lf'", double); // ObUNumberType=16, #define DEF_NUMBER_PRINT_FUNCS(OBJTYPE, TYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = common::OB_SUCCESS; \ number::ObNumber nmb; \ if (OB_FAIL(obj.get_##TYPE(nmb))) { \ @@ -323,10 +343,11 @@ DEF_NUMERIC_FUNCS(ObUDoubleType, udouble, double, "%2lf", "'%2lf'", double); return ret; \ } \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ + UNUSED(params); \ int ret = common::OB_SUCCESS; \ - UNUSED(tz_info); \ number::ObNumber nmb; \ const int64_t BUF_SIZE = common::number::ObNumber::MAX_PRINTABLE_SIZE; \ char tmp_buf[BUF_SIZE]; \ @@ -338,10 +359,11 @@ DEF_NUMERIC_FUNCS(ObUDoubleType, udouble, double, "%2lf", "'%2lf'", double); return ret; \ } \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ + UNUSED(params); \ int ret = common::OB_SUCCESS; \ - UNUSED(tz_info); \ number::ObNumber nmb; \ if (OB_FAIL(obj.get_##TYPE(nmb))) { \ } else if (OB_FAIL(nmb.format(buffer, length, pos, obj.get_scale()))) { \ @@ -349,9 +371,10 @@ DEF_NUMERIC_FUNCS(ObUDoubleType, udouble, double, "%2lf", "'%2lf'", double); return ret; \ } \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = common::OB_SUCCESS; \ number::ObNumber nmb; \ if (OB_FAIL(obj.get_##TYPE(nmb))) { \ @@ -447,51 +470,72 @@ DEF_NUMBER_FUNCS(ObNumberFloatType, number_float); //////////////// #define DEF_DATETIME_PRINT_FUNCS(OBJTYPE, TYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + inline int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ { \ + int ret = common::OB_SUCCESS; \ + static const char *CAST_PREFIX_ORACLE = "TO_DATE('"; \ + static const char *NORMAL_PREFIX = "'"; \ + static const char *CAST_SUFFIX_ORACLE = "', 'YYYY-MM-DD HH24:MI:SS')"; \ + static const char *NORMAL_SUFFIX = "'"; \ + const ObTimeZoneInfo *tz_info = params.tz_info_; \ ObString str(static_cast(length - pos - 1), 0, buffer + pos + 1); \ - int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ + const char *fmt_prefix = params.need_cast_expr_ && lib::is_oracle_mode() ? \ + CAST_PREFIX_ORACLE : NORMAL_PREFIX; \ + ret = databuff_printf(buffer, length, pos, "%s", fmt_prefix); \ + } \ + if (OB_SUCC(ret)) { \ if (ObTimestampType != obj.get_type()) { \ tz_info = NULL; \ } \ - ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, obj.get_scale(), buffer, length, pos); \ + ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, \ + obj.get_scale(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ - ret = databuff_printf(buffer, length, pos, "'"); \ + if (OB_SUCC(ret)) { \ + const char *fmt_suffix = params.need_cast_expr_ && lib::is_oracle_mode() ? \ + CAST_SUFFIX_ORACLE : NORMAL_SUFFIX; \ + ret = databuff_printf(buffer, length, pos, "%s", fmt_suffix); \ } \ return ret; \ } \ \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ - { \ - ObString str(static_cast(length - pos - 1), 0, buffer + pos + 1); \ + inline int obj_print_str(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ + { \ + const ObTimeZoneInfo *tz_info = params.tz_info_; \ + ObString str(static_cast(length - pos - 1), 0, buffer + pos + 1); \ int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ if (ObTimestampType != obj.get_type()) { \ tz_info = NULL; \ } \ - ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, obj.get_scale(), buffer, length, pos); \ + ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, \ + obj.get_scale(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = databuff_printf(buffer, length, pos, "'"); \ } \ return ret; \ } \ \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + inline int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ { \ + const ObTimeZoneInfo *tz_info = params.tz_info_; \ if (ObTimestampType != obj.get_type()) { \ tz_info = NULL; \ } \ - return ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, 6, buffer, length, pos); \ + return ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, OB_MAX_DATETIME_PRECISION, buffer, length, pos); \ } \ \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + inline int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ + const ObTimeZoneInfo *tz_info = params.tz_info_; \ J_OBJ_START(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ J_COLON(); \ @@ -499,8 +543,8 @@ DEF_NUMBER_FUNCS(ObNumberFloatType, number_float); if (ObTimestampType != obj.get_type()) { \ tz_info = NULL; \ } \ - int ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, 6, buf, buf_len, pos); \ - if (OB_SUCC(ret)) { \ + int ret = ObTimeConverter::datetime_to_str(obj.get_datetime(), tz_info, OB_MAX_DATETIME_PRECISION, buf, buf_len, pos); \ + if (OB_SUCC(ret)) { \ J_QUOTE(); \ J_OBJ_END(); \ } \ @@ -520,50 +564,54 @@ DEF_DATETIME_FUNCS(ObTimestampType, timestamp, int64_t); //////////////// #define DEF_DATE_YEAR_PRINT_FUNCS(OBJTYPE, TYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = databuff_printf(buffer, length, pos, "'"); \ } \ return ret; \ } \ \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = databuff_printf(buffer, length, pos, "'"); \ } \ return ret; \ } \ \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), buffer, length, pos); \ } \ \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ J_OBJ_START(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ J_COLON(); \ J_QUOTE(); \ int ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), buf, buf_len, pos); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ J_QUOTE(); \ J_OBJ_END(); \ } \ @@ -581,50 +629,54 @@ DEF_DATE_YEAR_FUNCS(ObDateType, date, int32_t); //////////////// #define DEF_TIME_PRINT_FUNCS(OBJTYPE, TYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), obj.get_scale(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = databuff_printf(buffer, length, pos, "'"); \ } \ return ret; \ } \ \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = databuff_printf(buffer, length, pos, "'"); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), obj.get_scale(), buffer, length, pos); \ } \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ ret = databuff_printf(buffer, length, pos, "'"); \ } \ return ret; \ } \ \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), obj.get_scale(), buffer, length, pos); \ } \ \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ J_OBJ_START(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ J_COLON(); \ - J_QUOTE(); \ + J_QUOTE(); \ int ret = ObTimeConverter::TYPE##_to_str(obj.get_##TYPE(), obj.get_scale(), buf, buf_len, pos); \ - if (OB_SUCC(ret)) { \ + if (OB_SUCC(ret)) { \ J_QUOTE(); \ J_OBJ_END(); \ } \ @@ -646,39 +698,46 @@ DEF_DATE_YEAR_FUNCS(ObYearType, year, uint8_t); // ObVarcharType=22, // charset: utf-8, collation: utf8_general_ci // ObCharType=23, // charset: utf-8, collation: utf8_general_ci template<> -int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); +int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms); + template<> -int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); +int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms); + template<> -int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); +int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms); #define DEF_VARCHAR_PRINT_FUNCS(OBJTYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ int ret = common::OB_SUCCESS; \ if (CS_TYPE_BINARY == obj.get_collation_type()) { \ - ret = obj_print_sql(obj, buffer, length, pos, tz_info); \ + ret = obj_print_sql(obj, buffer, length, pos, params); \ } else { \ ret = databuff_printf(buffer, length, pos, "'%.*s'", obj.get_string_len(), obj.get_string_ptr()); \ } \ return ret; \ } \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ if (CS_TYPE_BINARY == obj.get_collation_type()) { \ if (obj.get_string_len() > obj.get_data_length()) { \ } else { \ - int64_t zero_length = obj.get_data_length() - obj.get_string_len(); \ + int64_t zero_length = obj.get_data_length() - obj.get_string_len(); \ ret = databuff_printf(buffer, length, pos, "'%.*s", obj.get_string_len(), obj.get_string_ptr()); \ - for (int64_t i = 0; OB_SUCC(ret) && i < zero_length; ++i) { \ - ret = databuff_printf(buffer, length, pos, "\\0"); \ + for (int64_t i = 0; OB_SUCC(ret) && i < zero_length; ++i) { \ + ret = databuff_printf(buffer, length, pos, "\\0"); \ } \ - if (OB_SUCC(ret)) { \ - if (OB_FAIL(databuff_printf(buffer, length, pos, "'"))) { \ + if (OB_SUCC(ret)) { \ + if (OB_FAIL(databuff_printf(buffer, length, pos, "'"))) { \ } \ } \ } \ @@ -688,22 +747,24 @@ int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t return ret; \ } \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ return databuff_printf(buffer, length, pos, "%.*s", obj.get_string_len(), obj.get_string_ptr()); \ } \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ J_OBJ_START(); \ PRINT_META(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ J_COLON(); \ BUF_PRINTO(obj.get_varchar()); \ J_COMMA(); \ - J_KV(N_COLLATION, ObCharset::collation_name(obj.get_collation_type()));\ + J_KV(N_COLLATION, ObCharset::collation_name(obj.get_collation_type())); \ J_OBJ_END(); \ return OB_SUCCESS; \ } @@ -759,9 +820,10 @@ DEF_VARCHAR_FUNCS(ObCharType, char, ObString); // ObHexStringType=24, #define DEF_HEX_STRING_PRINT_FUNCS(OBJTYPE) \ template <> \ - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = databuff_printf(buffer, length, pos, "X'"))) { \ @@ -772,9 +834,10 @@ DEF_VARCHAR_FUNCS(ObCharType, char, ObString); return ret; \ } \ template <> \ - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = databuff_printf(buffer, length, pos, "X'"))) { \ @@ -785,9 +848,10 @@ DEF_VARCHAR_FUNCS(ObCharType, char, ObString); return ret; \ } \ template <> \ - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = databuff_printf(buffer, length, pos, "X"))) { \ @@ -796,9 +860,10 @@ DEF_VARCHAR_FUNCS(ObCharType, char, ObString); return ret; \ } \ template <> \ - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ J_OBJ_START(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ @@ -822,9 +887,10 @@ DEF_HEX_STRING_FUNCS(ObHexStringType, hex_string, ObString); //////////////// // ObExtendType=25, // Min, Max, etc. template <> - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; switch(obj.get_ext()) { case ObActionFlag::OP_MIN_OBJ: @@ -848,9 +914,10 @@ template <> } template <> - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; switch(obj.get_ext()) { case ObActionFlag::OP_MIN_OBJ: @@ -874,9 +941,10 @@ template <> } template <> - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; switch(obj.get_ext()) { case ObActionFlag::OP_MIN_OBJ: @@ -900,9 +968,10 @@ template <> } template <> - int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); int ret = OB_SUCCESS; J_OBJ_START(); BUF_PRINTO(ob_obj_type_str(obj.get_type())); @@ -934,30 +1003,34 @@ DEF_SERIALIZE_FUNCS(ObExtendType, ext, int64_t); //////////////// // 28, ObUnknownType template <> - int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); return databuff_printf(buffer, length, pos, "?"); } template <> - int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); return databuff_printf(buffer, length, pos, "'?'"); } template <> - int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, + const ObObjPrintParams ¶ms) { UNUSED(obj); - UNUSED(tz_info); + UNUSED(params); return databuff_printf(buffer, length, pos, "?"); } template <> - int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, const ObTimeZoneInfo *tz_info) + int obj_print_json(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos, + const ObObjPrintParams ¶ms) { - UNUSED(tz_info); + UNUSED(params); J_OBJ_START(); BUF_PRINTO(ob_obj_type_str(obj.get_type())); J_COLON(); @@ -1020,13 +1093,262 @@ template <> return len; } +//////////////// +// ObTimestampTZType=36, +// ObTimestampLTZType=37, +// ObTimestampNanoType=38, +#define DEF_ORACLE_TIMESTAMP_COMMON_PRINT_FUNCS(OBJTYPE) \ + template <> \ + inline int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ + { \ + int ret = common::OB_SUCCESS; \ + static const char *CAST_TZ_PREFIX = "TO_TIMESTAMP_TZ('"; \ + static const char *CAST_LTZ_PREFIX = "TO_TIMESTAMP('"; \ + static const char *CAST_NANO_PREFIX = "TO_TIMESTAMP('"; \ + static const char *NORMAL_PREFIX = "'"; \ + static const char *CAST_SUFFIX = "', '%.*s')"; \ + static const char *NORMAL_SUFFIX = "'"; \ + ObDataTypeCastParams dtc_params(params.tz_info_); \ + if (OB_SUCC(ret)) { \ + const char *fmt_prefix = NORMAL_PREFIX; \ + if (params.need_cast_expr_) { \ + switch (OBJTYPE) { \ + case ObTimestampTZType: \ + fmt_prefix = CAST_TZ_PREFIX; \ + break; \ + case ObTimestampLTZType: \ + fmt_prefix = CAST_LTZ_PREFIX; \ + break; \ + case ObTimestampNanoType: \ + fmt_prefix = CAST_NANO_PREFIX; \ + break; \ + default: \ + ret = OB_ERR_UNEXPECTED; \ + break; \ + } \ + } \ + if (OB_SUCC(ret)) { \ + ret = databuff_printf(buffer, length, pos, "%s", fmt_prefix); \ + dtc_params.force_use_standard_format_ = true; \ + } \ + } \ + if (OB_SUCC(ret)) { \ + ret = ObTimeConverter::otimestamp_to_str(obj.get_otimestamp_value(), dtc_params, \ + obj.get_scale(), OBJTYPE, buffer, length, pos);\ + } \ + if (OB_SUCC(ret)) { \ + const char *fmt_suffix = NORMAL_SUFFIX; \ + if (params.need_cast_expr_) { \ + const ObString NLS_FORMAT = dtc_params.get_nls_format(OBJTYPE); \ + fmt_suffix = CAST_SUFFIX; \ + ret = databuff_printf(buffer, length, pos, fmt_suffix, \ + NLS_FORMAT.length(), NLS_FORMAT.ptr()); \ + } else { \ + ret = databuff_printf(buffer, length, pos, "%s", fmt_suffix); \ + } \ + } \ + return ret; \ + } \ + template <> \ + inline int obj_print_str(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ + { \ + int ret = databuff_printf(buffer, length, pos, "'"); \ + if (OB_SUCC(ret)) { \ + const ObDataTypeCastParams dtc_params(params.tz_info_); \ + ret = ObTimeConverter::otimestamp_to_str(obj.get_otimestamp_value(), dtc_params, \ + obj.get_scale(), OBJTYPE, buffer, length, pos);\ + } \ + if (OB_SUCC(ret)) { \ + ret = databuff_printf(buffer, length, pos, "'"); \ + } \ + return ret; \ + } \ + template <> \ + inline int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, \ + int64_t &pos, const ObObjPrintParams ¶ms) \ + { \ + const ObDataTypeCastParams dtc_params(params.tz_info_); \ + return ObTimeConverter::otimestamp_to_str(obj.get_otimestamp_value(), dtc_params, \ + OB_MAX_TIMESTAMP_TZ_PRECISION, \ + OBJTYPE, buffer, length, pos); \ + } \ + template <> \ + inline int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ + { \ + UNUSED(params); \ + int ret = OB_SUCCESS; \ + J_OBJ_START(); \ + BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ + J_COLON(); \ + J_QUOTE(); \ + const ObDataTypeCastParams dtc_params(params.tz_info_); \ + ret = ObTimeConverter::otimestamp_to_str(obj.get_otimestamp_value(), dtc_params, \ + OB_MAX_TIMESTAMP_TZ_PRECISION, \ + OBJTYPE, buf, buf_len, pos); \ + if (OB_SUCC(ret)) { \ + J_QUOTE(); \ + J_OBJ_END(); \ + } \ + return ret; \ + } + +#define DEF_ORACLE_TIMESTAMP_TZ_CS_FUNCS(OBJTYPE) \ + template <> \ + inline int64_t obj_crc64(const ObObj& obj, const int64_t current) \ + { \ + int type = obj.get_type(); \ + int64_t ret = ob_crc64_sse42(current, &type, sizeof(type)); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + ret = ob_crc64_sse42(ret, &tmp_data.time_us_, sizeof(int64_t)); \ + ret = ob_crc64_sse42(ret, &tmp_data.time_ctx_.desc_, sizeof(uint32_t)); \ + return ret; \ + } \ + template <> \ + inline void obj_batch_checksum(const ObObj& obj, ObBatchChecksum& bc) \ + { \ + int type = obj.get_type(); \ + bc.fill(&type, sizeof(type)); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + bc.fill(&tmp_data.time_us_, sizeof(int64_t)); \ + bc.fill(&tmp_data.time_ctx_.desc_, sizeof(uint32_t)); \ + } \ + template \ + struct ObjHashCalculator { \ + static uint64_t calc_hash_value(const ObObj& obj, const uint64_t hash) \ + { \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + uint64_t ret = T::hash(&tmp_data.time_us_, static_cast(sizeof(int64_t)), hash); \ + ret = T::hash(&tmp_data.time_ctx_.desc_, static_cast(sizeof(uint32_t)), ret); \ + return ret; \ + } \ + }; \ + template <> \ + inline uint64_t obj_murmurhash(const ObObj& obj, const uint64_t hash) \ + { \ + int type = obj.get_type(); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + uint64_t ret = murmurhash(&type, sizeof(type), hash); \ + ret = murmurhash(&tmp_data.time_us_, static_cast(sizeof(int64_t)), ret); \ + ret = murmurhash(&tmp_data.time_ctx_.desc_, static_cast(sizeof(uint32_t)), ret); \ + return ret; \ + } \ + +#define DEF_ORACLE_TIMESTAMP_TZ_SERIALIZE_FUNCS(OBJTYPE) \ + template <> \ + inline int obj_val_serialize(const ObObj& obj, char* buf, const int64_t buf_len, int64_t& pos) \ + { \ + const ObOTimestampData& ot_data = obj.get_otimestamp_value(); \ + return serialization::encode_otimestamp_tz_type(buf, buf_len, pos, ot_data.time_us_, ot_data.time_ctx_.desc_); \ + } \ + \ + template <> \ + inline int obj_val_deserialize(ObObj & obj, const char* buf, const int64_t buf_len, int64_t& pos) \ + { \ + int ret = OB_SUCCESS; \ + ObOTimestampData ot_data; \ + ret = serialization::decode_otimestamp_tz_type( \ + buf, buf_len, pos, *((int64_t*)&ot_data.time_us_), *((uint32_t*)&ot_data.time_ctx_.desc_)); \ + obj.set_otimestamp_value(OBJTYPE, ot_data); \ + return ret; \ + } \ + template <> \ + inline int64_t obj_val_get_serialize_size(const ObObj& obj) \ + { \ + UNUSED(obj); \ + return serialization::encode_length_otimestamp_tz_type(); \ + } \ + +#define DEF_ORACLE_TIMESTAMP_CS_FUNCS(OBJTYPE) \ + template <> \ + inline int64_t obj_crc64(const ObObj& obj, const int64_t current) \ + { \ + int type = obj.get_type(); \ + int64_t ret = ob_crc64_sse42(current, &type, sizeof(type)); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + ret = ob_crc64_sse42(ret, &tmp_data.time_us_, sizeof(int64_t)); \ + ret = ob_crc64_sse42(ret, &tmp_data.time_ctx_.time_desc_, sizeof(uint16_t)); \ + return ret; \ + } \ + template <> \ + inline void obj_batch_checksum(const ObObj& obj, ObBatchChecksum& bc) \ + { \ + int type = obj.get_type(); \ + bc.fill(&type, sizeof(type)); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + bc.fill(&tmp_data.time_us_, sizeof(int64_t)); \ + bc.fill(&tmp_data.time_ctx_.time_desc_, sizeof(uint16_t)); \ + } \ + template \ + struct ObjHashCalculator { \ + static uint64_t calc_hash_value(const ObObj &obj, const uint64_t hash) \ + { \ + uint64_t ret = OB_SUCCESS; \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + ret = T::hash(&tmp_data.time_us_, static_cast(sizeof(int64_t)), hash); \ + ret = T::hash(&tmp_data.time_ctx_.time_desc_, static_cast(sizeof(uint16_t)), ret); \ + return ret; \ + } \ + }; \ + template <> \ + inline uint64_t obj_murmurhash(const ObObj &obj, const uint64_t hash) \ + { \ + int type = obj.get_type(); \ + uint64_t ret = murmurhash(&type, sizeof(type), hash); \ + ObOTimestampData tmp_data = obj.get_otimestamp_value(); \ + ret = murmurhash(&tmp_data.time_us_, static_cast(sizeof(int64_t)), ret); \ + ret = murmurhash(&tmp_data.time_ctx_.time_desc_, static_cast(sizeof(uint16_t)), ret); \ + return ret; \ + } \ + +#define DEF_ORACLE_TIMESTAMP_SERIALIZE_FUNCS(OBJTYPE) \ + template <> \ + inline int obj_val_serialize(const ObObj &obj, char *buf, const int64_t buf_len, int64_t &pos) \ + { \ + const ObOTimestampData &ot_data = obj.get_otimestamp_value(); \ + return serialization::encode_otimestamp_type(buf, buf_len, pos, ot_data.time_us_, ot_data.time_ctx_.time_desc_); \ + } \ + template <> \ + inline int obj_val_deserialize(ObObj &obj, const char *buf, const int64_t buf_len, int64_t &pos) \ + { \ + int ret = OB_SUCCESS; \ + ObOTimestampData ot_data; \ + ret = serialization::decode_otimestamp_type(buf, buf_len, pos, \ + *((int64_t*)&ot_data.time_us_), \ + *((uint16_t*)&ot_data.time_ctx_.time_desc_)); \ + obj.set_otimestamp_value(OBJTYPE, ot_data); \ + return ret; \ + } \ + template <> \ + inline int64_t obj_val_get_serialize_size(const ObObj& obj) \ + { \ + UNUSED(obj); \ + return serialization::encode_length_otimestamp_type(); \ + } + +#define DEF_ORACLE_TIMESTAMP_TZ_FUNCS(OBJTYPE) \ + DEF_ORACLE_TIMESTAMP_COMMON_PRINT_FUNCS(OBJTYPE); \ + DEF_ORACLE_TIMESTAMP_TZ_CS_FUNCS(OBJTYPE); \ + DEF_ORACLE_TIMESTAMP_TZ_SERIALIZE_FUNCS(OBJTYPE) + +#define DEF_ORACLE_TIMESTAMP_FUNCS(OBJTYPE) \ + DEF_ORACLE_TIMESTAMP_COMMON_PRINT_FUNCS(OBJTYPE); \ + DEF_ORACLE_TIMESTAMP_CS_FUNCS(OBJTYPE); \ + DEF_ORACLE_TIMESTAMP_SERIALIZE_FUNCS(OBJTYPE) + +DEF_ORACLE_TIMESTAMP_TZ_FUNCS(ObTimestampTZType); +DEF_ORACLE_TIMESTAMP_FUNCS(ObTimestampLTZType); +DEF_ORACLE_TIMESTAMP_FUNCS(ObTimestampNanoType); + // ObRawType=39 -#define DEF_RAW_PRINT_FUNCS(OBJTYPE) \ +#define DEF_RAW_PRINT_FUNCS(OBJTYPE) \ template <> \ inline int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ - const ObTimeZoneInfo *tz_info) \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = databuff_printf(buffer, length, pos, "'"))) { \ @@ -1038,9 +1360,9 @@ template <> } \ template <> \ inline int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ - const ObTimeZoneInfo *tz_info) \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = databuff_printf(buffer, length, pos, "'"))) { \ @@ -1052,9 +1374,9 @@ template <> } \ template <> \ inline int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, \ - int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int64_t &pos, const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ ObString str = obj.get_string(); \ if (OB_SUCCESS != (ret = hex_print(str.ptr(), str.length(), buffer, length, pos))) { \ @@ -1063,9 +1385,9 @@ template <> } \ template <> \ inline int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ - const ObTimeZoneInfo *tz_info) \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ J_OBJ_START(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ @@ -1086,12 +1408,13 @@ template <> DEF_RAW_FUNCS(ObRawType, raw, ObString); -#define DEF_NVARCHAR_PRINT_FUNCS(OBJTYPE) \ +#define DEF_NVARCHAR_PRINT_FUNCS(OBJTYPE) \ template <> \ - inline int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info) \ + inline int obj_print_sql(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ - int ret = OB_SUCCESS; \ + UNUSED(params); \ + int ret = OB_SUCCESS; \ if (OB_FAIL(databuff_printf(buffer, length, pos, "n'"))) { \ } else if (ObCharset::charset_type_by_coll(obj.get_collation_type()) == CHARSET_UTF8MB4) { \ ObHexEscapeSqlStr sql_str(obj.get_string()); \ @@ -1123,9 +1446,9 @@ DEF_RAW_FUNCS(ObRawType, raw, ObString); } \ template <> \ inline int obj_print_str(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, \ - const ObTimeZoneInfo *tz_info) \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ uint32_t result_len = 0; \ if (OB_FAIL(databuff_printf(buffer, length, pos, "'"))) { \ @@ -1142,9 +1465,9 @@ DEF_RAW_FUNCS(ObRawType, raw, ObString); } \ template <> \ inline int obj_print_plain_str(const ObObj &obj, char *buffer, int64_t length, \ - int64_t &pos, const ObTimeZoneInfo *tz_info) \ + int64_t &pos, const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ int ret = OB_SUCCESS; \ uint32_t result_len = 0; \ if (OB_FAIL(ObCharset::charset_convert(obj.get_collation_type(), \ @@ -1159,18 +1482,18 @@ DEF_RAW_FUNCS(ObRawType, raw, ObString); } \ template <> \ inline int obj_print_json(const ObObj &obj, char *buf, int64_t buf_len, int64_t &pos, \ - const ObTimeZoneInfo *tz_info) \ + const ObObjPrintParams ¶ms) \ { \ - UNUSED(tz_info); \ + UNUSED(params); \ J_OBJ_START(); \ PRINT_META(); \ BUF_PRINTO(ob_obj_type_str(obj.get_type())); \ J_COLON(); \ uint32_t result_len = 0; \ ObCharset::charset_convert(obj.get_collation_type(), \ - obj.get_string().ptr(), obj.get_string().length(),\ + obj.get_string().ptr(), obj.get_string().length(), \ ObCharset::get_system_collation(), \ - buf + pos, static_cast(buf_len - pos), \ + buf + pos, static_cast(buf_len - pos), \ result_len); \ pos += result_len; \ J_COMMA(); \ @@ -1181,7 +1504,7 @@ DEF_RAW_FUNCS(ObRawType, raw, ObString); #define DEF_NVARCHAR_FUNCS(OBJTYPE, TYPE, VTYPE) \ DEF_NVARCHAR_PRINT_FUNCS(OBJTYPE); \ - DEF_STRING_CS_FUNCS(OBJTYPE); \ + DEF_STRING_CS_FUNCS(OBJTYPE); \ DEF_SERIALIZE_FUNCS(OBJTYPE, TYPE, VTYPE) DEF_NVARCHAR_FUNCS(ObNVarchar2Type, nvarchar2, ObString); diff --git a/src/common/ob_obj_type.cpp b/src/common/ob_obj_type.cpp index 2d83dbe4e3b3ae914b47bef9ddbbf9324897e666..16312b14b59c1e2706c43db1dfcf67047d8dc06a 100644 --- a/src/common/ob_obj_type.cpp +++ b/src/common/ob_obj_type.cpp @@ -273,6 +273,14 @@ const char *ob_sql_tc_str(ObObjTypeClass tc) "EXT", "UNKNOWN", "TEXT", + "BIT", + "ENUMSET", + "ENUMSETINNER", + "OTIMESTAMP", + "RAW", + "INTERVAL", + "ROWID", + "LOB", "" }; return sql_tc_name[OB_LIKELY(tc < ObMaxTC) ? tc : ObMaxTC]; diff --git a/src/common/ob_obj_type.h b/src/common/ob_obj_type.h index 17e908c8e731c134c2d270949bfd9ee1aa1374ad..625e4bba80bc3d16da38d442fa7e2d8466c3d60f 100644 --- a/src/common/ob_obj_type.h +++ b/src/common/ob_obj_type.h @@ -89,7 +89,8 @@ enum ObObjType ObNVarchar2Type = 43, // nvarchar2 ObNCharType = 44, // nchar ObURowIDType = 45, // UROWID - ObMaxType // invalid type, or count of obj type + ObLobType = 46, // Oracle Lob + ObMaxType // invalid type, or count of obj type }; enum ObObjTypeClass @@ -107,7 +108,15 @@ enum ObObjTypeClass ObStringTC = 10, // varchar, char, varbinary, binary. ObExtendTC = 11, // extend ObUnknownTC = 12, // unknown - ObTextTC = 13, //TinyText,MediumText, Text ,LongText + ObTextTC = 13, // TinyText, MediumText, Text, LongText + ObBitTC = 14, // bit + ObEnumSetTC = 15, // enum, set + ObEnumSetInnerTC = 16, + ObOTimestampTC = 17, // timestamp with time zone + ObRawTC = 18, // raw + ObIntervalTC = 19, // oracle interval type class include interval year to month and interval day to second + ObRowIDTC = 20, // oracle rowid typeclass, includes urowid and rowid + ObLobTC = 21, // oracle lob typeclass ObMaxTC, // invalid type classes are below, only used as the result of XXXX_type_promotion() // to indicate that the two obj can't be promoted to the same type. @@ -119,57 +128,83 @@ enum ObObjTypeClass static ObObjTypeClass OBJ_TYPE_TO_CLASS[ObMaxType] = { - ObNullTC, // null - ObIntTC, // int8 - ObIntTC, // int16 - ObIntTC, // int24 - ObIntTC, // int32 - ObIntTC, // int64 - ObUIntTC, // uint8 - ObUIntTC, // uint16 - ObUIntTC, // uint24 - ObUIntTC, // uint32 - ObUIntTC, // uint64 - ObFloatTC, // float - ObDoubleTC, // double - ObFloatTC, // ufloat - ObDoubleTC, // udouble - ObNumberTC, // number - ObNumberTC, // unumber - ObDateTimeTC, // datetime - ObDateTimeTC, // timestamp - ObDateTC, // date - ObTimeTC, // time - ObYearTC, // year - ObStringTC, // varchar - ObStringTC, // char - ObStringTC, // hexadecimal literal - ObExtendTC, // extend - ObUnknownTC, // unknown - ObTextTC, // TinyText - ObTextTC, // MediumText - ObTextTC, // Text - ObTextTC // LongText + ObNullTC, // null obNullType + ObIntTC, // int8 ObTinyIntType + ObIntTC, // int16 ObSmallIntType + ObIntTC, // int24 ObMediumIntType + ObIntTC, // int32 ObInt32Type + ObIntTC, // int64 ObIntType + ObUIntTC, // uint8 ObUTinyIntType + ObUIntTC, // uint16 ObUSmallIntType + ObUIntTC, // uint24 ObUMediumIntType + ObUIntTC, // uint32 ObUInt32Type + ObUIntTC, // uint64 ObUInt64Type + ObFloatTC, // float ObFloatType + ObDoubleTC, // double ObDoubleType + ObFloatTC, // ufloat ObUFloatType + ObDoubleTC, // udouble ObUDoubleType + ObNumberTC, // number ObNumberType + ObNumberTC, // unumber ObUNumberType + ObDateTimeTC, // datetime ObDateTimeType + ObDateTimeTC, // timestamp ObTimestampType + ObDateTC, // date ObDateType + ObTimeTC, // time ObTimeType + ObYearTC, // year ObYearType + ObStringTC, // varchar ObVarcharType + ObStringTC, // char ObCharType + ObStringTC, // hexadecimal literal ObHexStringType + ObExtendTC, // extend ObExtendType + ObUnknownTC, // unknown ObUnknownType + ObTextTC, // TinyText ObTinyTextType + ObTextTC, // Text ObTextType + ObTextTC, // MediumText ObMediumTextType + ObTextTC, // LongText ObLongTextType + ObBitTC, // ObBitType + ObEnumSetTC, // ObEnumType + ObEnumSetTC, // ObSetType + ObEnumSetInnerTC, // ObEnumInnerType + ObEnumSetInnerTC, // ObSetInnerType + ObOTimestampTC, // ObTimestampTZType + ObOTimestampTC, // ObTimestampLTZType + ObOTimestampTC, // ObTimestampNanoType + ObRawTC, // ObRawType + ObIntervalTC, // ObIntervalYMType + ObIntervalTC, // ObIntervalDSType + ObNumberTC, // ObNumberFloatType + ObStringTC, // ObNVarchar2Type + ObStringTC, // ObNCharType + ObRowIDTC, // ObURowIDType + ObLobTC // ObLobType }; static ObObjType OBJ_DEFAULT_TYPE[ObActualMaxTC] = { - ObNullType, // null - ObIntType, // int - ObUInt64Type, // uint - ObFloatType, // float - ObDoubleType, // double - ObNumberType, // number - ObDateTimeType, // datetime - ObDateType, // date - ObTimeType, // time - ObYearType, // year - ObVarcharType, // varchar - ObExtendType, // extend - ObUnknownType, // unknown + ObNullType, // null + ObIntType, // int + ObUInt64Type, // uint + ObFloatType, // float + ObDoubleType, // double + ObNumberType, // number + ObDateTimeType, // datetime + ObDateType, // date + ObTimeType, // time + ObYearType, // year + ObVarcharType, // varchar + ObExtendType, // extend + ObUnknownType, // unknown ObLongTextType, // text - ObMaxType, // maxtype - ObUInt64Type, // int&uint + ObBitType, // bit + ObUInt64Type, // enumset + ObMaxType, // enumsetInner + ObTimestampNanoType, // timestamp nano + ObRawType, // raw + ObMaxType, // no default type for interval type class + ObMaxType, // no default type for rowid type class + ObLobType, // lob + ObMaxType, // maxtype + ObUInt64Type, // int&uint + ObMaxType, // lefttype + ObMaxType, // righttype }; OB_INLINE ObObjTypeClass ob_obj_type_class(const ObObjType type) @@ -237,16 +272,30 @@ inline bool ob_is_number_tc(ObObjType type) { return ObNumberTC == ob_obj_type_c inline bool ob_is_datetime_tc(ObObjType type) { return ObDateTimeTC == ob_obj_type_class(type); } inline bool ob_is_time_tc(ObObjType type) { return ObTimeTC == ob_obj_type_class(type); } inline bool ob_is_string_tc(ObObjType type) { return ObStringTC == ob_obj_type_class(type); } +inline bool ob_is_text_tc(ObObjType type) { return ObTextTC == ob_obj_type_class(type); } inline bool ob_is_nvarchar2(const ObObjType type) { return ObNVarchar2Type == type; } inline bool ob_is_nchar(const ObObjType type) { return ObNCharType == type; } -inline bool ob_is_nstring_type(const ObObjType type) +inline bool ob_is_nstring_type(const ObObjType type) { return ob_is_nchar(type) || ob_is_nvarchar2(type); } +inline bool ob_is_accurate_numeric_type(ObObjType type) { - return ob_is_nchar(type) || ob_is_nvarchar2(type); + return (ObTinyIntType <= type && type <= ObUInt64Type) || (ob_is_number_tc(type)); +} +inline bool ob_is_otimestamp_type(const ObObjType type) +{ + return (ObTimestampTZType <= type && type <= ObTimestampNanoType); +} +inline bool ob_is_timestamp_tz(const ObObjType type) +{ + return ObTimestampTZType == type; +} +inline bool ob_is_blob(const ObObjType type, const ObCollationType cs_type) +{ + return ObTextTC == ob_obj_type_class(type) && CS_TYPE_BINARY == cs_type; } - inline bool is_obj_type_supported(ObObjType type) { - return type > ObNullType && type < ObUnknownType; + return (type > ObNullType && type < ObUnknownType) + || ob_is_otimestamp_type(type); } // to_string adapter diff --git a/src/common/ob_object.cpp b/src/common/ob_object.cpp index f534460557de575708fa3ce05aeb59af7f765fbb..33b29d482f78d3bb1c4f27332c56ca498f9004d9 100644 --- a/src/common/ob_object.cpp +++ b/src/common/ob_object.cpp @@ -28,6 +28,7 @@ #include "lib/allocator/ob_stack_allocator.h" #include "lib/number/ob_number_v2.h" #include "common/ob_obj_funcs.h" + using namespace oceanbase; using namespace oceanbase::common; @@ -230,6 +231,12 @@ int ObObj::build_not_strict_default_value() set_char(null_str); } break; + case ObTimestampTZType: + case ObTimestampLTZType: + case ObTimestampNanoType: { + set_otimestamp_null(data_type); + break; + } default: ret = OB_INVALID_ARGUMENT; _OB_LOG(WARN, "unexpected data type=%hhd", data_type); @@ -421,7 +428,22 @@ ObObjTypeFuncs OBJ_FUNCS[ObMaxType] = DEF_FUNC_ENTRY(ObHexStringType), // 24, hex_string DEF_FUNC_ENTRY(ObExtendType), // 25, ext DEF_FUNC_ENTRY(ObUnknownType), // 26, unknown - DEF_FUNC_ENTRY(ObRawType), // 39, timestamp (9) + DEF_FUNC_ENTRY(ObNullType), // 27 + DEF_FUNC_ENTRY(ObNullType), // 28 + DEF_FUNC_ENTRY(ObNullType), // 29 + DEF_FUNC_ENTRY(ObNullType), // 30 + DEF_FUNC_ENTRY(ObNullType), // 31 + DEF_FUNC_ENTRY(ObNullType), // 32 + DEF_FUNC_ENTRY(ObNullType), // 33 + DEF_FUNC_ENTRY(ObNullType), // 34 + DEF_FUNC_ENTRY(ObNullType), // 35 + DEF_FUNC_ENTRY(ObTimestampTZType), // 36, timestamp with time zone + DEF_FUNC_ENTRY(ObTimestampLTZType), // 37, timestamp with local time zone + DEF_FUNC_ENTRY(ObTimestampNanoType), // 38, timestamp (9) + DEF_FUNC_ENTRY(ObRawType), // 39, raw + DEF_FUNC_ENTRY(ObNullType), // 40 + DEF_FUNC_ENTRY(ObNullType), // 41 + DEF_FUNC_ENTRY(ObNullType), // 42 DEF_FUNC_ENTRY(ObNVarchar2Type), // 43, nvarchar2 DEF_FUNC_ENTRY(ObNCharType), // 44, nchar }; diff --git a/src/common/ob_object.h b/src/common/ob_object.h index beccff8ccf7542983de4233cbe0e75c3a360b158..8db10adb4d769215931aa6bfdc00b068592ccc57 100644 --- a/src/common/ob_object.h +++ b/src/common/ob_object.h @@ -133,6 +133,12 @@ public: set_collation_level(CS_LEVEL_INVALID); set_collation_type(CS_TYPE_BINARY); } + OB_INLINE void set_otimestamp_type(const ObObjType type) + { + type_ = static_cast(type); + set_collation_level(CS_LEVEL_NUMERIC); + set_collation_type(CS_TYPE_BINARY); + } OB_INLINE bool is_valid() const { return ob_is_valid_obj_type(static_cast(type_)); } OB_INLINE bool is_invalid() const { return !ob_is_valid_obj_type(static_cast(type_)); } @@ -188,6 +194,14 @@ public: OB_INLINE bool is_integer_type() const { return ob_is_integer_type(get_type()); } OB_INLINE bool is_string_type() const { return ob_is_string_tc(get_type()); } OB_INLINE bool is_temporal_type() const { return ob_is_temporal_type(get_type()); } + OB_INLINE bool is_nchar() const { return type_ == static_cast(ObNCharType); } + OB_INLINE bool is_nvarchar2() const { return type_ == static_cast(ObNVarchar2Type); } + OB_INLINE bool is_nstring() const { return is_nvarchar2() || is_nchar(); } + OB_INLINE bool is_blob() const { return (ob_is_text_tc(get_type()) && CS_TYPE_BINARY == cs_type_); } + OB_INLINE bool is_character_type() const { return is_nstring() || is_varchar_or_char(); } + OB_INLINE bool is_timestamp_tz() const { return type_ == static_cast(ObTimestampTZType); } + OB_INLINE bool is_timestamp_ltz() const { return type_ == static_cast(ObTimestampLTZType); } + OB_INLINE bool is_timestamp_nano() const { return type_ == static_cast(ObTimestampNanoType); } OB_INLINE bool is_unsigned_integer() const { return (static_cast(ObUTinyIntType) <= type_ @@ -222,6 +236,28 @@ private: int8_t scale_; // scale }; +struct ObObjPrintParams { + ObObjPrintParams(const ObTimeZoneInfo* tz_info, ObCollationType cs_type) + : tz_info_(tz_info), cs_type_(cs_type), print_flags_(0) + {} + ObObjPrintParams(const ObTimeZoneInfo* tz_info) + : tz_info_(tz_info), cs_type_(CS_TYPE_UTF8MB4_GENERAL_CI), print_flags_(0) + {} + ObObjPrintParams() : tz_info_(NULL), cs_type_(CS_TYPE_UTF8MB4_GENERAL_CI), print_flags_(0) + {} + TO_STRING_KV(K_(tz_info), K_(cs_type)); + const ObTimeZoneInfo* tz_info_; + ObCollationType cs_type_; + union { + uint32_t print_flags_; + struct { + uint32_t need_cast_expr_ : 1; + uint32_t is_show_create_view_ : 1; + uint32_t reserved_ : 30; + }; + }; +}; + // sizeof(ObObjValue)=8 union ObObjValue { @@ -339,11 +375,14 @@ public: void set_datetime_value(const int64_t value); void set_timestamp_value(const int64_t value); + void set_otimestamp_value(const ObObjType type, const ObOTimestampData &value); + void set_otimestamp_value(const ObObjType type, const int64_t time_us, const uint32_t time_ctx_desc); + void set_otimestamp_value(const ObObjType type, const int64_t time_us, const uint16_t time_desc); + void set_otimestamp_null(const ObObjType type); void set_date_value(const int32_t value); void set_time_value(const int64_t value); void set_year_value(const uint8_t value); - void set_string(const ObObjType type, const char *ptr, const ObString::obstr_size_t size); void set_string(const ObObjType type, const ObString &value); void set_varchar(const ObString &value); @@ -385,6 +424,7 @@ public: v_.string_ = ptr; val_len_ = static_cast(size); } + //@} //@{ getters @@ -474,6 +514,7 @@ public: OB_INLINE ObString get_raw() const { return ObString(val_len_, v_.string_); } OB_INLINE ObString get_binary() const { return ObString(val_len_, v_.string_); } OB_INLINE ObString get_hex_string() const { return ObString(val_len_, v_.string_); } + OB_INLINE int64_t get_number_byte_length() const { return nmb_desc_.len_ * sizeof(uint32_t); } OB_INLINE bool get_bool() const { return (0 != v_.int64_); } inline int64_t get_ext() const; @@ -487,6 +528,24 @@ public: inline ObString get_nvarchar2() const { return ObString(val_len_, v_.string_); } inline ObString get_nchar() const { return ObString(val_len_, v_.string_); } + + inline ObOTimestampData::UnionTZCtx get_tz_desc() const + { + return time_ctx_; + } + inline ObOTimestampData get_otimestamp_value() const + { + return ObOTimestampData(v_.datetime_, time_ctx_); + } + static int64_t get_otimestamp_store_size(const bool is_timestamp_tz) + { + return static_cast(sizeof(int64_t) + (is_timestamp_tz ? sizeof(uint32_t) : sizeof(uint16_t))); + } + inline int64_t get_otimestamp_store_size() const + { + return get_otimestamp_store_size(is_timestamp_tz()); + } + //@} //@{ test functions @@ -531,6 +590,11 @@ public: OB_INLINE bool is_temporal_type() const { return meta_.is_temporal_type(); } OB_INLINE bool is_varchar_or_char() const { return meta_.is_varchar_or_char(); } OB_INLINE bool is_varying_len_char_type() const { return meta_.is_varying_len_char_type(); } + OB_INLINE bool is_character_type() const { return meta_.is_character_type(); } + OB_INLINE bool is_blob() const { return meta_.is_blob(); } + OB_INLINE bool is_timestamp_tz() const { return meta_.is_timestamp_tz(); } + OB_INLINE bool is_timestamp_nano() const { return meta_.is_timestamp_nano(); } + OB_INLINE bool is_varbinary_or_binary() const { return meta_.is_varbinary_or_binary(); } inline bool is_min_value() const; inline bool is_max_value() const; @@ -611,6 +675,7 @@ public: { int32_t val_len_; number::ObNumber::Desc nmb_desc_; + ObOTimestampData::UnionTZCtx time_ctx_; }; // sizeof = 4 ObObjValue v_; // sizeof = 8 }; @@ -947,6 +1012,36 @@ inline void ObObj::set_timestamp_value(const int64_t value) v_.datetime_ = value; } +inline void ObObj::set_otimestamp_value(const ObObjType type, const ObOTimestampData &value) +{ + meta_.set_otimestamp_type(type); + time_ctx_ = value.time_ctx_; + v_.datetime_ = value.time_us_; +} + +inline void ObObj::set_otimestamp_value(const ObObjType type, const int64_t time_us, const uint32_t time_ctx_desc) +{ + meta_.set_otimestamp_type(type); + time_ctx_.desc_ = time_ctx_desc; + v_.datetime_ = time_us; +} + +inline void ObObj::set_otimestamp_value(const ObObjType type, const int64_t time_us, const uint16_t time_desc) +{ + meta_.set_otimestamp_type(type); + time_ctx_.tz_desc_ = 0; + time_ctx_.time_desc_ = time_desc; + v_.datetime_ = time_us; +} + +inline void ObObj::set_otimestamp_null(const ObObjType type) +{ + meta_.set_otimestamp_type(type); + time_ctx_.tz_desc_ = 0; + time_ctx_.time_desc_ = 0; + time_ctx_.is_null_ = 1; +} + inline void ObObj::set_date(const int32_t value) { meta_.set_date(); @@ -1662,7 +1757,7 @@ private: ParamFlag flag_; }; -typedef int (*ob_obj_print)(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObTimeZoneInfo *tz_info); +typedef int (*ob_obj_print)(const ObObj &obj, char *buffer, int64_t length, int64_t &pos, const ObObjPrintParams ¶ms); typedef int64_t (*ob_obj_crc64)(const ObObj &obj, const int64_t current); typedef void (*ob_obj_batch_checksum)(const ObObj &obj, ObBatchChecksum &bc); typedef uint64_t (*ob_obj_hash)(const ObObj &obj, const uint64_t hash); diff --git a/src/common/obsm_utils.cpp b/src/common/obsm_utils.cpp index a634b5396fc412d31672d86716a61a2ceb1dac6f..db2a841c2e7e6cedabe81ab099c764ad2c3138b3 100644 --- a/src/common/obsm_utils.cpp +++ b/src/common/obsm_utils.cpp @@ -134,9 +134,13 @@ int ObSMUtils::cell_str( case ObYearTC: ret = ObMySQLUtil::year_cell_str(buf, len, obj.get_year(), type, pos); break; + case ObRawTC: + case ObTextTC: // TODO@hanhui texttc share the stringtc temporarily case ObStringTC: + case ObLobTC: { ret = ObMySQLUtil::varchar_cell_str(buf, len, obj.get_string(), pos); break; + } default: _OB_LOG(ERROR, "invalid ob type=%d", obj.get_type()); ret = OB_ERROR; @@ -309,3 +313,35 @@ int ObSMUtils::get_ob_type(ObObjType &ob_type, EMySQLFieldType mysql_type) } return ret; } + +int ObSMUtils::to_ob_field(const ObMySQLField &field, ObField &mfield) +{ + int ret = OB_SUCCESS; + mfield.dname_ = field.dname_; + mfield.tname_ = field.tname_; + mfield.org_tname_ = field.org_tname_; + mfield.cname_ = field.cname_; + mfield.org_cname_ = field.org_cname_; + + mfield.accuracy_ = field.accuracy_; + + // To distinguish between binary and nonbinary data for string data types, + // check whether the charsetnr value is 63. Also, flag must be set to binary accordingly + mfield.charsetnr_ = field.charsetnr_; + mfield.flags_ = field.flags_; + mfield.length_ = field.length_; + + // For Varchar class, check charset: + // TODO:Handling incorrect character sets + if (ObCharset::is_valid_collation(static_cast(field.charsetnr_)) + && ObCharset::is_bin_sort(static_cast(field.charsetnr_))) { + mfield.flags_ |= OB_MYSQL_BINARY_FLAG; + } + + ObObjType ob_type; + ret = ObSMUtils::get_ob_type(ob_type, field.type_); + mfield.type_.set_type(ob_type); + + OB_LOG(DEBUG, "to ob field", K(ret), K(mfield), K(field)); + return ret; +} diff --git a/src/common/obsm_utils.h b/src/common/obsm_utils.h index 56fe2064482bd946fa80891535a72c41da3a4f4e..80217dc2c76585145816be31b444d42813b181d4 100644 --- a/src/common/obsm_utils.h +++ b/src/common/obsm_utils.h @@ -19,6 +19,7 @@ #include "lib/timezone/ob_timezone_info.h" #include "rpc/obmysql/ob_mysql_global.h" #include "rpc/obmysql/ob_mysql_util.h" +#include "rpc/obmysql/ob_mysql_field.h" #include "common/ob_object.h" #include "common/ob_accuracy.h" @@ -55,6 +56,7 @@ public: uint16_t &flags, ObScale &num_decimals); static int get_ob_type(ObObjType &ob_type, obmysql::EMySQLFieldType mysql_type); + static int to_ob_field(const obmysql::ObMySQLField &field, ObField &mfield); }; } // end of namespace common diff --git a/src/lib/Makemodule.am b/src/lib/Makemodule.am index 7eb0581a2de7a118fe687ad9289b837efc47f23a..fab2bb59f4fe7ad0ae6fd7eb5b4b9d33b3fc637f 100644 --- a/src/lib/Makemodule.am +++ b/src/lib/Makemodule.am @@ -194,6 +194,8 @@ lib/timezone/ob_time_convert.h\ lib/timezone/ob_time_convert.cpp\ lib/timezone/ob_timezone_info.h\ lib/timezone/ob_timezone_info.cpp\ +lib/timezone/ob_oracle_format_models.h\ +lib/timezone/ob_oracle_format_models.cpp\ lib/utility/ob_preload.h\ lib/utility/ob_print_kv.h\ lib/utility/ob_print_utils.h\ diff --git a/src/lib/alloc/alloc_struct.h b/src/lib/alloc/alloc_struct.h index 24e3e6d93b8785ae53c1959f8c95bf00444025b0..5c0cf208e445e38b441bcbf42e6ff9640ee9db9a 100644 --- a/src/lib/alloc/alloc_struct.h +++ b/src/lib/alloc/alloc_struct.h @@ -17,6 +17,7 @@ #include #include #include "lib/utility/ob_macro_utils.h" +#include "lib/utility/ob_template_utils.h" namespace oceanbase { diff --git a/src/lib/alloc/object_set.cpp b/src/lib/alloc/object_set.cpp index a50553e8f673c46baddf940675428b63d8f3c425..bc223d7673ae960ec56a21f0ab9ab61a97ea81f8 100644 --- a/src/lib/alloc/object_set.cpp +++ b/src/lib/alloc/object_set.cpp @@ -384,7 +384,7 @@ void ObjectSet::free_object(AObject *obj) if (NULL == bm_ && NULL == free_lists_ && normal_hold_bytes_ > ABLOCK_SIZE - && (normal_used_bytes_ + && (static_cast(normal_used_bytes_) < static_cast(normal_hold_bytes_) * (1. - FREE_LISTS_BUILD_RATIO))) { build_free_lists(); diff --git a/src/lib/allocator/ob_mod_define.h b/src/lib/allocator/ob_mod_define.h index b63f60bb23c33fb5ef54248b34fc84b2ff6befe4..9be908f848173e1ebe0f2c8ef524f9304e6ff1c0 100644 --- a/src/lib/allocator/ob_mod_define.h +++ b/src/lib/allocator/ob_mod_define.h @@ -205,6 +205,7 @@ MOD_ITEM_DEF(OB_PARTITION_TABLE_TASK) MOD_ITEM_DEF(OB_TEMP_VARIABLES) MOD_ITEM_DEF(OB_EXTERNAL_SORTER) MOD_ITEM_DEF(OB_OBJ_STORE) +MOD_ITEM_DEF(OB_COMPRESSOR) //obproxy MOD_ITEM_DEF(OB_PROXY_SESSION) @@ -228,6 +229,12 @@ MOD_ITEM_DEF(OB_PROXY_PARAM_OBJ_ARRAY) MOD_ITEM_DEF(OB_PROXY_SHARDING_OPTIMIZER) MOD_ITEM_DEF(OB_PROXY_SHARDING_EXPR) MOD_ITEM_DEF(OB_PROXY_SHARDING_PARSE) +MOD_ITEM_DEF(OB_PROXY_SHARDING_CONFIG) +MOD_ITEM_DEF(OB_PROXY_SHARDING_DDL) +MOD_ITEM_DEF(OB_PROXY_QOS) +MOD_ITEM_DEF(OB_PROXY_SSL_RELATED) +MOD_ITEM_DEF(OB_PROXY_CONFIG_TABLE) +MOD_ITEM_DEF(OB_PROMETHEUS_RELATED) //mergeservermodules MOD_ITEM_DEF(OB_MS_CELL_ARRAY) @@ -633,7 +640,6 @@ MOD_ITEM_DEF(OB_TRANS_SUBMIT_LOG_TASK) MOD_ITEM_DEF(OB_TRANS_LOCATION_CACHE) MOD_ITEM_DEF(OB_TRANS_FREEZE_TASK) - // rootservice MOD_ITEM_DEF(OB_HASH_BUCKET_LEADER_STATUS_MAP) MOD_ITEM_DEF(OB_HASH_BUCKET_PARTITION_DIST_MAP) diff --git a/src/lib/allocator/ob_stack_allocator.cpp b/src/lib/allocator/ob_stack_allocator.cpp index 1dce04f3ea4bf3f9edc1a3366d4fca221f3a1f11..8eaec7b3ba7c59f61b37f874b7f6cb54a8f5182a 100644 --- a/src/lib/allocator/ob_stack_allocator.cpp +++ b/src/lib/allocator/ob_stack_allocator.cpp @@ -125,7 +125,7 @@ int StackAllocator::Block::init(const int64_t limit) int64_t StackAllocator::Block::remain() const { - return NULL == this ? -1 : limit_ - pos_; + return limit_ - pos_; } int StackAllocator::init(ObIAllocator *allocator, const int64_t block_size) diff --git a/src/lib/atomic/ob_atomic.h b/src/lib/atomic/ob_atomic.h index 00f0dbf54cb778e42275c12ab1e5863008705826..09f1147ad6cfe499be00835847be75098fefb2d8 100644 --- a/src/lib/atomic/ob_atomic.h +++ b/src/lib/atomic/ob_atomic.h @@ -85,6 +85,18 @@ inline int64_t inc_update(int64_t* v_, int64_t x) #define ATOMIC_INC(val) do { IGNORE_RETURN ATOMIC_AAF((val), 1); } while (0) #define ATOMIC_DEC(val) do { IGNORE_RETURN ATOMIC_SAF((val), 1); } while (0) +#define ATOMIC_LOAD64(addr) \ + ({ \ + int64_t x = __atomic_load_n((int64_t*)addr, __ATOMIC_SEQ_CST); \ + *(typeof(addr)) & x; \ + }) +#define ATOMIC_STORE64(addr, v) \ + ({ \ + typeof(v) v1 = v; \ + __atomic_store_n((int64_t*)addr, *(int64_t*)&v1, __ATOMIC_SEQ_CST); \ + }) + + } } diff --git a/src/lib/compress/Makemodule.am b/src/lib/compress/Makemodule.am index 586ff30594edfb44fe9955231f931c1c99c7b36c..ef1528c5066efdde4fd7a26b5fc5f861ac2af842 100644 --- a/src/lib/compress/Makemodule.am +++ b/src/lib/compress/Makemodule.am @@ -61,4 +61,58 @@ lib/compress/zlib/zutil.h lib_compress_libzlib_1_0_la_CPPFLAGS:= -I$(top_srcdir)/src -I${DEP_DIR}/include/ -w -noinst_LTLIBRARIES += lib/compress/libnone.la lib/compress/libzlib_1.0.la lib/compress/libsnappy_1.0.la lib/compress/liblz4_1.0.la +lib_compress_libzstd_1_3_8_la_SOURCES:=\ +lib/compress/zstd_1_3_8/error_private.c\ +lib/compress/zstd_1_3_8/error_private.h\ +lib/compress/zstd_1_3_8/entropy_common.c\ +lib/compress/zstd_1_3_8/debug.h\ +lib/compress/zstd_1_3_8/debug.c\ +lib/compress/zstd_1_3_8/cpu.h\ +lib/compress/zstd_1_3_8/compiler.h\ +lib/compress/zstd_1_3_8/bitstream.h\ +lib/compress/zstd_1_3_8/hist.h\ +lib/compress/zstd_1_3_8/hist.c\ +lib/compress/zstd_1_3_8/fse.h\ +lib/compress/zstd_1_3_8/fse_decompress.c\ +lib/compress/zstd_1_3_8/fse_compress.c\ +lib/compress/zstd_1_3_8/ob_zstd_wrapper.h\ +lib/compress/zstd_1_3_8/ob_zstd_wrapper.cpp\ +lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.h\ +lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.cpp\ +lib/compress/zstd_1_3_8/mem.h\ +lib/compress/zstd_1_3_8/huf.h\ +lib/compress/zstd_1_3_8/huf_decompress.c\ +lib/compress/zstd_1_3_8/huf_compress.c\ +lib/compress/zstd_1_3_8/xxhash.h\ +lib/compress/zstd_1_3_8/xxhash.c\ +lib/compress/zstd_1_3_8/threading.h\ +lib/compress/zstd_1_3_8/threading.c\ +lib/compress/zstd_1_3_8/pool.h\ +lib/compress/zstd_1_3_8/pool.c\ +lib/compress/zstd_1_3_8/zstd.h\ +lib/compress/zstd_1_3_8/zstd_common.c\ +lib/compress/zstd_1_3_8/zstd_compress.c\ +lib/compress/zstd_1_3_8/zstd_ddict.h\ +lib/compress/zstd_1_3_8/zstd_ddict.c\ +lib/compress/zstd_1_3_8/zstd_compress_internal.h\ +lib/compress/zstd_1_3_8/zstd_decompress_internal.h\ +lib/compress/zstd_1_3_8/zstd_decompress.c\ +lib/compress/zstd_1_3_8/zstd_decompress_block.h\ +lib/compress/zstd_1_3_8/zstd_decompress_block.c\ +lib/compress/zstd_1_3_8/zstd_internal.h\ +lib/compress/zstd_1_3_8/zstd_fast.h\ +lib/compress/zstd_1_3_8/zstd_fast.c\ +lib/compress/zstd_1_3_8/zstd_errors.h\ +lib/compress/zstd_1_3_8/zstd_double_fast.h\ +lib/compress/zstd_1_3_8/zstd_double_fast.c\ +lib/compress/zstd_1_3_8/zstd_opt.c\ +lib/compress/zstd_1_3_8/zstd_opt.h\ +lib/compress/zstd_1_3_8/zstd_ldm.h\ +lib/compress/zstd_1_3_8/zstd_ldm.c\ +lib/compress/zstd_1_3_8/zstd_lazy.h\ +lib/compress/zstd_1_3_8/zstd_lazy.c\ +lib/compress/zstd_1_3_8/zstdmt_compress.c\ +lib/compress/zstd_1_3_8/zstdmt_compress.h +lib_compress_libzstd_1_3_8_la_CPPFLAGS:= -I$(top_srcdir)/src -I${DEP_DIR}/include/ -w + +noinst_LTLIBRARIES += lib/compress/libnone.la lib/compress/libzlib_1.0.la lib/compress/libsnappy_1.0.la lib/compress/liblz4_1.0.la lib/compress/libzstd_1.3.8.la diff --git a/src/lib/compress/ob_compressor_pool.cpp b/src/lib/compress/ob_compressor_pool.cpp index 509fa28e7ec3e33163bdb20042bc56da660835a6..fe284840206cafe66ace35b8e2cdd52486d7c434 100644 --- a/src/lib/compress/ob_compressor_pool.cpp +++ b/src/lib/compress/ob_compressor_pool.cpp @@ -19,9 +19,9 @@ namespace common ObCompressorPool::ObCompressorPool() :none_compressor(), lz4_compressor(), - lzo_compressor(), snappy_compressor(), - zlib_compressor() + zlib_compressor(), + zstd_compressor_1_3_8() { } ObCompressorPool &ObCompressorPool::get_instance() @@ -41,32 +41,39 @@ int ObCompressorPool::get_compressor(const char *compressor_name, LIB_LOG(WARN, "invalid compressor name argument, ", K(ret), KP(compressor_name)); } else if (OB_FAIL(get_compressor_type(compressor_name, compressor_type))) { LIB_LOG(WARN, "fail to get compressor type, ", K(ret), K(compressor_name)); - } else { - switch(compressor_type) { - case NONE_COMPRESSOR: - compressor = &none_compressor; - break; - case LZ4_COMPRESSOR: - compressor = &lz4_compressor; - break; - case LZO_COMPRESSOR: - compressor = &lzo_compressor; - break; - case SNAPPY_COMPRESSOR: - compressor = &snappy_compressor; - break; - case ZLIB_COMPRESSOR: - compressor = &zlib_compressor; - break; - default: - compressor = NULL; - ret = OB_NOT_SUPPORTED; - LIB_LOG(WARN, "not support compress type, ", K(ret), K(compressor_type)); - } + } else if (OB_FAIL(get_compressor(compressor_type, compressor))) { + LIB_LOG(WARN, "fail to get compressor", K(ret), K(compressor_type)); } return ret; } +int ObCompressorPool::get_compressor(const ObCompressorType& compressor_type, ObCompressor*& compressor) +{ + int ret = OB_SUCCESS; + switch (compressor_type) { + case NONE_COMPRESSOR: + compressor = &none_compressor; + break; + case LZ4_COMPRESSOR: + compressor = &lz4_compressor; + break; + case SNAPPY_COMPRESSOR: + compressor = &snappy_compressor; + break; + case ZLIB_COMPRESSOR: + compressor = &zlib_compressor; + break; + case ZSTD_1_3_8_COMPRESSOR: + compressor = &zstd_compressor_1_3_8; + break; + default: + compressor = NULL; + ret = OB_NOT_SUPPORTED; + LIB_LOG(WARN, "not support compress type, ", K(ret), K(compressor_type)); + } + return ret; +} + int ObCompressorPool::get_compressor_type(const char *compressor_name, ObCompressorType &compressor_type) const { @@ -78,12 +85,12 @@ int ObCompressorPool::get_compressor_type(const char *compressor_name, compressor_type = NONE_COMPRESSOR; } else if (!strcmp(compressor_name, "lz4_1.0")) { compressor_type = LZ4_COMPRESSOR; - } else if (!strcmp(compressor_name, "lzo_1.0")) { - compressor_type = LZO_COMPRESSOR; } else if (!strcmp(compressor_name, "snappy_1.0")) { compressor_type = SNAPPY_COMPRESSOR; } else if (!strcmp(compressor_name, "zlib_1.0")) { compressor_type = ZLIB_COMPRESSOR; + } else if (!strcmp(compressor_name, "zstd_1.3.8")) { + compressor_type = ZSTD_1_3_8_COMPRESSOR; } else { ret = OB_NOT_SUPPORTED; LIB_LOG(WARN, "no support compressor type, ", K(ret), K(compressor_name)); diff --git a/src/lib/compress/ob_compressor_pool.h b/src/lib/compress/ob_compressor_pool.h index 079f90c1cc60f29f955bbcd500e163ce75d6c32a..4e25f999e93cd3e762aa94e6aafe4c43a03c120a 100644 --- a/src/lib/compress/ob_compressor_pool.h +++ b/src/lib/compress/ob_compressor_pool.h @@ -16,38 +16,39 @@ #include "lib/compress/ob_compressor.h" #include "none/ob_none_compressor.h" #include "lz4/ob_lz4_compressor.h" -#include "lzo/ob_lzo_compressor.h" #include "snappy/ob_snappy_compressor.h" #include "zlib/ob_zlib_compressor.h" +#include "zstd_1_3_8/ob_zstd_compressor_1_3_8.h" namespace oceanbase { namespace common { +enum ObCompressorType { + INVALID_COMPRESSOR = 0 , + NONE_COMPRESSOR , + LZ4_COMPRESSOR , + SNAPPY_COMPRESSOR , + ZLIB_COMPRESSOR , + ZSTD_1_3_8_COMPRESSOR +}; + class ObCompressorPool { public: static ObCompressorPool &get_instance(); int get_compressor(const char *compressor_name, ObCompressor *&compressor); + int get_compressor(const ObCompressorType& compressor_type, ObCompressor*& compressor); + int get_compressor_type(const char *compressor_name, ObCompressorType &compressor_type) const; private: ObCompressorPool(); virtual ~ObCompressorPool() {} - enum ObCompressorType { - INVALID_COMPRESSOR = 0 , - NONE_COMPRESSOR , - LZ4_COMPRESSOR , - LZO_COMPRESSOR , - SNAPPY_COMPRESSOR , - ZLIB_COMPRESSOR - }; - int get_compressor_type(const char *compressor_name, ObCompressorType &compressor_type) const; - ObNoneCompressor none_compressor; ObLZ4Compressor lz4_compressor; - ObLZOCompressor lzo_compressor; ObSnappyCompressor snappy_compressor; ObZlibCompressor zlib_compressor; + zstd_1_3_8::ObZstdCompressor_1_3_8 zstd_compressor_1_3_8; }; } /* namespace common */ diff --git a/src/lib/compress/zstd_1_3_8/LICENSE b/src/lib/compress/zstd_1_3_8/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a793a802892567f17d464a831e2e531dc8833f55 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/lib/compress/zstd_1_3_8/bitstream.h b/src/lib/compress/zstd_1_3_8/bitstream.h new file mode 100644 index 0000000000000000000000000000000000000000..bdb8c3f861fc4ee5c69fbaa3c5baa22f1b1c182a --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/bitstream.h @@ -0,0 +1,519 @@ +/* ****************************************************************** + bitstream + Part of FSE library + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined(__cplusplus) +extern "C" { +#endif + +/* + * This API consists of small unitary functions, which must be inlined for best performance. + * Since link-time-optimization is not available for all compilers, + * these functions are defined into a .h to be included. + */ + +/*-**************************************** + * Dependencies + ******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +#include /* support for bextr (experimental) */ +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + +/*-****************************************** + * bitStream encoding API (write forward) + ********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. + * bitStream will never write outside of this buffer. + * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. + * + * bits are first added to a local register. + * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. + * Writing data into memory is an explicit operation, performed by the flushBits function. + * Hence keep track how many bits are potentially stored into local register to avoid register overflow. + * After a flushBits, a maximum of 7 bits might still be stored into local register. + * + * Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. + * + * Last operation is to close the bitStream. + * The function returns the final size of CStream in bytes. + * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) + */ + +/*-******************************************** + * bitStream decoding API (read backward) + **********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { + BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 +} BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + +/* Start by invoking BIT_initDStream(). + * A chunk of the bitStream is then stored into a local register. + * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). + * You can then retrieve bitFields stored into the local register, **in reverse order**. + * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. + * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. + * Otherwise, it can be less than that, so proceed accordingly. + * Checking if DStream has reached its end can be performed with BIT_endOfDStream(). + */ + +/*-**************************************** + * unsafe API + ******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*-************************************************************** + * Internal functions + ****************************************************************/ +MEM_STATIC unsigned BIT_highbit32(U32 val) +{ + assert(val != 0); + { +#if defined(_MSC_VER) /* Visual */ + unsigned long r = 0; + _BitScanReverse(&r, val); + return (unsigned)r; +#elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz(val); +#else /* Software version */ + static const unsigned DeBruijnClz[32] = {0, + 9, + 1, + 10, + 13, + 21, + 2, + 29, + 11, + 14, + 16, + 18, + 22, + 25, + 3, + 30, + 8, + 12, + 20, + 28, + 15, + 17, + 24, + 7, + 19, + 27, + 23, + 6, + 26, + 5, + 4, + 31}; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; +#endif + } +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = {0, + 1, + 3, + 7, + 0xF, + 0x1F, + 0x3F, + 0x7F, + 0xFF, + 0x1FF, + 0x3FF, + 0x7FF, + 0xFFF, + 0x1FFF, + 0x3FFF, + 0x7FFF, + 0xFFFF, + 0x1FFFF, + 0x3FFFF, + 0x7FFFF, + 0xFFFFF, + 0x1FFFFF, + 0x3FFFFF, + 0x7FFFFF, + 0xFFFFFF, + 0x1FFFFFF, + 0x3FFFFFF, + 0x7FFFFFF, + 0xFFFFFFF, + 0x1FFFFFFF, + 0x3FFFFFFF, + 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** + * bitStream encoding + ****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) + return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + assert((value >> nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + assert(bitC->ptr <= bitC->endPtr); + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes * 8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) + bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes * 8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) + return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + +/*-******************************************************** + * bitStream decoding + **********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { + memset(bitD, 0, sizeof(*bitD)); + return ERROR(srcSize_wrong); + } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { + BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize - 1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) + return ERROR(GENERIC); /* endMark not present */ + } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch (srcSize) { + case 7: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16); + /* fall-through */ + + case 6: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24); + /* fall-through */ + + case 5: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32); + /* fall-through */ + + case 4: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + /* fall-through */ + + case 3: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + /* fall-through */ + + case 2: + bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + /* fall-through */ + + default: + break; + } + { + BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize - 1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) + return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize) * 8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer) * 8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer) * 8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer) * 8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask - nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer) * 8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask + 1) - nbBits) & regMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer) * 8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer) * 8) + return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { + U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes * 8; + bitD->bitContainer = MEM_readLEST( + bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer) * 8)); +} + +#if defined(__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/src/lib/compress/zstd_1_3_8/compiler.h b/src/lib/compress/zstd_1_3_8/compiler.h new file mode 100644 index 0000000000000000000000000000000000000000..f817e7e73cb030d0cee1ecf052bd53a8038f8270 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/compiler.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +/*-******************************************************* + * Compiler specifics + *********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if defined(__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +#define INLINE_KEYWORD inline +#else +#define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) +#define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define FORCE_INLINE_ATTR __forceinline +#else +#define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to elimininate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +#define HINT_INLINE static INLINE_KEYWORD +#else +#define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +#define FORCE_NOINLINE static __declspec(noinline) +#else +#ifdef __GNUC__ +#define FORCE_NOINLINE static __attribute__((__noinline__)) +#else +#define FORCE_NOINLINE static +#endif +#endif + +/* target attribute */ +#ifndef __has_attribute +#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ +#endif +#if defined(__GNUC__) +#define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +#define TARGET_ATTRIBUTE(target) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 +#if ((defined(__clang__) && __has_attribute(__target__)) || \ + (defined(__GNUC__) && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) && \ + (defined(__x86_64__) || defined(_M_X86)) && !defined(__BMI2__) +#define DYNAMIC_BMI2 1 +#else +#define DYNAMIC_BMI2 0 +#endif +#endif + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +#define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +#define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +#include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +#define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +#define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +#elif defined(__GNUC__) && ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) +#define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +#define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +#else +#define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +#define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) \ + { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos = 0; _pos < _size; _pos += CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +#include /* For Visual 2005 */ +#pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/src/lib/compress/zstd_1_3_8/cpu.h b/src/lib/compress/zstd_1_3_8/cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..a2d7638e97bd80e3e0a6658967186bdb85a418f0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/cpu.h @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2018-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) +{ + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__("pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__("pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" : "=a"(f7a), "=b"(f7b), "=c"(f7c) : "a"(7), "c"(0) : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) \ + { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) +C(sse3, 0) +C(pclmuldq, 1) +C(dtes64, 2) +C(monitor, 3) +C(dscpl, 4) +C(vmx, 5) +C(smx, 6) +C(eist, 7) +C(tm2, 8) +C(ssse3, 9) +C(cnxtid, 10) +C(fma, 12) +C(cx16, 13) +C(xtpr, 14) +C(pdcm, 15) +C(pcid, 17) +C(dca, 18) +C(sse41, 19) +C(sse42, 20) +C(x2apic, 21) +C(movbe, 22) +C(popcnt, 23) +C(tscdeadline, 24) +C(aes, 25) +C(xsave, 26) +C(osxsave, 27) +C(avx, 28) +C(f16c, 29) +C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) +D(fpu, 0) +D(vme, 1) +D(de, 2) +D(pse, 3) +D(tsc, 4) +D(msr, 5) +D(pae, 6) +D(mce, 7) +D(cx8, 8) +D(apic, 9) +D(sep, 11) +D(mtrr, 12) +D(pge, 13) +D(mca, 14) +D(cmov, 15) +D(pat, 16) +D(pse36, 17) +D(psn, 18) +D(clfsh, 19) +D(ds, 21) +D(acpi, 22) +D(mmx, 23) +D(fxsr, 24) +D(sse, 25) +D(sse2, 26) +D(ss, 27) +D(htt, 28) +D(tm, 29) +D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) +B(bmi1, 3) +B(hle, 4) +B(avx2, 5) +B(smep, 7) +B(bmi2, 8) +B(erms, 9) +B(invpcid, 10) +B(rtm, 11) +B(mpx, 14) +B(avx512f, 16) +B(avx512dq, 17) +B(rdseed, 18) +B(adx, 19) +B(smap, 20) +B(avx512ifma, 21) +B(pcommit, 22) +B(clflushopt, 23) +B(clwb, 24) +B(avx512pf, 26) +B(avx512er, 27) +B(avx512cd, 28) +B(sha, 29) +B(avx512bw, 30) +B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) +C(prefetchwt1, 0) +C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/src/lib/compress/zstd_1_3_8/debug.c b/src/lib/compress/zstd_1_3_8/debug.c new file mode 100644 index 0000000000000000000000000000000000000000..af8821dfe2b30c87a39d1d0c1dabe3816bc25b89 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/debug.c @@ -0,0 +1,43 @@ +/* ****************************************************************** + debug + Part of FSE library + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/src/lib/compress/zstd_1_3_8/debug.h b/src/lib/compress/zstd_1_3_8/debug.h new file mode 100644 index 0000000000000000000000000000000000000000..b99921980b383be612526d3eb1d5cfeee568471a --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/debug.h @@ -0,0 +1,134 @@ +/* ****************************************************************** + debug + Part of FSE library + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined(__cplusplus) +extern "C" { +#endif + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +#define DEBUGLEVEL 0 +#endif + +/* DEBUGFILE can be defined externally, + * typically through compiler command line. + * note : currently useless. + * Value must be stderr or stdout */ +#ifndef DEBUGFILE +#define DEBUGFILE stderr +#endif + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL >= 1) +#include +#else +#ifndef assert /* assert may be already defined, due to prior #include */ +#define assert(condition) ((void)0) /* disable assert (default) */ +#endif +#endif + +#if (DEBUGLEVEL >= 2) +#include +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +#define RAWLOG(l, ...) \ + { \ + if (l <= g_debuglevel) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } +#define DEBUGLOG(l, ...) \ + { \ + if (l <= g_debuglevel) { \ + fprintf(stderr, __FILE__ ": " __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } \ + } +#else +#define RAWLOG(l, ...) \ + {} /* disabled */ +#define DEBUGLOG(l, ...) \ + {} /* disabled */ +#endif + +#if defined(__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/src/lib/compress/zstd_1_3_8/entropy_common.c b/src/lib/compress/zstd_1_3_8/entropy_common.c new file mode 100644 index 0000000000000000000000000000000000000000..d0b14f5245a09b663e949492b8118db1996d6b65 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/entropy_common.c @@ -0,0 +1,285 @@ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* + * Dependencies + ***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) +{ + return FSE_VERSION_NUMBER; +} + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) +{ + return ERR_isError(code); +} +const char* FSE_getErrorName(size_t code) +{ + return ERR_getErrorName(code); +} + +unsigned HUF_isError(size_t code) +{ + return ERR_isError(code); +} +const char* HUF_getErrorName(size_t code) +{ + return ERR_getErrorName(code); +} + +/*-************************************************************** + * FSE NCount encoding-decoding + ****************************************************************/ +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*)headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) { + /* This function only works when hbSize >= 4 */ + char buffer[4]; + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, headerBuffer, hbSize); + { + size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, buffer, sizeof(buffer)); + if (FSE_isError(countSize)) + return countSize; + if (countSize > hbSize) + return ERROR(corruption_detected); + return countSize; + } + } + assert(hbSize >= 4); + + /* init */ + memset(normalizedCounter, + 0, + (*maxSVPtr + 1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) + return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1 << nbBits) + 1; + threshold = 1 << nbBits; + nbBits++; + + while ((remaining > 1) & (charnum <= *maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend - 5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } + } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) + return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) + normalizedCounter[charnum++] = 0; + if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount >> 3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } + } + { + int const max = (2 * threshold - 1) - remaining; + int count; + + if ((bitStream & (threshold - 1)) < (U32)max) { + count = bitStream & (threshold - 1); + bitCount += nbBits - 1; + } else { + count = bitStream & (2 * threshold - 1); + if (count >= threshold) + count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= count < 0 ? -count : count; /* -1 means +1 */ + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4)) { + ip += bitCount >> 3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } + } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) + return ERROR(corruption_detected); + if (bitCount > 32) + return ERROR(corruption_detected); + *maxSVPtr = charnum - 1; + + ip += (bitCount + 7) >> 3; + return ip - istart; +} + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*)src; + size_t iSize; + size_t oSize; + + if (!srcSize) + return ERROR(srcSize_wrong); + iSize = ip[0]; + /* memset(huffWeight, 0, hwSize); */ /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize + 1) / 2); + if (iSize + 1 > srcSize) + return ERROR(srcSize_wrong); + if (oSize >= hwSize) + return ERROR(corruption_detected); + ip += 1; + { + U32 n; + for (n = 0; n < oSize; n += 2) { + huffWeight[n] = ip[n / 2] >> 4; + huffWeight[n + 1] = ip[n / 2] & 15; + } + } + } else { /* header compressed with FSE (normal case) */ + FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be + tested) */ + if (iSize + 1 > srcSize) + return ERROR(srcSize_wrong); + oSize = FSE_decompress_wksp(huffWeight, + hwSize - 1, + ip + 1, + iSize, + fseWorkspace, + 6); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) + return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { + U32 n; + for (n = 0; n < oSize; n++) { + if (huffWeight[n] >= HUF_TABLELOG_MAX) + return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } + } + if (weightTotal == 0) + return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { + U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) + return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { + U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) + return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } + } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) + return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize + 1); + return iSize + 1; +} diff --git a/src/lib/compress/zstd_1_3_8/error_private.c b/src/lib/compress/zstd_1_3_8/error_private.c new file mode 100644 index 0000000000000000000000000000000000000000..5cbf4f615d2836b055720947ae8f6a13a181ae87 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/error_private.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch (code) { + case PREFIX(no_error): + return "No error detected"; + case PREFIX(GENERIC): + return "Error (generic)"; + case PREFIX(prefix_unknown): + return "Unknown frame descriptor"; + case PREFIX(version_unsupported): + return "Version not supported"; + case PREFIX(frameParameter_unsupported): + return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): + return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): + return "Corrupted block detected"; + case PREFIX(checksum_wrong): + return "Restored data doesn't match checksum"; + case PREFIX(parameter_unsupported): + return "Unsupported parameter"; + case PREFIX(parameter_outOfBound): + return "Parameter is out of bound"; + case PREFIX(init_missing): + return "Context should be init first"; + case PREFIX(memory_allocation): + return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): + return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): + return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): + return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): + return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): + return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): + return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): + return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): + return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): + return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): + return "Src size is incorrect"; + case PREFIX(dstBuffer_null): + return "Operation on NULL destination buffer"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): + return "Frame index is too large"; + case PREFIX(seekableIO): + return "An I/O error occurred when reading/seeking"; + case PREFIX(maxCode): + default: + return notErrorCode; + } +#endif +} diff --git a/src/lib/compress/zstd_1_3_8/error_private.h b/src/lib/compress/zstd_1_3_8/error_private.h new file mode 100644 index 0000000000000000000000000000000000000000..838dc2c81a231d48e486ee47908fc5092caa8743 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/error_private.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined(__cplusplus) +extern "C" { +#endif + +/* **************************************** + * Dependencies + ******************************************/ +#include /* size_t */ +#include "zstd_errors.h" /* enum list */ + +/* **************************************** + * Compiler-specific + ******************************************/ +#if defined(__GNUC__) +#define ERR_STATIC static __attribute__((unused)) +#elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#define ERR_STATIC static inline +#elif defined(_MSC_VER) +#define ERR_STATIC static __inline +#else +#define ERR_STATIC \ + static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/*-**************************************** + * Customization (error_public.h) + ******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + +/*-**************************************** + * Error codes handling + ******************************************/ +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) +{ + return (code > ERROR(maxCode)); +} + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) +{ + if (!ERR_isError(code)) + return (ERR_enum)0; + return (ERR_enum)(0 - code); +} + +/*-**************************************** + * Error Strings + ******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined(__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/src/lib/compress/zstd_1_3_8/fse.h b/src/lib/compress/zstd_1_3_8/fse.h new file mode 100644 index 0000000000000000000000000000000000000000..de1e47e953476a734a7a49a20e0b545baa6f4fa1 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/fse.h @@ -0,0 +1,702 @@ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +#if defined(__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + +/*-***************************************** + * Dependencies + ******************************************/ +#include /* size_t, ptrdiff_t */ + +/*-***************************************** + * FSE_PUBLIC_API : control library symbols visibility + ******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT == 1) && defined(__GNUC__) && (__GNUC__ >= 4) +#define FSE_PUBLIC_API __attribute__((visibility("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT == 1) /* Visual expected */ +#define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT == 1) +#define FSE_PUBLIC_API \ + __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from \ + the IAT and an indirect jump.*/ +#else +#define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + +/*-**************************************** + * FSE simple functions + ******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize); + +/*-***************************************** + * Tool functions + ******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + +/*-***************************************** + * FSE advanced functions + ******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2( + void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +/*-***************************************** + * FSE detailed API + ******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount( + short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount( + void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable(unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API void FSE_freeCTable(FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable( + FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable( + void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have +'maxSymbolValuePtr[0]+1' cells. 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= +maxSymbolValuePtr[0] maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means +"default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using +FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size +too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable( + FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable( + void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer +too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + +/* ***************************************** + * Static allocation + *******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size >> 7)) +#define FSE_COMPRESSBOUND(size) \ + (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) \ + (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) \ + (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024)) +size_t FSE_compress_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw(FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle(FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `(1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + +/* ***************************************** + * FSE unsafe API + *******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + +/* ***************************************** + * Implementation of inlined functions + *******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*)ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1 << tableLog; + statePtr->stateTable = u16ptr + 2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1); + statePtr->stateLog = tableLog; +} + +/*! FSE_initCState2() : + * Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) + * uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { + const FSE_symbolCompressionTransform symbolTT = + ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*)symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1 << 16) - 1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*)symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits + 1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31 - tableLog); /* ensure enough room for renormalization double shift */ + { + U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = + (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits + 1) * bitMultiplier - normalizedDeltaFromThreshold; + } +} + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct { + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** + * Tuning parameters + ****************************************************************/ +/*!MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +#define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +#define FSE_DEFAULT_MEMORY_USAGE 13 +#endif + +/*!FSE_MAX_SYMBOL_VALUE : + * Maximum symbol value authorized. + * Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +#define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** + * template functions type & suffix + ****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + +#endif /* !FSE_COMMONDEFS_ONLY */ + +/* *************************************************************** + * Constants + *****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2) +#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG) +#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1) +#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2) +#define FSE_MIN_TABLELOG 5 + +#define FSE_TABLELOG_ABSOLUTE_MAX 15 +#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX +#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) + +#endif /* FSE_STATIC_LINKING_ONLY */ + +#if defined(__cplusplus) +} +#endif diff --git a/src/lib/compress/zstd_1_3_8/fse_compress.c b/src/lib/compress/zstd_1_3_8/fse_compress.c new file mode 100644 index 0000000000000000000000000000000000000000..bc9adb302346ef215c8413dbab084c9ec639dccc --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/fse_compress.c @@ -0,0 +1,791 @@ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** + * Includes + ****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include "compiler.h" +#include "mem.h" /* U32, U16, etc. */ +#include "debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" + +/* ************************************************************** + * Error Management + ****************************************************************/ +#define FSE_isError ERR_isError + +/* ************************************************************** + * Templates + ****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +#error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +#error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X, Y) X##Y +#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) +#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<> 1 : 1); + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*)(FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE + 2]; + + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; + U32 highThreshold = tableSize - 1; + + /* CTable header */ + if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) + return ERROR(tableLog_tooLarge); + tableU16[-2] = (U16)tableLog; + tableU16[-1] = (U16)maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + +#ifdef __clang_analyzer__ + memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ +#endif + + /* symbol start positions */ + { + U32 u; + cumul[0] = 0; + for (u = 1; u <= maxSymbolValue + 1; u++) { + if (normalizedCounter[u - 1] == -1) { /* Low proba symbol */ + cumul[u] = cumul[u - 1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u - 1); + } else { + cumul[u] = cumul[u - 1] + normalizedCounter[u - 1]; + } + } + cumul[maxSymbolValue + 1] = tableSize + 1; + } + + /* Spread symbols */ + { + U32 position = 0; + U32 symbol; + for (symbol = 0; symbol <= maxSymbolValue; symbol++) { + int nbOccurences; + int const freq = normalizedCounter[symbol]; + for (nbOccurences = 0; nbOccurences < freq; nbOccurences++) { + tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; + position = (position + step) & tableMask; + while (position > highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } + } + + assert(position == 0); /* Must have initialized all positions */ + } + + /* Build table */ + { + U32 u; + for (u = 0; u < tableSize; u++) { + FSE_FUNCTION_TYPE s = + tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ + tableU16[cumul[s]++] = (U16)(tableSize + u); /* TableU16 : sorted by symbol order; gives next state value */ + } + } + + /* Build Symbol Transformation Table */ + { + unsigned total = 0; + unsigned s; + for (s = 0; s <= maxSymbolValue; s++) { + switch (normalizedCounter[s]) { + case 0: + /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */ + symbolTT[s].deltaNbBits = ((tableLog + 1) << 16) - (1 << tableLog); + break; + + case -1: + case 1: + symbolTT[s].deltaNbBits = (tableLog << 16) - (1 << tableLog); + symbolTT[s].deltaFindState = total - 1; + total++; + break; + default: { + U32 const maxBitsOut = tableLog - BIT_highbit32(normalizedCounter[s] - 1); + U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = total - normalizedCounter[s]; + total += normalizedCounter[s]; + } + } + } + } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } + } +#endif + + return 0; +} + +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about + it */ + return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol)); +} + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** + * FSE NCount encoding + ****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t FSE_writeNCount_generic(void* header, size_t headerBufferSize, const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog, unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*)header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog - FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize + 1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog + 1; + + while ((symbol < alphabetSize) && (remaining > 1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) + symbol++; + if (symbol == alphabetSize) + break; /* incorrect distribution */ + while (symbol >= start + 24) { + start += 24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream >> 8); + out += 2; + bitStream >>= 16; + } + while (symbol >= start + 3) { + start += 3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (symbol - start) << bitCount; + bitCount += 2; + if (bitCount > 16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream >> 8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } + } + { + int count = normalizedCounter[symbol++]; + int const max = (2 * threshold - 1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count >= threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count < max); + previousIs0 = (count == 1); + if (remaining < 1) + return ERROR(GENERIC); + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + } + if (bitCount > 16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream >> 8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } + } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream >> 8); + out += (bitCount + 7) / 8; + + return (out - ostart); +} + +size_t FSE_writeNCount( + void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) + return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) + return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic( + buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + +/*-************************************************************** + * FSE Compression Code + ****************************************************************/ + +FSE_CTable* FSE_createCTable(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) + tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable(FSE_CTable* ct) +{ + free(ct); +} + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog == 0) + tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) + tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) + tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) + tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) + tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s = 0; s <= maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s] = 0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s] = NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s = 0; s <= maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue + 1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s = 0; s <= maxSymbolValue; s++) + if (count[s] > maxC) { + maxV = s; + maxC = count[s]; + } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1)) + if (norm[s] > 0) { + ToDistribute--; + norm[s]++; + } + return 0; + } + + { + U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog - 1)) - 1; + U64 const rStep = ((((U64)1 << vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ + U64 tmpTotal = mid; + for (s = 0; s <= maxSymbolValue; s++) { + if (norm[s] == NOT_YET_ASSIGNED) { + U64 const end = tmpTotal + (count[s] * rStep); + U32 const sStart = (U32)(tmpTotal >> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } + } + } + + return 0; +} + +size_t FSE_normalizeCount( + short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t total, unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog == 0) + tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) + return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) + return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) + return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { + static U32 const rtbTable[] = {0, 473195, 504333, 520860, 550000, 700000, 750000, 830000}; + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1 << 62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL << (scale - 20); + int stillToDistribute = 1 << tableLog; + unsigned s; + unsigned largest = 0; + short largestP = 0; + U32 lowThreshold = (U32)(total >> tableLog); + + for (s = 0; s <= maxSymbolValue; s++) { + if (count[s] == total) + return 0; /* rle special case */ + if (count[s] == 0) { + normalizedCounter[s] = 0; + continue; + } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s] * step) >> scale); + if (proba < 8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s] * step) - ((U64)proba << scale) > restToBeat; + } + if (proba > largestP) { + largestP = proba; + largest = s; + } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } + } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) + return errorCode; + } else + normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<> 1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*)(FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) + return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16)nbBits; + tableU16[-1] = (U16)maxSymbolValue; + + /* Build table */ + for (s = 0; s < tableSize; s++) + tableU16[s] = (U16)(tableSize + s); + + /* Build Symbol Transformation Table */ + { + const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); + for (s = 0; s <= maxSymbolValue; s++) { + symbolTT[s].deltaNbBits = deltaNbBits; + symbolTT[s].deltaFindState = s - 1; + } + } + + return 0; +} + +/* fake FSE_CTable, for rle input (always same symbol) */ +size_t FSE_buildCTable_rle(FSE_CTable* ct, BYTE symbolValue) +{ + void* ptr = ct; + U16* tableU16 = ((U16*)ptr) + 2; + void* FSCTptr = (U32*)ptr + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)FSCTptr; + + /* header */ + tableU16[-2] = (U16)0; + tableU16[-1] = (U16)symbolValue; + + /* Build table */ + tableU16[0] = 0; + tableU16[1] = 0; /* just in case */ + + /* Build Symbol Transformation Table */ + symbolTT[symbolValue].deltaNbBits = 0; + symbolTT[symbolValue].deltaFindState = 0; + + return 0; +} + +static size_t FSE_compress_usingCTable_generic( + void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct, const unsigned fast) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = iend; + + BIT_CStream_t bitC; + FSE_CState_t CState1, CState2; + + /* init */ + if (srcSize <= 2) + return 0; + { + size_t const initError = BIT_initCStream(&bitC, dst, dstSize); + if (FSE_isError(initError)) + return 0; /* not enough space available to write a bitstream */ + } + +#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + + if (srcSize & 1) { + FSE_initCState2(&CState1, ct, *--ip); + FSE_initCState2(&CState2, ct, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } else { + FSE_initCState2(&CState2, ct, *--ip); + FSE_initCState2(&CState1, ct, *--ip); + } + + /* join to mod 4 */ + srcSize -= 2; + if ((sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while (ip > istart) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer) * 8 < FSE_MAX_TABLELOG * 2 + 7) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer) * 8 > FSE_MAX_TABLELOG * 4 + 7) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + +size_t FSE_compressBound(size_t size) +{ + return FSE_COMPRESSBOUND(size); +} + +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + if (ERR_isError(e)) \ + return e +#define CHECK_F(f) \ + { \ + CHECK_V_F(_var_err__, f); \ + } + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` size must be `(1< not compressible */ + if (maxCount < (srcSize >> 7)) + return 0; /* Heuristic : not compressible enough */ + } + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + CHECK_F(FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue)); + + /* Write table description header */ + { + CHECK_V_F(nc_err, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); + op += nc_err; + } + + /* Compress */ + CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize)); + { + CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable)); + if (cSize == 0) + return 0; /* not enough space for compressed data */ + op += cSize; + } + + /* check compressibility */ + if ((size_t)(op - ostart) >= srcSize - 1) + return 0; + + return op - ostart; +} + +typedef struct { + FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; +} fseWkspMax_t; + +size_t FSE_compress2( + void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + fseWkspMax_t scratchBuffer; + DEBUG_STATIC_ASSERT( + sizeof(scratchBuffer) >= + FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, + FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) + return ERROR(tableLog_tooLarge); + return FSE_compress_wksp( + dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); +} + +size_t FSE_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/lib/compress/zstd_1_3_8/fse_decompress.c b/src/lib/compress/zstd_1_3_8/fse_decompress.c new file mode 100644 index 0000000000000000000000000000000000000000..2d75d2f9c72a2389694a79a6b0ec1520ec0a4a15 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/fse_decompress.c @@ -0,0 +1,329 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** + * Includes + ****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" + +/* ************************************************************** + * Error Management + ****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + +/* check and forward error code */ +#define CHECK_F(f) \ + { \ + size_t const e = f; \ + if (FSE_isError(e)) \ + return e; \ + } + +/* ************************************************************** + * Templates + ****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +#error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +#error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X, Y) X##Y +#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y) +#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y) + +/* Function templates */ +FSE_DTable* FSE_createDTable(unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) + tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc(FSE_DTABLE_SIZE_U32(tableLog) * sizeof(U32)); +} + +void FSE_freeDTable(FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE + 1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize - 1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) + return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) + return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { + FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { + S16 const largeLimit = (S16)(1 << (tableLog - 1)); + U32 s; + for (s = 0; s < maxSV1; s++) { + if (normalizedCounter[s] == -1) { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) + DTableH.fastMode = 0; + symbolNext[s] = normalizedCounter[s]; + } + } + } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { + U32 const tableMask = tableSize - 1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s = 0; s < maxSV1; s++) { + int i; + for (i = 0; i < normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) + position = (position + step) & tableMask; /* lowprob area */ + } + } + if (position != 0) + return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u = 0; u < tableSize; u++) { + FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); + U32 const nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32(nextState)); + tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); + } + } + + return 0; +} + +#ifndef FSE_COMMONDEFS_ONLY + +/*-******************************************************* + * Decompression (Byte symbols) + *********************************************************/ +size_t FSE_buildDTable_rle(FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + +size_t FSE_buildDTable_raw(FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask + 1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) + return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s = 0; s < maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax - 3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + + /* Init */ + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) { + op[0] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ + { + if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { + op += 2; + break; + } + } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op > (omax - 2)) + return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op > (omax - 2)) + return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } + } + + return op - ostart; +} + +size_t FSE_decompress_usingDTable( + void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + +size_t FSE_decompress_wksp( + void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE + 1]; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + /* normal FSE decoding mode */ + size_t const NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) + return NCountLength; + // if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already + // checked in NCountLength, only remaining case : NCountLength==cSrcSize */ + if (tableLog > maxLog) + return ERROR(tableLog_tooLarge); + ip += NCountLength; + cSrcSize -= NCountLength; + + CHECK_F(FSE_buildDTable(workSpace, counting, maxSymbolValue, tableLog)); + + return FSE_decompress_usingDTable( + dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ +} + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) +{ + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/lib/compress/zstd_1_3_8/hist.c b/src/lib/compress/zstd_1_3_8/hist.c new file mode 100644 index 0000000000000000000000000000000000000000..b07fab0df127a4d123bcaebe1892cdfb7e69e684 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/hist.c @@ -0,0 +1,227 @@ +/* ****************************************************************** + hist : Histogram functions + part of Finite State Entropy project + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* --- dependencies --- */ +#include "mem.h" /* U32, BYTE, etc. */ +#include "debug.h" /* assert, DEBUGLOG */ +#include "error_private.h" /* ERROR */ +#include "hist.h" + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) +{ + return ERR_isError(code); +} + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount = 0; + + memset(count, 0, (maxSymbolValue + 1) * sizeof(*count)); + if (srcSize == 0) { + *maxSymbolValuePtr = 0; + return 0; + } + + while (ip < end) { + assert(*ip <= maxSymbolValue); + count[*ip++]++; + } + + while (!count[maxSymbolValue]) + maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + + { + U32 s; + for (s = 0; s <= maxSymbolValue; s++) + if (count[s] > largestCount) + largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` size must be a table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram would be larger than *maxSymbolValuePtr). */ +static size_t HIST_count_parallel_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, + size_t sourceSize, HIST_checkInput_e check, U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip + sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max = 0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + memset(workSpace, 0, 4 * 256 * sizeof(unsigned)); + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) + maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { + U32 cached = MEM_read32(ip); + ip += 4; + while (ip < iend - 15) { + U32 c = cached; + cached = MEM_read32(ip); + ip += 4; + Counting1[(BYTE)c]++; + Counting2[(BYTE)(c >> 8)]++; + Counting3[(BYTE)(c >> 16)]++; + Counting4[c >> 24]++; + c = cached; + cached = MEM_read32(ip); + ip += 4; + Counting1[(BYTE)c]++; + Counting2[(BYTE)(c >> 8)]++; + Counting3[(BYTE)(c >> 16)]++; + Counting4[c >> 24]++; + c = cached; + cached = MEM_read32(ip); + ip += 4; + Counting1[(BYTE)c]++; + Counting2[(BYTE)(c >> 8)]++; + Counting3[(BYTE)(c >> 16)]++; + Counting4[c >> 24]++; + c = cached; + cached = MEM_read32(ip); + ip += 4; + Counting1[(BYTE)c]++; + Counting2[(BYTE)(c >> 8)]++; + Counting3[(BYTE)(c >> 16)]++; + Counting4[c >> 24]++; + } + ip -= 4; + } + + /* finish last symbols */ + while (ip < iend) + Counting1[*ip++]++; + + if (check) { /* verify stats will fit into destination table */ + U32 s; + for (s = 255; s > maxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) + return ERROR(maxSymbolValue_tooSmall); + } + } + + { + U32 s; + if (maxSymbolValue > 255) + maxSymbolValue = 255; + for (s = 0; s <= maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) + max = count[s]; + } + } + + while (!count[maxSymbolValue]) + maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) + return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) + return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) + return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) + return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} diff --git a/src/lib/compress/zstd_1_3_8/hist.h b/src/lib/compress/zstd_1_3_8/hist.h new file mode 100644 index 0000000000000000000000000000000000000000..45a47011525efb812e820ff883a0a35b4d5bc396 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/hist.h @@ -0,0 +1,88 @@ +/* ****************************************************************** + hist : Histogram functions + part of Finite State Entropy project + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* --- dependencies --- */ +#include /* size_t */ + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, void* workSpace, + size_t workSpaceSize); + +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); diff --git a/src/lib/compress/zstd_1_3_8/huf.h b/src/lib/compress/zstd_1_3_8/huf.h new file mode 100644 index 0000000000000000000000000000000000000000..87581eec99ba6c91d223a6762e60473b5a8a0bbc --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/huf.h @@ -0,0 +1,373 @@ +/* ****************************************************************** + huff0 huffman codec, + part of Finite State Entropy library + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +#if defined(__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include /* size_t */ + +/* *** library symbols visibility *** */ +/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, + * HUF symbols remain "private" (internal symbols for library only). + * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT == 1) && defined(__GNUC__) && (__GNUC__ >= 4) +#define HUF_PUBLIC_API __attribute__((visibility("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT == 1) /* Visual expected */ +#define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT == 1) +#define HUF_PUBLIC_API \ + __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an \ + indirect jump) */ +#else +#define HUF_PUBLIC_API +#endif + +/* ========================== */ +/* *** simple functions *** */ +/* ========================== */ + +/** HUF_compress() : + * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + * 'dst' buffer must be already allocated. + * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + * @return : size of compressed data (<= `dstCapacity`). + * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) + */ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/** HUF_decompress() : + * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + * into already allocated buffer 'dst', of minimum size 'dstSize'. + * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + * Note : in contrast with FSE, HUF_decompress can regenerate + * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + * because it knows size to regenerate (originalSize). + * @return : size of regenerated data (== originalSize), + * or an error code, which can be tested using HUF_isError() + */ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize); + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. + * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2( + void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ +#define HUF_WORKSPACE_SIZE (6 << 10) +#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp(void* dst, size_t dstCapacity, const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#endif /* HUF_H_298734234 */ + +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of a dynamic library, + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + * *****************************************************************/ +#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) +#define HUF_H_HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX \ + 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +#error "HUF_TABLELOG_MAX is too large !" +#endif + +/* **************************************** + * Static allocation + ******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) \ + (size + (size >> 8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) \ + (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue) + 1) /* Use tables of U32, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 << (maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = {((U32)((maxTableLog)-1) * 0x01000001)} +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = {((U32)(maxTableLog)*0x01000001)} + +/* **************************************** + * Advanced decompression functions + ******************************************/ +size_t HUF_decompress4X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +#endif + +size_t HUF_decompress4X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X1_DCtx( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_DCtx( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + +/* **************************************** + * HUF detailed API + * ****************************************/ + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, + unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite + count content */ +size_t HUF_writeCTable(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been + constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ +} HUF_repeat; +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned tableLog, void* workSpace, + size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2 * HUF_SYMBOLVALUE_MAX + 1 + 1) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp( + HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable(HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/** HUF_getNbBits() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private + * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ +U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif + +size_t HUF_decompress4X_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress4X1_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X( + void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned tableLog, void* workSpace, + size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned tableLog, void* workSpace, + size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); + +size_t HUF_decompress1X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +#endif + +size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X_DCtx_wksp( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable_bmi2( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2); +#endif +size_t HUF_decompress4X_usingDTable_bmi2( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); + +#endif /* HUF_STATIC_LINKING_ONLY */ + +#if defined(__cplusplus) +} +#endif diff --git a/src/lib/compress/zstd_1_3_8/huf_compress.c b/src/lib/compress/zstd_1_3_8/huf_compress.c new file mode 100644 index 0000000000000000000000000000000000000000..4ba2e1c5e9b624a82a6a9aa11adf9ca32b7281cb --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/huf_compress.c @@ -0,0 +1,902 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** + * Compiler specifics + ****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +/* ************************************************************** + * Includes + ****************************************************************/ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "compiler.h" +#include "bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "error_private.h" + +/* ************************************************************** + * Error Management + ****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + if (ERR_isError(e)) \ + return e +#define CHECK_F(f) \ + { \ + CHECK_V_F(_var_err__, f); \ + } + +/* ************************************************************** + * Utils + ****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + +/* ******************************************************* + * HUF : Huffman block compression + *********************************************************/ +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 +static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + BYTE scratchBuffer[1 << MAX_FSE_TABLELOG_FOR_HUFF_HEADER]; + + unsigned count[HUF_TABLELOG_MAX + 1]; + S16 norm[HUF_TABLELOG_MAX + 1]; + + /* init conditions */ + if (wtSize <= 1) + return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { + unsigned const maxCount = HIST_count_simple(count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) + return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) + return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F(FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue)); + + /* Write table description header */ + { + CHECK_V_F(hSize, FSE_writeNCount(op, oend - op, norm, maxSymbolValue, tableLog)); + op += hSize; + } + + /* Compress */ + CHECK_F(FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer))); + { + CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable)); + if (cSize == 0) + return 0; /* not enough space for compressed data */ + op += cSize; + } + + return op - ostart; +} + +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within "huf.h" */ + +/*! HUF_writeCTable() : + `CTable` : Huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) +{ + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; + BYTE* op = (BYTE*)dst; + U32 n; + + /* check conditions */ + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n = 1; n < huffLog + 1; n++) + bitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n = 0; n < maxSymbolValue; n++) + huffWeight[n] = bitsToWeight[CTable[n].nbBits]; + + /* attempt weights compression by FSE */ + { + CHECK_V_F(hSize, HUF_compressWeights(op + 1, maxDstSize - 1, huffWeight, maxSymbolValue)); + if ((hSize > 1) & (hSize < maxSymbolValue / 2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize + 1; + } + } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256 - 128)) + return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue + 1) / 2) + 1 > maxDstSize) + return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue - 1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n = 0; n < maxSymbolValue; n += 2) + op[(n / 2) + 1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n + 1]); + return ((maxSymbolValue + 1) / 2) + 1; +} + +size_t HUF_readCTable(HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) + return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr + 1) + return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { + U32 n, nextRankStart = 0; + for (n = 1; n <= tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n - 1)); + rankVal[n] = current; + } + } + + /* fill nbBits */ + { + U32 n; + for (n = 0; n < nbSymbols; n++) { + const U32 w = huffWeight[n]; + CTable[n].nbBits = (BYTE)(tableLog + 1 - w); + } + } + + /* fill val */ + { + U16 nbPerRank[HUF_TABLELOG_MAX + 2] = {0}; /* support w=0=>n=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX + 2] = {0}; + { + U32 n; + for (n = 0; n < nbSymbols; n++) + nbPerRank[CTable[n].nbBits]++; + } + /* determine stating value per rank */ + valPerRank[tableLog + 1] = 0; /* for w==0 */ + { + U16 min = 0; + U32 n; + for (n = tableLog; n > 0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } + } + /* assign value within rank, symbol order */ + { + U32 n; + for (n = 0; n < nbSymbols; n++) + CTable[n].val = valPerRank[CTable[n].nbBits]++; + } + } + + *maxSymbolValuePtr = nbSymbols - 1; + return readSize; +} + +U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue) +{ + const HUF_CElt* table = (const HUF_CElt*)symbolTable; + assert(symbolValue <= HUF_SYMBOLVALUE_MAX); + return table[symbolValue].nbBits; +} + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) + return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { + int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n--; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) + n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { + U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX + 2]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { + U32 currentNbBits = maxNbBits; + for (pos = n; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) + continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits - currentNbBits] = pos; + } + } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for (; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease - 1]; + if (highPos == noSymbol) + continue; + if (lowPos == noSymbol) + break; + { + U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) + break; + } + } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one + * !) */ + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease <= HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + totalCost -= 1 << (nBitsToDecrease - 1); + if (rankLast[nBitsToDecrease - 1] == noSymbol) + rankLast[nBitsToDecrease - 1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits - nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from + largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) + n--; + huffNode[n + 1].nbBits--; + rankLast[1] = n + 1; + totalCost++; + continue; + } + huffNode[rankLast[1] + 1].nbBits--; + rankLast[1]++; + totalCost++; + } + } + } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n = 0; n <= maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base++; + } + for (n = 30; n > 0; n--) + rank[n - 1].base += rank[n].base; + for (n = 0; n < 32; n++) + rank[n].current = rank[n].base; + for (n = 0; n <= maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c + 1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos - 1].count)) { + huffNode[pos] = huffNode[pos - 1]; + pos--; + } + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of + * HUF_CTABLE_WORKSPACE_SIZE_U32 unsigned. + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX + 1) +typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; +size_t HUF_buildCTable_wksp( + HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +{ + nodeElt* const huffNode0 = (nodeElt*)workSpace; + nodeElt* const huffNode = huffNode0 + 1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (((size_t)workSpace & 3) != 0) + return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (wkspSize < sizeof(huffNodeTable)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) + maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while (huffNode[nonNullRank].count == 0) + nonNullRank--; + lowS = nonNullRank; + nodeRoot = nodeNb + lowS - 1; + lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS - 1].count; + huffNode[lowS].parent = huffNode[lowS - 1].parent = nodeNb; + nodeNb++; + lowS -= 2; + for (n = nodeNb; n <= nodeRoot; n++) + huffNode[n].count = (U32)(1U << 30); + huffNode0[0].count = (U32)(1U << 31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n = nodeRoot - 1; n >= STARTNODE; n--) + huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; + for (n = 0; n <= nonNullRank; n++) + huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { + U16 nbPerRank[HUF_TABLELOG_MAX + 1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX + 1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) + return ERROR(GENERIC); /* check fit into table */ + for (n = 0; n <= nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { + U16 min = 0; + for (n = maxNbBits; n > 0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } + } + for (n = 0; n <= maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n = 0; n <= maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +/** HUF_buildCTable() : + * @return : maxNbBits + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable(HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) +{ + huffNodeTable nodeTable; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); +} + +static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += CTable[s].nbBits * count[s]; + } + return nbBits >> 3; +} + +static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (CTable[s].nbBits == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) +{ + return HUF_COMPRESSBOUND(size); +} + +FORCE_INLINE_TEMPLATE void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +#define HUF_FLUSHBITS(s) BIT_flushBits(s) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 2 + 7) \ + HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer) * 8 < HUF_TABLELOG_MAX * 4 + 7) \ + HUF_FLUSHBITS(stream) + +FORCE_INLINE_TEMPLATE size_t HUF_compress1X_usingCTable_internal_body( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) + return 0; /* not enough space to compress */ + { + size_t const initErr = BIT_initCStream(&bitC, op, oend - op); + if (HUF_isError(initErr)) + return 0; + } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) { + case 3: + HUF_encodeSymbol(&bitC, ip[n + 2], CTable); + HUF_FLUSHBITS_2(&bitC); + /* fall-through */ + case 2: + HUF_encodeSymbol(&bitC, ip[n + 1], CTable); + HUF_FLUSHBITS_1(&bitC); + /* fall-through */ + case 1: + HUF_encodeSymbol(&bitC, ip[n + 0], CTable); + HUF_FLUSHBITS(&bitC); + /* fall-through */ + case 0: /* fall-through */ + default: + break; + } + + for (; n > 0; n -= 4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n - 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n - 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n - 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n - 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t HUF_compress1X_usingCTable_internal_bmi2( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t HUF_compress1X_usingCTable_internal_default( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t HUF_compress1X_usingCTable_internal( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, const int bmi2) +{ + if (bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t HUF_compress1X_usingCTable_internal( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, const int bmi2) +{ + (void)bmi2; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + +static size_t HUF_compress4X_usingCTable_internal( + void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2) +{ + size_t const segmentSize = (srcSize + 3) / 4; /* first 3 segments */ + const BYTE* ip = (const BYTE*)src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) + return 0; /* minimum space to compress successfully */ + if (srcSize < 12) + return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + { + CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend - op, ip, segmentSize, CTable, bmi2)); + if (cSize == 0) + return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { + CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend - op, ip, segmentSize, CTable, bmi2)); + if (cSize == 0) + return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart + 2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { + CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend - op, ip, segmentSize, CTable, bmi2)); + if (cSize == 0) + return 0; + assert(cSize <= 65535); + MEM_writeLE16(ostart + 4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { + CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend - op, ip, iend - ip, CTable, bmi2)); + if (cSize == 0) + return 0; + op += cSize; + } + + return op - ostart; +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal(BYTE* const ostart, BYTE* op, BYTE* const oend, const void* src, + size_t srcSize, HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) +{ + size_t const cSize = (nbStreams == HUF_singleStream) + ? HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) + : HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2); + if (HUF_isError(cSize)) { + return cSize; + } + if (cSize == 0) { + return 0; + } /* uncompressible */ + op += cSize; + /* check compressibility */ + if ((size_t)(op - ostart) >= srcSize - 1) { + return 0; + } + return op - ostart; +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1]; + huffNodeTable nodeTable; +} HUF_compress_tables_t; + +/* HUF_compress_internal() : + * `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ +static size_t HUF_compress_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned huffLog, HUF_nbStreams_e nbStreams, void* workSpace, size_t wkspSize, HUF_CElt* oldHufTable, + HUF_repeat* repeat, int preferRepeat, const int bmi2) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + /* checks & inits */ + if (((size_t)workSpace & 3) != 0) + return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (wkspSize < HUF_WORKSPACE_SIZE) + return ERROR(workSpace_tooSmall); + if (!srcSize) + return 0; /* Uncompressed */ + if (!dstSize) + return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) + return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) + return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) + maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) + huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, nbStreams, oldHufTable, bmi2); + } + + /* Scan input and build symbol stats */ + { + CHECK_V_F(largest, HIST_count_wksp(table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize)); + if (largest == srcSize) { + *ostart = ((const BYTE*)src)[0]; + return 1; + } /* single symbol, rle */ + if (largest <= (srcSize >> 7) + 4) + return 0; /* heuristic : probably not compressible enough */ + } + + /* Check validity of previous table */ + if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, nbStreams, oldHufTable, bmi2); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { + size_t const maxBits = HUF_buildCTable_wksp( + table->CTable, table->count, maxSymbolValue, huffLog, table->nodeTable, sizeof(table->nodeTable)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + /* Zero unused symbols in CTable, so we can check it for validity */ + memset(table->CTable + (maxSymbolValue + 1), 0, sizeof(table->CTable) - ((maxSymbolValue + 1) * sizeof(HUF_CElt))); + } + + /* Write table description header */ + { + CHECK_V_F(hSize, HUF_writeCTable(op, dstSize, table->CTable, maxSymbolValue, huffLog)); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, nbStreams, oldHufTable, bmi2); + } + } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { + return 0; + } + op += hSize; + if (repeat) { + *repeat = HUF_repeat_none; + } + if (oldHufTable) + memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, nbStreams, table->CTable, bmi2); +} + +size_t HUF_compress1X_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned huffLog, void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, + dstSize, + src, + srcSize, + maxSymbolValue, + huffLog, + HUF_singleStream, + workSpace, + wkspSize, + NULL, + NULL, + 0, + 0 /*bmi2*/); +} + +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned huffLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, + int bmi2) +{ + return HUF_compress_internal(dst, + dstSize, + src, + srcSize, + maxSymbolValue, + huffLog, + HUF_singleStream, + workSpace, + wkspSize, + hufTable, + repeat, + preferRepeat, + bmi2); +} + +size_t HUF_compress1X( + void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * provide workspace to generate compression tables */ +size_t HUF_compress4X_wksp(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned huffLog, void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, + dstSize, + src, + srcSize, + maxSymbolValue, + huffLog, + HUF_fourStreams, + workSpace, + wkspSize, + NULL, + NULL, + 0, + 0 /*bmi2*/); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * re-use an existing huffman compression table */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, + unsigned huffLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, + int bmi2) +{ + return HUF_compress_internal(dst, + dstSize, + src, + srcSize, + maxSymbolValue, + huffLog, + HUF_fourStreams, + workSpace, + wkspSize, + hufTable, + repeat, + preferRepeat, + bmi2); +} + +size_t HUF_compress2( + void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[HUF_WORKSPACE_SIZE_U32]; + return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); +} diff --git a/src/lib/compress/zstd_1_3_8/huf_decompress.c b/src/lib/compress/zstd_1_3_8/huf_decompress.c new file mode 100644 index 0000000000000000000000000000000000000000..d8354a41cbe22124c913b025de9ffff8835c76ca --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/huf_decompress.c @@ -0,0 +1,1280 @@ +/* ****************************************************************** + huff0 huffman decoder, + part of Finite State Entropy library + Copyright (C) 2013-present, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +/* ************************************************************** + * Dependencies + ****************************************************************/ +#include /* memcpy, memset */ +#include "compiler.h" +#include "bitstream.h" /* BIT_* */ +#include "fse.h" /* to compress headers */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "error_private.h" + +/* ************************************************************** + * Macros + ****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* ************************************************************** + * Error Management + ****************************************************************/ +#define HUF_isError ERR_isError +#define CHECK_F(f) \ + { \ + size_t const err_ = (f); \ + if (HUF_isError(err_)) \ + return err_; \ + } + +/* ************************************************************** + * Byte alignment for workSpace management + ****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a)-1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + +/* ************************************************************** + * BMI2 Variant Wrappers + ****************************************************************/ +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static TARGET_ATTRIBUTE("bmi2") \ + size_t fn##_bmi2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + if (bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + (void)bmi2; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { + BYTE maxTableLog; + BYTE tableType; + BYTE tableLog; + BYTE reserved; +} DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { + BYTE byte; + BYTE nbBits; +} HUF_DEltX1; /* single-symbol decoding */ + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + + U32* rankVal; + BYTE* huffWeight; + size_t spaceUsed32 = 0; + + rankVal = (U32*)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; + huffWeight = (BYTE*)((U32*)workSpace + spaceUsed32); + spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; + + if ((spaceUsed32 << 2) > wkspSize) + return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) + return iSize; + + /* Table header */ + { + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog + 1)) + return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Calculate starting value for each rank */ + { + U32 n, nextRankStart = 0; + for (n = 1; n < tableLog + 1; n++) { + U32 const current = nextRankStart; + nextRankStart += (rankVal[n] << (n - 1)); + rankVal[n] = current; + } + } + + /* fill DTable */ + { + U32 n; + for (n = 0; n < nbSymbols; n++) { + U32 const w = huffWeight[n]; + U32 const length = (1 << w) >> 1; + U32 u; + HUF_DEltX1 D; + D.byte = (BYTE)n; + D.nbBits = (BYTE)(tableLog + 1 - w); + for (u = rankVal[w]; u < rankVal[w] + length; u++) + dt[u] = D; + rankVal[w] += length; + } + } + + return iSize; +} + +size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX1_wksp(DTable, src, srcSize, workSpace, sizeof(workSpace)); +} + +FORCE_INLINE_TEMPLATE BYTE HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX <= 12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t HUF_decodeStreamX1( + BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - 3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return pEnd - pStart; +} + +FORCE_INLINE_TEMPLATE size_t HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) + return ERROR(corruption_detected); + + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) + return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { + const BYTE* const istart = (const BYTE*)cSrc; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart + 2); + size_t const length3 = MEM_readLE16(istart + 4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize + 3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = BIT_DStream_unfinished; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) + return ERROR(corruption_detected); /* overflow */ + CHECK_F(BIT_initDStream(&bitD1, istart1, length1)); + CHECK_F(BIT_initDStream(&bitD2, istart2, length2)); + CHECK_F(BIT_initDStream(&bitD3, istart3, length3)); + CHECK_F(BIT_initDStream(&bitD4, istart4, length4)); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + endSignal = + BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + while ((endSignal == BIT_DStream_unfinished) && (op4 < (oend - 3))) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + BIT_reloadDStream(&bitD1); + BIT_reloadDStream(&bitD2); + BIT_reloadDStream(&bitD3); + BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) + return ERROR(corruption_detected); + if (op2 > opStart3) + return ERROR(corruption_detected); + if (op3 > opStart4) + return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { + U32 const endCheck = + BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) + return ERROR(corruption_detected); + } + + /* decoded size */ + return dstSize; + } +} + +typedef size_t (*HUF_decompress_usingDTable_t)( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) +HUF_DGEN(HUF_decompress4X1_usingDTable_internal) + +size_t HUF_decompress1X1_usingDTable( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) + return ERROR(GENERIC); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X1_DCtx_wksp( + HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*)cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) + return hSize; + if (hSize >= cSrcSize) + return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + +size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +size_t HUF_decompress4X1_usingDTable( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) + return ERROR(GENERIC); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*)cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) + return hSize; + if (hSize >= cSrcSize) + return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X1_DCtx_wksp( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); +} + +size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} +size_t HUF_decompress4X1(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { + U16 sequence; + BYTE nbBits; + BYTE length; +} HUF_DEltX2; /* double-symbols decoding */ +typedef struct { + BYTE symbol; + BYTE weight; +} sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed, const U32* rankValOrigin, + const int minWeight, const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX2 DElt; + U32 rankVal[HUF_TABLELOG_MAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight > 1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { + U32 s; + for (s = 0; s < sortedListSize; s++) { /* note : sortedSymbols already skipped */ + const U32 symbol = sortedSymbols[s].symbol; + const U32 weight = sortedSymbols[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 length = 1 << (sizeLog - nbBits); + const U32 start = rankVal[weight]; + U32 i = start; + const U32 end = start + length; + + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { + DTable[i++] = DElt; + } while (i < end); /* since length >= 1 */ + + rankVal[weight] += length; + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, const sortedSymbol_t* sortedList, + const U32 sortedListSize, const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_TABLELOG_MAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s = 0; s < sortedListSize; s++) { + const U16 symbol = sortedList[s].symbol; + const U32 weight = sortedList[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 start = rankVal[weight]; + const U32 length = 1 << (targetLog - nbBits); + + if (targetLog - nbBits >= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) + minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX2Level2(DTable + start, + targetLog - nbBits, + nbBits, + rankValOrigin[nbBits], + minWeight, + sortedList + sortedRank, + sortedListSize - sortedRank, + nbBitsBaseline, + symbol); + } else { + HUF_DEltX2 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { + U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) + DTable[u] = DElt; + } + } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable + 1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32* rankStart; + + rankValCol_t* rankVal; + U32* rankStats; + U32* rankStart0; + sortedSymbol_t* sortedSymbol; + BYTE* weightList; + size_t spaceUsed32 = 0; + + rankVal = (rankValCol_t*)((U32*)workSpace + spaceUsed32); + spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; + rankStats = (U32*)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_MAX + 1; + rankStart0 = (U32*)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_MAX + 2; + sortedSymbol = (sortedSymbol_t*)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); + spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; + weightList = (BYTE*)((U32*)workSpace + spaceUsed32); + spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; + + if ((spaceUsed32 << 2) > wkspSize) + return ERROR(tableLog_tooLarge); + + rankStart = rankStart0 + 1; + memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) + return ERROR(tableLog_tooLarge); + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) + return iSize; + + /* check result */ + if (tableLog > maxTableLog) + return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW] == 0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { + U32 w, nextRankStart = 0; + for (w = 1; w < maxW + 1; w++) { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { + U32 s; + for (s = 0; s < nbSymbols; s++) { + U32 const w = weightList[s]; + U32 const r = rankStart[w]++; + sortedSymbol[r].symbol = (BYTE)s; + sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { + U32* const rankVal0 = rankVal[0]; + { + int const rescale = (maxTableLog - tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w = 1; w < maxW + 1; w++) { + U32 current = nextRankVal; + nextRankVal += rankStats[w] << (w + rescale); + rankVal0[w] = current; + } + } + { + U32 const minBits = tableLog + 1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = rankVal[consumed]; + U32 w; + for (w = 1; w < maxW + 1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } + } + } + } + + HUF_fillDTableX2(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog + 1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + +size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX2_wksp(DTable, src, srcSize, workSpace, sizeof(workSpace)); +} + +FORCE_INLINE_TEMPLATE U32 HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt + val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 HUF_decodeLastSymbolX2( + void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt + val, 1); + if (dt[val].length == 1) + BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer) * 8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer) * 8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol + */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer) * 8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX <= 12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t HUF_decodeStreamX2( + BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd - (sizeof(bitDPtr->bitContainer) - 1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd - 2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd - 2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p - pStart; +} + +FORCE_INLINE_TEMPLATE size_t HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); + + /* decode */ + { + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) + return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +FORCE_INLINE_TEMPLATE size_t HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + if (cSrcSize < 10) + return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { + const BYTE* const istart = (const BYTE*)cSrc; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart + 2); + size_t const length3 = MEM_readLE16(istart + 4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize + 3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) + return ERROR(corruption_detected); /* overflow */ + CHECK_F(BIT_initDStream(&bitD1, istart1, length1)); + CHECK_F(BIT_initDStream(&bitD2, istart2, length2)); + CHECK_F(BIT_initDStream(&bitD3, istart3, length3)); + CHECK_F(BIT_initDStream(&bitD4, istart4, length4)); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = + BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for (; (endSignal == BIT_DStream_unfinished) & (op4 < (oend - (sizeof(bitD4.bitContainer) - 1)));) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + + endSignal = + BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) + return ERROR(corruption_detected); + if (op2 > opStart3) + return ERROR(corruption_detected); + if (op3 > opStart4) + return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { + U32 const endCheck = + BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) + return ERROR(corruption_detected); + } + + /* decoded size */ + return dstSize; + } +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) +HUF_DGEN(HUF_decompress4X2_usingDTable_internal) + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) + return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +size_t HUF_decompress1X2_DCtx_wksp( + HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*)cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) + return hSize; + if (hSize >= cSrcSize) + return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); +} + +size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) + return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +} + +static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*)cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) + return hSize; + if (hSize >= cSrcSize) + return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} + +size_t HUF_decompress4X2_DCtx_wksp( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); +} + +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress4X2(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + +size_t HUF_decompress1X_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) + : HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + +size_t HUF_decompress4X_usingDTable( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) + : HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif +} + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { + U32 tableTime; + U32 decode256Time; +} algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = { + /* single, double, quad */ + {{0, 0}, {1, 1}, {2, 2}}, /* Q==0 : impossible */ + {{0, 0}, {1, 1}, {2, 2}}, /* Q==1 : impossible */ + {{38, 130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{448, 128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{556, 128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{714, 128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{883, 128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{897, 128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{926, 128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{947, 128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107, 128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177, 128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242, 128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349, 128}, {2644, 106}, {5260, 106}}, /* Q ==13 : 81-87% */ + {{1455, 128}, {2422, 124}, {4174, 124}}, /* Q ==14 : 87-93% */ + {{722, 128}, {1891, 145}, {1936, 146}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder(size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128 * 1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { + U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) + static const decompressionAlgo decompress[2] = {HUF_decompress4X1, HUF_decompress4X2}; +#endif + + /* validation checks */ + if (dstSize == 0) + return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) + return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { + memcpy(dst, cSrc, dstSize); + return dstSize; + } /* not compressed */ + if (cSrcSize == 1) { + memset(dst, *(const BYTE*)cSrc, dstSize); + return dstSize; + } /* RLE */ + + { + U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); +#else + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); +#endif + } +} + +size_t HUF_decompress4X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) + return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) + return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { + memcpy(dst, cSrc, dstSize); + return dstSize; + } /* not compressed */ + if (cSrcSize == 1) { + memset(dst, *(const BYTE*)cSrc, dstSize); + return dstSize; + } /* RLE */ + + { + U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#else + return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) + : HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#endif + } +} + +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress4X_hufOnly_wksp( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) + return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) + return ERROR(corruption_detected); + + { + U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize) + : HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif + } +} + +size_t HUF_decompress1X_DCtx_wksp( + HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) + return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) + return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { + memcpy(dst, cSrc, dstSize); + return dstSize; + } /* not compressed */ + if (cSrcSize == 1) { + memset(dst, *(const BYTE*)cSrc, dstSize); + return dstSize; + } /* RLE */ + + { + U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize) + : HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif + } +} + +size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X_usingDTable_bmi2( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) + : HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int bmi2) +{ + const BYTE* ip = (const BYTE*)cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) + return hSize; + if (hSize >= cSrcSize) + return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); +} +#endif + +size_t HUF_decompress4X_usingDTable_bmi2( + void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) + : HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +{ + /* validation checks */ + if (dstSize == 0) + return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) + return ERROR(corruption_detected); + + { + U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) + : HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#endif + } +} diff --git a/src/lib/compress/zstd_1_3_8/mem.h b/src/lib/compress/zstd_1_3_8/mem.h new file mode 100644 index 0000000000000000000000000000000000000000..f7ada7bb298e7edf142d2696d9f5ba72ce4b9f7a --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/mem.h @@ -0,0 +1,452 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined(__cplusplus) +extern "C" { +#endif + +/*-**************************************** + * Dependencies + ******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + +/*-**************************************** + * Compiler specifics + ******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +#include /* _byteswap_ulong */ +#include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +#define MEM_STATIC static __inline __attribute__((unused)) +#elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#define MEM_STATIC static inline +#elif defined(_MSC_VER) +#define MEM_STATIC static __inline +#else +#define MEM_STATIC \ + static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 /* compat. with non-clang compilers */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) \ + { \ + enum { MEM_static_assert = 1 / (int)(!!(c)) }; \ + } +MEM_STATIC void MEM_check(void) +{ + MEM_STATIC_ASSERT((sizeof(size_t) == 4) || (sizeof(size_t) == 8)); +} + +/*-************************************************************** + * Basic Types + *****************************************************************/ +#if !defined(__VMS) && (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 \ + */)) +#include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef int16_t S16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +typedef int64_t S64; +#else +#include +#if CHAR_BIT != 8 +#error "this implementation requires char to be exactly 8-bit type" +#endif +typedef unsigned char BYTE; +#if USHRT_MAX != 65535 +#error "this implementation requires short to be exactly 16-bit type" +#endif +typedef unsigned short U16; +typedef signed short S16; +#if UINT_MAX != 4294967295 +#error "this implementation requires int to be exactly 32-bit type" +#endif +typedef unsigned int U32; +typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ +typedef unsigned long long U64; +typedef signed long long S64; +#endif + +/*-************************************************************** + * Memory I/O + *****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +#if defined(__GNUC__) && (defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || \ + defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__)) +#define MEM_FORCE_MEMORY_ACCESS 2 +#elif defined(__INTEL_COMPILER) || defined(__GNUC__) +#define MEM_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) +{ + return sizeof(size_t) == 4; +} +MEM_STATIC unsigned MEM_64bits(void) +{ + return sizeof(size_t) == 8; +} + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { + U32 u; + BYTE c[4]; + } one = {1}; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS == 2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + return *(const U16*)memPtr; +} +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + return *(const U32*)memPtr; +} +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + return *(const U64*)memPtr; +} +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + return *(const size_t*)memPtr; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + *(U16*)memPtr = value; +} +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + *(U32*)memPtr = value; +} +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + *(U64*)memPtr = value; +} + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS == 1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) +__pragma(pack(push, 1)) typedef struct { + U16 v; +} unalign16; +typedef struct { + U32 v; +} unalign32; +typedef struct { + U64 v; +} unalign64; +typedef struct { + size_t v; +} unalignArch; +__pragma(pack(pop)) +#else +typedef struct { + U16 v; +} __attribute__((packed)) unalign16; +typedef struct { + U32 v; +} __attribute__((packed)) unalign32; +typedef struct { + U64 v; +} __attribute__((packed)) unalign64; +typedef struct { + size_t v; +} __attribute__((packed)) unalignArch; +#endif + + MEM_STATIC U16 MEM_read16(const void* ptr) +{ + return ((const unalign16*)ptr)->v; +} +MEM_STATIC U32 MEM_read32(const void* ptr) +{ + return ((const unalign32*)ptr)->v; +} +MEM_STATIC U64 MEM_read64(const void* ptr) +{ + return ((const unalign64*)ptr)->v; +} +MEM_STATIC size_t MEM_readST(const void* ptr) +{ + return ((const unalignArch*)ptr)->v; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ((unalign16*)memPtr)->v = value; +} +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ((unalign32*)memPtr)->v = value; +} +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ((unalign64*)memPtr)->v = value; +} + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) || \ + (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000) | ((in << 8) & 0x00ff0000) | ((in >> 8) & 0x0000ff00) | ((in >> 24) & 0x000000ff); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) || \ + (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1] << 8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val >> 8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val >> 16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +#if defined(__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.cpp b/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b77d53cdcc231eb6986e56aa713d13f651372b3f --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.cpp @@ -0,0 +1,157 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_zstd_compressor_1_3_8.h" + +#include "lib/ob_errno.h" +#include "lib/thread_local/ob_tsi_factory.h" +#include "ob_zstd_wrapper.h" + +using namespace oceanbase; +using namespace common; +using namespace zstd_1_3_8; + +const char* ObZstdCompressor_1_3_8::compressor_name = "zstd_1.3.8"; + +static void* ob_zstd_malloc(void* opaque, size_t size) +{ + void* buf = NULL; + if (NULL != opaque) { + ObZstdCtxAllocator* allocator = reinterpret_cast(opaque); + buf = allocator->alloc(size); + } + return buf; +} + +static void ob_zstd_free(void* opaque, void* address) +{ + if (NULL != opaque) { + ObZstdCtxAllocator* allocator = reinterpret_cast(opaque); + allocator->free(address); + } +} + +/** + * ------------------------------ObZstdCtxAllocator--------------------- + */ +ObZstdCtxAllocator::ObZstdCtxAllocator() : allocator_(ObModIds::OB_COMPRESSOR, OB_MALLOC_BIG_BLOCK_SIZE) +{} + +ObZstdCtxAllocator::~ObZstdCtxAllocator() +{} + +void* ObZstdCtxAllocator::alloc(size_t size) +{ + return allocator_.alloc(size); +} + +void ObZstdCtxAllocator::free(void* addr) +{ + allocator_.free(addr); +} + +void ObZstdCtxAllocator::reuse() +{ + allocator_.reuse(); +} + +/** + * ----------------------------ObZstdCompressor--------------------------- + */ +int ObZstdCompressor_1_3_8::compress(const char* src_buffer, const int64_t src_data_size, char* dst_buffer, + const int64_t dst_buffer_size, int64_t& dst_data_size) +{ + int ret = OB_SUCCESS; + int64_t max_overflow_size = 0; + size_t compress_ret_size = 0; + ObZstdCtxAllocator* zstd_allocator = GET_TSI_MULT(ObZstdCtxAllocator, 1); + OB_ZSTD_customMem zstd_mem = {ob_zstd_malloc, ob_zstd_free, zstd_allocator}; + dst_data_size = 0; + + if (NULL == src_buffer || 0 >= src_data_size || NULL == dst_buffer || 0 >= dst_buffer_size) { + ret = OB_INVALID_ARGUMENT; + LIB_LOG(WARN, + "invalid compress argument, ", + K(ret), + KP(src_buffer), + K(src_data_size), + KP(dst_buffer), + K(dst_buffer_size)); + } else if (OB_FAIL(get_max_overflow_size(src_data_size, max_overflow_size))) { + LIB_LOG(WARN, "fail to get max_overflow_size, ", K(ret), K(src_data_size)); + } else if ((src_data_size + max_overflow_size) > dst_buffer_size) { + ret = OB_BUF_NOT_ENOUGH; + LIB_LOG(WARN, "dst buffer not enough, ", K(ret), K(src_data_size), K(max_overflow_size), K(dst_buffer_size)); + } else if (OB_FAIL(ObZstdWrapper::compress(zstd_mem, + src_buffer, + static_cast(src_data_size), + dst_buffer, + static_cast(dst_buffer_size), + compress_ret_size))) { + LIB_LOG(WARN, "failed to compress zstd", K(ret), K(compress_ret_size)); + } else { + dst_data_size = compress_ret_size; + } + + if (NULL != zstd_allocator) { + zstd_allocator->reuse(); + } + return ret; +} + +int ObZstdCompressor_1_3_8::decompress(const char* src_buffer, const int64_t src_data_size, char* dst_buffer, + const int64_t dst_buffer_size, int64_t& dst_data_size) +{ + int ret = OB_SUCCESS; + size_t decompress_ret_size = 0; + ObZstdCtxAllocator* zstd_allocator = GET_TSI_MULT(ObZstdCtxAllocator, 1); + OB_ZSTD_customMem zstd_mem = {ob_zstd_malloc, ob_zstd_free, zstd_allocator}; + dst_data_size = 0; + + if (NULL == src_buffer || 0 >= src_data_size || NULL == dst_buffer || 0 >= dst_buffer_size) { + ret = OB_INVALID_ARGUMENT; + LIB_LOG(WARN, + "invalid decompress argument, ", + K(ret), + KP(src_buffer), + K(src_data_size), + KP(dst_buffer), + K(dst_buffer_size)); + } else if (OB_FAIL(ObZstdWrapper::decompress( + zstd_mem, src_buffer, src_data_size, dst_buffer, dst_buffer_size, decompress_ret_size))) { + LIB_LOG(WARN, "failed to decompress zstd", K(ret), K(decompress_ret_size)); + } else { + dst_data_size = decompress_ret_size; + } + + if (NULL != zstd_allocator) { + zstd_allocator->reuse(); + } + return ret; +} + +const char* ObZstdCompressor_1_3_8::get_compressor_name() const +{ + return compressor_name; +} + +int ObZstdCompressor_1_3_8::get_max_overflow_size(const int64_t src_data_size, int64_t& max_overflow_size) const +{ + int ret = OB_SUCCESS; + if (src_data_size < 0) { + ret = OB_INVALID_ARGUMENT; + LIB_LOG(WARN, "invalid argument, ", K(ret), K(src_data_size)); + } else { + max_overflow_size = (src_data_size >> 7) + 512 + 12; + } + return ret; +} diff --git a/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.h b/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.h new file mode 100644 index 0000000000000000000000000000000000000000..2dad0b7bc72e8169f549cea39457390097243a47 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.h @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_COMMON_COMPRESS_ZSTD_1_3_8_COMPRESSOR_ +#define OCEANBASE_COMMON_COMPRESS_ZSTD_1_3_8_COMPRESSOR_ +#include "lib/compress/ob_compressor.h" +#include "lib/allocator/page_arena.h" + +namespace oceanbase { +namespace common { + +namespace zstd_1_3_8 { + +class ObZstdCtxAllocator { +public: + ObZstdCtxAllocator(); + virtual ~ObZstdCtxAllocator(); + void* alloc(size_t size); + void free(void* addr); + void reuse(); + +private: + ObArenaAllocator allocator_; +}; + +class __attribute__((visibility("default"))) ObZstdCompressor_1_3_8 : public ObCompressor { +public: + explicit ObZstdCompressor_1_3_8() + {} + virtual ~ObZstdCompressor_1_3_8() + {} + int compress(const char* src_buffer, const int64_t src_data_size, char* dst_buffer, const int64_t dst_buffer_size, + int64_t& dst_data_size); + int decompress(const char* src_buffer, const int64_t src_data_size, char* dst_buffer, const int64_t dst_buffer_size, + int64_t& dst_data_size); + const char* get_compressor_name() const; + int get_max_overflow_size(const int64_t src_data_size, int64_t& max_overflow_size) const; + +private: + static const char* compressor_name; +}; +} // namespace zstd_1_3_8 +} // namespace common +} // namespace oceanbase +#endif // OCEANBASE_COMMON_COMPRESS_ZSTD_1_3_8_COMPRESSOR_ diff --git a/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.cpp b/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9bf9fcee1bae7c7f0fef896741424d9b70f362d6 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.cpp @@ -0,0 +1,285 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_zstd_wrapper.h" +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +using namespace oceanbase; +using namespace common; +using namespace zstd_1_3_8; + +constexpr int OB_SUCCESS = 0; +constexpr int OB_INVALID_ARGUMENT = -4002; +constexpr int OB_ALLOCATE_MEMORY_FAILED = -4013; +constexpr int OB_ERR_COMPRESS_DECOMPRESS_DATA = -4257; +constexpr int OB_IO_ERROR = -4009; +constexpr int OB_ZSTD_VERSION_138 = 138; + +static const int OB_ZSTD_COMPRESS_LEVEL = 1; + +int ObZstdWrapper::compress(OB_ZSTD_customMem& ob_zstd_mem, const char* src_buffer, const size_t src_data_size, + char* dst_buffer, const size_t dst_buffer_size, size_t& compress_ret_size) +{ + int ret = OB_SUCCESS; + ZSTD_CCtx* zstd_cctx = NULL; + ZSTD_customMem zstd_mem; + int zstd_version = 0; + zstd_mem.customAlloc = ob_zstd_mem.customAlloc; + zstd_mem.customFree = ob_zstd_mem.customFree; + zstd_mem.opaque = ob_zstd_mem.opaque; + + if (NULL == src_buffer || 0 >= src_data_size || NULL == dst_buffer || 0 >= dst_buffer_size) { + ret = OB_INVALID_ARGUMENT; + fprintf(stderr, + __FILE__ ": invalid args, ret=%d src_buffer=%p src_data_size=%lu dst_buffer=%p dst_buffer_size=%lu\n", + ret, + src_buffer, + src_data_size, + dst_buffer, + dst_buffer_size); + } else if (NULL == (zstd_cctx = ZSTD_createCCtx_advanced(zstd_mem))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + fprintf(stderr, __FILE__ ": failed to create cctx\n"); + } else { + compress_ret_size = ZSTD_compressCCtx( + zstd_cctx, dst_buffer, dst_buffer_size, src_buffer, src_data_size, OB_ZSTD_COMPRESS_LEVEL, &zstd_version); + if (0 != ZSTD_isError(compress_ret_size)) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, + __FILE__ ": fail to compress data, ret=%d compress_ret_size=%lu src_buffer=%p src_data_size=%lu " + "dst_buffer=%p dst_buffer_size=%lu compress_level=%d\n", + ret, + compress_ret_size, + src_buffer, + src_data_size, + dst_buffer, + dst_buffer_size, + OB_ZSTD_COMPRESS_LEVEL); + } else if (OB_ZSTD_VERSION_138 != zstd_version) { + ret = OB_IO_ERROR; + fprintf(stderr, + __FILE__ ": invalid ZSTD_compressCCtx version, ret=%d lib version=%d expect version=%d", + ret, + zstd_version, + OB_ZSTD_VERSION_138); + } + } + + if (NULL != zstd_cctx) { + ZSTD_freeCCtx(zstd_cctx); + zstd_cctx = NULL; + } + return ret; +} + +int ObZstdWrapper::decompress(OB_ZSTD_customMem& ob_zstd_mem, const char* src_buffer, const size_t src_data_size, + char* dst_buffer, const size_t dst_buffer_size, size_t& dst_data_size) +{ + int ret = OB_SUCCESS; + ZSTD_DCtx* zstd_dctx = NULL; + ZSTD_customMem zstd_mem; + int zstd_version = 0; + zstd_mem.customAlloc = ob_zstd_mem.customAlloc; + zstd_mem.customFree = ob_zstd_mem.customFree; + zstd_mem.opaque = ob_zstd_mem.opaque; + dst_data_size = 0; + + if (NULL == src_buffer || 0 >= src_data_size || NULL == dst_buffer || 0 >= dst_buffer_size) { + ret = OB_INVALID_ARGUMENT; + fprintf(stderr, + __FILE__ ": invalid args, ret=%d src_buffer=%p src_data_size=%lu dst_buffer=%p dst_buffer_size=%lu\n", + ret, + src_buffer, + src_data_size, + dst_buffer, + dst_buffer_size); + } else if (NULL == (zstd_dctx = ZSTD_createDCtx_advanced(zstd_mem))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + fprintf(stderr, __FILE__ ": failed to create dctx\n"); + } else { + dst_data_size = + ZSTD_decompressDCtx(zstd_dctx, dst_buffer, dst_buffer_size, src_buffer, src_data_size, &zstd_version); + if (0 != ZSTD_isError(dst_data_size)) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, + __FILE__ ": failed to decompress data, ret=%d src_buffer=%p src_data_size=%lu dst_buffer=%p " + "dst_buffer_size=%lu dst_data_size =%lu\n", + ret, + src_buffer, + src_data_size, + dst_buffer, + dst_buffer_size, + dst_data_size); + } else if (OB_ZSTD_VERSION_138 != zstd_version) { + ret = OB_IO_ERROR; + fprintf(stderr, + __FILE__ ": invalid ZSTD_decompressDCtx version, ret=%d lib version=%d expect version=%d", + ret, + zstd_version, + OB_ZSTD_VERSION_138); + } + } + + if (NULL != zstd_dctx) { + ZSTD_freeDCtx(zstd_dctx); + zstd_dctx = NULL; + } + return ret; +} + +int ObZstdWrapper::create_cctx(OB_ZSTD_customMem& ob_zstd_mem, void*& ctx) +{ + int ret = OB_SUCCESS; + size_t ret_code = 0; + ZSTD_CCtx* cctx = NULL; + ZSTD_customMem zstd_mem; + zstd_mem.customAlloc = ob_zstd_mem.customAlloc; + zstd_mem.customFree = ob_zstd_mem.customFree; + zstd_mem.opaque = ob_zstd_mem.opaque; + + ctx = NULL; + + if (NULL == (cctx = ZSTD_createCCtx_advanced(zstd_mem))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + fprintf(stderr, __FILE__ ": failed to create cctx\n"); + } else if (0 != ZSTD_isError(ret_code = ZSTD_compressBegin(cctx, OB_ZSTD_COMPRESS_LEVEL))) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, __FILE__ ": failed to begin compress, ret=%d ret_code=%lu\n", ret, ret_code); + ZSTD_freeCCtx(cctx); + cctx = NULL; + } else { + ctx = cctx; + } + return ret; +} + +void ObZstdWrapper::free_cctx(void*& ctx) +{ + ZSTD_CCtx* cctx = static_cast(ctx); + ZSTD_freeCCtx(cctx); +} + +int ObZstdWrapper::compress_block( + void* ctx, const char* src, const size_t src_size, char* dest, const size_t dest_capacity, size_t& compressed_size) +{ + int ret = OB_SUCCESS; + ZSTD_CCtx* cctx = static_cast(ctx); + compressed_size = 0; + + if (NULL == ctx || NULL == src || NULL == dest) { + ret = OB_INVALID_ARGUMENT; + fprintf(stderr, __FILE__ ":invalid args, ret=%d ctx=%p src=%p dest=%p\n", ret, ctx, src, dest); + } else { + compressed_size = ZSTD_compressBlock(cctx, dest, dest_capacity, src, src_size); + if (0 != ZSTD_isError(compressed_size)) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, + __FILE__ ": failed to compress block, ret=%d src=%p src_size=%lu dest=%p dest=%lu compressed_size=%lu\n", + ret, + src, + src_size, + dest, + dest_capacity, + compressed_size); + } + } + + return ret; +} + +int ObZstdWrapper::create_dctx(OB_ZSTD_customMem& ob_zstd_mem, void*& ctx) +{ + int ret = OB_SUCCESS; + size_t ret_code = 0; + ZSTD_DCtx* dctx = NULL; + ZSTD_customMem zstd_mem; + zstd_mem.customAlloc = ob_zstd_mem.customAlloc; + zstd_mem.customFree = ob_zstd_mem.customFree; + zstd_mem.opaque = ob_zstd_mem.opaque; + ctx = NULL; + + if (NULL == (dctx = ZSTD_createDCtx_advanced(zstd_mem))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + fprintf(stderr, __FILE__ ": failed to create dctx\n"); + } else if (ZSTD_isError(ret_code = ZSTD_decompressBegin(dctx))) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, __FILE__ ": failed to begin decompress, ret=%d ret_code=%lu\n", ret, ret_code); + ZSTD_freeDCtx(dctx); + dctx = NULL; + } else { + ctx = dctx; + } + return ret; +} + +void ObZstdWrapper::free_dctx(void*& ctx) +{ + ZSTD_DCtx* dctx = static_cast(ctx); + ZSTD_freeDCtx(dctx); +} + +int ObZstdWrapper::decompress_block(void* ctx, const char* src, const size_t src_size, char* dest, + const size_t dest_capacity, size_t& decompressed_size) +{ + int ret = OB_SUCCESS; + decompressed_size = 0; + + if (NULL == ctx || NULL == src || NULL == dest || src_size <= 0 || dest_capacity <= 0) { + ret = OB_INVALID_ARGUMENT; + fprintf(stderr, + __FILE__ ": invalid args, ret=%d ctx=%p src=%p src_size=%lu dest=%p dest_capacity=%lu\n", + ret, + ctx, + src, + src_size, + dest, + dest_capacity); + } else { + ZSTD_DCtx* dctx = static_cast(ctx); + decompressed_size = ZSTD_decompressBlock(dctx, dest, dest_capacity, src, src_size); + if (0 != ZSTD_isError(decompressed_size)) { + ret = OB_ERR_COMPRESS_DECOMPRESS_DATA; + fprintf(stderr, + __FILE__ ": failed to decompress block, ret=%d ctx=%p src=%p src_size=%lu dest=%p dest_capacity=%lu " + "decompressed_size-%lu\n", + ret, + ctx, + src, + src_size, + dest, + dest_capacity, + decompressed_size); + } + } + return ret; +} + +size_t ObZstdWrapper::compress_bound(const size_t src_size) +{ + return ZSTD_compressBound(src_size); +} + +int ObZstdWrapper::insert_block(void* ctx, const void* block, const size_t block_size) +{ + int ret = OB_SUCCESS; + if (NULL == ctx || NULL == block || 0 >= block_size) { + ret = OB_INVALID_ARGUMENT; + fprintf(stderr, __FILE__ ": invalid args, ret=%d ctx=%p block=%p block_size=%lu\n", ret, ctx, block, block_size); + } else { + ZSTD_DCtx* dctx = static_cast(ctx); + ZSTD_insertBlock(dctx, block, block_size); + } + return ret; +} diff --git a/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.h b/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..e4c0937d0710f5c123c44739b38d26f4412b3724 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/ob_zstd_wrapper.h @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef DEPS_OBLIB_SRC_LIB_COMPRESS_ZSTD_OB_ZSTD_WRAPPER_H_ +#define DEPS_OBLIB_SRC_LIB_COMPRESS_ZSTD_OB_ZSTD_WRAPPER_H_ + +#include + +namespace oceanbase { +namespace common { +namespace zstd_1_3_8 { + +/*= Custom memory allocation functions */ +typedef void* (*OB_ZSTD_allocFunction)(void* opaque, size_t size); +typedef void (*OB_ZSTD_freeFunction)(void* opaque, void* address); +typedef struct { + OB_ZSTD_allocFunction customAlloc; + OB_ZSTD_freeFunction customFree; + void* opaque; +} OB_ZSTD_customMem; + +#define OB_PUBLIC_API __attribute__((visibility("default"))) + +class OB_PUBLIC_API ObZstdWrapper final { +public: + // for normal + static int compress(OB_ZSTD_customMem& zstd_mem, const char* src_buffer, const size_t src_data_size, char* dst_buffer, + const size_t dst_buffer_size, size_t& compress_ret_size); + static int decompress(OB_ZSTD_customMem& zstd_mem, const char* src_buffer, const size_t src_data_size, + char* dst_buffer, const size_t dst_buffer_size, size_t& dst_data_size); + + // for stream + static int create_cctx(OB_ZSTD_customMem& ob_zstd_mem, void*& ctx); + static void free_cctx(void*& ctx); + static int compress_block(void* ctx, const char* src, const size_t src_size, char* dest, const size_t dest_capacity, + size_t& compressed_size); + static int create_dctx(OB_ZSTD_customMem& ob_zstd_mem, void*& ctx); + static void free_dctx(void*& ctx); + static int decompress_block(void* ctx, const char* src, const size_t src_size, char* dest, const size_t dest_capacity, + size_t& decompressed_size); + static size_t compress_bound(const size_t src_size); + static int insert_block(void* ctx, const void* block, const size_t block_size); +}; + +#undef OB_PUBLIC_API + +} // namespace zstd_1_3_8 +} // namespace common +} // namespace oceanbase + +#endif /* DEPS_OBLIB_SRC_LIB_COMPRESS_ZSTD_OB_ZSTD_WRAPPER_H_ */ diff --git a/src/lib/compress/zstd_1_3_8/pool.c b/src/lib/compress/zstd_1_3_8/pool.c new file mode 100644 index 0000000000000000000000000000000000000000..c9a2a7528a43ced3ef725b6d10988533108874a0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/pool.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* ====== Dependencies ======= */ +#include /* size_t */ +#include "debug.h" /* assert */ +#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +#pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void* opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job* queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) +{ + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { + return NULL; + } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while (ctx->queueEmpty || (ctx->numThreadsBusy >= ctx->threadLimit)) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { + POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = ctx->queueHead == ctx->queueTail; + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + if (ctx->queueSize == 1) { + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) +{ + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { + return NULL; + } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem); + if (!ctx) { + return NULL; + } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { + POOL_free(ctx); + return NULL; + } + /* Initialize the threads */ + { + size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } + } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) +{ + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { + size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */ + } + } +} + +void POOL_free(POOL_ctx* ctx) +{ + if (!ctx) { + return; + } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_free(ctx->queue, ctx->customMem); + ZSTD_free(ctx->threads, ctx->customMem); + ZSTD_free(ctx, ctx->customMem); +} + +size_t POOL_sizeof(POOL_ctx* ctx) +{ + if (ctx == NULL) + return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + ctx->queueSize * sizeof(POOL_job) + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) + return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { + ZSTD_pthread_t* const threadPool = + (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) + return 1; + /* replace existing thread pool */ + memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_free(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { + size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } + } + } + } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx == NULL) + return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) +{ + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || !ctx->queueEmpty; + } +} + +static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + POOL_job const job = {function, opaque}; + assert(ctx != NULL); + if (ctx->shutdown) + return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_ctx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) +{ + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_ctx; +} + +void POOL_free(POOL_ctx* ctx) +{ + assert(!ctx || ctx == &g_ctx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + (void)ctx; + (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(POOL_ctx* ctx) +{ + if (ctx == NULL) + return 0; /* supports sizeof NULL */ + assert(ctx == &g_ctx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/src/lib/compress/zstd_1_3_8/pool.h b/src/lib/compress/zstd_1_3_8/pool.h new file mode 100644 index 0000000000000000000000000000000000000000..ddcf7ca456ac8aa9a6a8f05a9ddfef90e1d28eb2 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/pool.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. + */ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and re-use existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a worker is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/src/lib/compress/zstd_1_3_8/threading.c b/src/lib/compress/zstd_1_3_8/threading.c new file mode 100644 index 0000000000000000000000000000000000000000..1110b8e970ec03a7bb22a9a0fc36758871960ada --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/threading.c @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +/* create fake symbol to avoid empty trnaslation unit warning */ +int g_ZSTD_threading_useles_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ + +/* === Dependencies === */ +#include +#include +#include "threading.h" + +/* === Implementation === */ + +static unsigned __stdcall worker(void* arg) +{ + ZSTD_pthread_t* const thread = (ZSTD_pthread_t*)arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine)(void*), void* arg) +{ + (void)unused; + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = (HANDLE)_beginthreadex(NULL, 0, worker, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr) +{ + DWORD result; + + if (!thread.handle) + return 0; + + result = WaitForSingleObject(thread.handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) + *value_ptr = thread.arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/src/lib/compress/zstd_1_3_8/threading.h b/src/lib/compress/zstd_1_3_8/threading.h new file mode 100644 index 0000000000000000000000000000000000000000..28b12d5eabf48cc9b9e68d577daa9408dd3904b0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/threading.h @@ -0,0 +1,120 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#if defined(__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ +#ifdef WINVER +#undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef struct { + HANDLE handle; + void* (*start_routine)(void*); + void* arg; +} ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine)(void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); + +/** + * add here more wrappers as required + */ + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +#include + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a), (b)) + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/src/lib/compress/zstd_1_3_8/xxhash.c b/src/lib/compress/zstd_1_3_8/xxhash.c new file mode 100644 index 0000000000000000000000000000000000000000..fc498468892b8be2cb0251bc4c2b8e71d978244b --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/xxhash.c @@ -0,0 +1,906 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (C) 2012-2016, Yann Collet + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - xxHash homepage: http://www.xxhash.com + * - xxHash source repository : https://github.com/Cyan4973/xxHash + */ + +/* ************************************* + * Tuning parameters + ***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +#if defined(__GNUC__) && (defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || \ + defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__)) +#define XXH_FORCE_MEMORY_ACCESS 2 +#elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || \ + defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__))) +#define XXH_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a + * bad pointer. When this option is enabled, xxHash output for null input pointers will be the same as a null-length + * input. By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +#define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +#define XXH_FORCE_ALIGN_CHECK 0 +#else +#define XXH_FORCE_ALIGN_CHECK 1 +#endif +#endif + +/* ************************************* + * Includes & Memory related functions + ***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include +#include /* size_t */ +static void* XXH_malloc(size_t s) +{ + return malloc(s); +} +static void XXH_free(void* p) +{ + free(p); +} +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest, src, size); +} + +#ifndef XXH_STATIC_LINKING_ONLY +#define XXH_STATIC_LINKING_ONLY +#endif +#include "xxhash.h" + +/* ************************************* + * Compiler Specific Options + ***************************************/ +#if defined(__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +#define INLINE_KEYWORD inline +#else +#define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) +#define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define FORCE_INLINE_ATTR __forceinline +#else +#define FORCE_INLINE_ATTR +#endif + +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR + +#ifdef _MSC_VER +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +/* ************************************* + * Basic Types + ***************************************/ +#ifndef MEM_MODULE +#define MEM_MODULE +#if !defined(__VMS) && (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 \ + */)) +#include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type + here. Note that xxhash.h will also need to be updated. */ +#endif +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) +{ + return *(const U32*)memPtr; +} +static U64 XXH_read64(const void* memPtr) +{ + return *(const U64*)memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { + U32 u32; + U64 u64; +} __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) +{ + return ((const unalign*)ptr)->u32; +} +static U64 XXH_read64(const void* ptr) +{ + return ((const unalign*)ptr)->u64; +} + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +/* **************************************** + * Compiler-specific Functions and Macros + ******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +#define XXH_rotl32(x, r) _rotl(x, r) +#define XXH_rotl64(x, r) _rotl64(x, r) +#else +#define XXH_rotl32(x, r) ((x << r) | (x >> (32 - r))) +#define XXH_rotl64(x, r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +#define XXH_swap32 _byteswap_ulong +#define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +#define XXH_swap32 __builtin_bswap32 +#define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32(U32 x) +{ + return ((x << 24) & 0xff000000) | ((x << 8) & 0x00ff0000) | ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); +} +static U64 XXH_swap64(U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | ((x << 8) & 0x000000ff00000000ULL) | ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + +/* ************************************* + * Architecture Macros + ***************************************/ +typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN +static const int g_one = 1; +#define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + +/* *************************** + * Memory reads + *****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align == XXH_unaligned) + return endian == XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian == XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align == XXH_unaligned) + return endian == XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian == XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + +/* ************************************* + * Macros + ***************************************/ +#define XXH_STATIC_ASSERT(c) \ + { \ + enum { XXH_static_assert = 1 / (int)(!!(c)) }; \ + } /* use only *after* variable declarations */ + +/* ************************************* + * Constants + ***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber(void) +{ + return XXH_VERSION_NUMBER; +} + +/* ************************** + * Utils + ****************************/ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +/* *************************** + * Simple Hash Functions + *****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE_TEMPLATE U32 XXH32_endian_align( + const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p == NULL) { + len = 0; + bEnd = p = (const BYTE*)(size_t)16; + } +#endif + + if (len >= 16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); + p += 4; + v2 = XXH32_round(v2, XXH_get32bits(p)); + p += 4; + v3 = XXH32_round(v3, XXH_get32bits(p)); + p += 4; + v4 = XXH32_round(v4, XXH_get32bits(p)); + p += 4; + } while (p <= limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32)len; + + while (p + 4 <= bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p += 4; + } + + while (p < bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + +XXH_PUBLIC_API unsigned int XXH32(const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } + } + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE_TEMPLATE U64 XXH64_endian_align( + const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p == NULL) { + len = 0; + bEnd = p = (const BYTE*)(size_t)32; + } +#endif + + if (len >= 32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); + p += 8; + v2 = XXH64_round(v2, XXH_get64bits(p)); + p += 8; + v3 = XXH64_round(v3, XXH_get64bits(p)); + p += 8; + v4 = XXH64_round(v4, XXH_get64bits(p)); + p += 8; + } while (p <= limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64)len; + + while (p + 8 <= bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; + p += 8; + } + + if (p + 4 <= bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p += 4; + } + + while (p < bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + +XXH_PUBLIC_API unsigned long long XXH64(const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7) == 0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } + } + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/* ************************************************** + * Advanced Hash Functions + ****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state) - 4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state) - 8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian( + XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input == NULL) + return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len >= 16) | (state->total_len_32 >= 16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16 - state->memsize); + { + const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); + p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); + p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); + p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + p32++; + } + p += 16 - state->memsize; + state->memsize = 0; + } + + if (p <= bEnd - 16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); + p += 4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); + p += 4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); + p += 4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); + p += 4; + } while (p <= limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd - p)); + state->memsize = (unsigned)(bEnd - p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + +FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian(const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p + 4 <= bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p += 4; + } + + while (p < bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + +XXH_PUBLIC_API unsigned int XXH32_digest(const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + +/* **** XXH64 **** */ + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian( + XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input == NULL) + return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32 - state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64 + 0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64 + 1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64 + 2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64 + 3, endian)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p + 32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); + p += 8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); + p += 8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); + p += 8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); + p += 8; + } while (p <= limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd - p)); + state->memsize = (unsigned)(bEnd - p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + +FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian(const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64)state->total_len; + + while (p + 8 <= bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; + p += 8; + } + + if (p + 4 <= bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p += 4; + } + + while (p < bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + +XXH_PUBLIC_API unsigned long long XXH64_digest(const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected == XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + +/* ************************** + * Canonical representation + ****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. + * The canonical representation follows human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file or buffer, and remain comparable across different systems and + * programs. + */ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) + hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) + hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} diff --git a/src/lib/compress/zstd_1_3_8/xxhash.h b/src/lib/compress/zstd_1_3_8/xxhash.h new file mode 100644 index 0000000000000000000000000000000000000000..d707d7a65e1c90f0f08ddea1af1991c89e114d83 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/xxhash.h @@ -0,0 +1,298 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined(__cplusplus) +extern "C" { +#endif + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/* **************************** + * Definitions + ******************************/ +#include /* size_t */ +typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; + +/* **************************** + * API modifier + ******************************/ +/** XXH_PRIVATE_API + * This is useful if you want to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Methodology : + * #define XXH_PRIVATE_API + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module anymore. + */ +#ifdef XXH_PRIVATE_API +#ifndef XXH_STATIC_LINKING_ONLY +#define XXH_STATIC_LINKING_ONLY +#endif +#if defined(__GNUC__) +#define XXH_PUBLIC_API static __inline __attribute__((unused)) +#elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#define XXH_PUBLIC_API static inline +#elif defined(_MSC_VER) +#define XXH_PUBLIC_API static __inline +#else +#define XXH_PUBLIC_API \ + static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif +#else +#define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +#define XXH_CAT(A, B) A##B +#define XXH_NAME2(A, B) XXH_CAT(A, B) +#define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +#define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +#define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +#define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +#define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +#define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +#define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +#define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +#define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +#define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +#define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +#define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +#define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +#define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +#define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +#define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +#define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +#define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + +/* ************************************* + * Version + ***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber(void); + +/* **************************** + * Simple Hash Functions + ******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32(const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + +/* **************************** + * Streaming Hash Functions + ******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! State allocation, compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + +/* ************************** + * Utils + ****************************/ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ +#define restrict /* disable restrict */ +#endif + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); + +/* ************************** + * Canonical representation + ****************************/ +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ +typedef struct { + unsigned char digest[4]; +} XXH32_canonical_t; +typedef struct { + unsigned char digest[8]; +} XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#endif /* XXHASH_H_5627135585666179 */ + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) +#define XXH_STATIC_H_3543687687345 + +/* These definitions are only meant to allow allocation of XXH state + statically, on stack, or in a struct for example. + Do not use members directly. */ + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +#ifdef XXH_PRIVATE_API +#include "xxhash.c" /* include xxhash functions as `static`, for inlining */ +#endif + +#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ + +#if defined(__cplusplus) +} +#endif diff --git a/src/lib/compress/zstd_1_3_8/zstd.h b/src/lib/compress/zstd_1_3_8/zstd.h new file mode 100644 index 0000000000000000000000000000000000000000..4cd1d248b59be41cf1afd68f0f14a6f17cff2013 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd.h @@ -0,0 +1,1762 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined(__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 +#define OB_ZSTD_LIB_VERSION_138 138 + +/* ====== Dependency ======*/ +#include /* size_t */ + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBILITY +#if defined(__GNUC__) && (__GNUC__ >= 4) +#define ZSTDLIB_VISIBILITY __attribute__((visibility("default"))) +#else +#define ZSTDLIB_VISIBILITY +#endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT == 1) +#define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT == 1) +#define ZSTDLIB_API \ + __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function \ + pointer load from the IAT and an indirect jump.*/ +#else +#define ZSTDLIB_API ZSTDLIB_VISIBILITY +#endif + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 3 +#define ZSTD_VERSION_RELEASE 8 + +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR * 100 * 100 + ZSTD_VERSION_MINOR * 100 + ZSTD_VERSION_RELEASE) +ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) +ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ + +/*************************************** + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +#define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/*************************************** + * Simple API + ***************************************/ +/*! ZSTD_compress() : + * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void* src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*====== Helper functions ======*/ +#define ZSTD_COMPRESSBOUND(srcSize) \ + ((srcSize) + ((srcSize) >> 8) + \ + (((srcSize) < (128 << 10)) \ + ? (((128 << 10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ \ + : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound( + size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ + +/*************************************** + * Explicit context + ***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution in multi-threaded environments. */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx + * The function will compress at requested compression level, + * ignoring any other parameter */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + int compressionLevel, int* zstd_version); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int* zstd_version); + +/************************** + * Simple dictionary API + ***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see dictBuilder/zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const void* dict, size_t dictSize, int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize); + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. + * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup + * cost. ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. + * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. + */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict( + ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict); + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_DDict* ddict); + +/**************************** + * Streaming + ****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + +/*-*********************************************************************** + * Streaming compression - HowTo + * + * A ZSTD_CStream object is required to track streaming operation. + * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. + * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. + * It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already + * allocated memory. + * + * For parallel execution, use one separate ZSTD_CStream per thread. + * + * note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. + * + * Parameters are sticky : when starting a new compression on the same context, + * it will re-use the same sticky parameters as previous compression session. + * When in doubt, it's recommended to fully initialize the context before usage. + * Use ZSTD_initCStream() to set the parameter to a selected compression level. + * Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters. + * + * Use ZSTD_compressStream() as many times as necessary to consume input stream. + * The function will automatically update both `pos` fields within `input` and `output`. + * Note that the function may not consume the entire input, + * for example, because the output buffer is already full, + * in which case `input.pos < input.size`. + * The caller must check if input has been entirely consumed. + * If not, the caller must make some room to receive more compressed data, + * and then present again remaining input data. + * @return : a size hint, preferred nb of bytes to use as input for next function call + * or an error code, which can be tested using ZSTD_isError(). + * Note 1 : it's just a hint, to help latency a little, any value will work fine. + * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() + * + * At any moment, it's possible to flush whatever data might remain stuck within internal buffer, + * using ZSTD_flushStream(). `output->pos` will be updated. + * Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return + * code > 0). In which case, make some room to receive more compressed data, and call again ZSTD_flushStream(). + * @return : 0 if internal buffers are entirely flushed, + * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), + * or an error code, which can be tested using ZSTD_isError(). + * + * ZSTD_endStream() instructs to finish a frame. + * It will perform a flush and write frame epilogue. + * The epilogue is required for decoders to consider a frame completed. + * flush() operation is the same, and follows same rules as ZSTD_flushStream(). + * @return : 0 if frame fully completed and fully flushed, + * >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), + * or an error code, which can be tested using ZSTD_isError(). + * + * *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +/*===== Streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush + at least one complete compressed block in all circumstances. */ + +/*-*************************************************************************** + * Streaming decompression - HowTo + * + * A ZSTD_DStream object is required to track streaming operations. + * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. + * ZSTD_DStream objects can be re-used multiple times. + * + * Use ZSTD_initDStream() to start a new decompression operation. + * @return : recommended first input size + * Alternatively, use advanced API to set specific properties. + * + * Use ZSTD_decompressStream() repetitively to consume your input. + * The function will update both `pos` fields. + * If `input.pos < input.size`, some input has not been consumed. + * It's up to the caller to present again remaining data. + * The function tries to flush all data decoded immediately, respecting output buffer size. + * If `output.pos < output.size`, decoder has flushed everything it could. + * But if `output.pos == output.size`, there might be some data left within internal buffers., + * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. + * Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : + * the return value is a suggested next input size (just a hint for better latency) + * that will never request more than the remaining frame size. + * *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +/*===== Streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush + at least one complete block in all circumstances. */ + +#endif /* ZSTD_H_235446 */ + +/**************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/**************************************************************************************** + * Candidate API for promotion to stable status + **************************************************************************************** + * The following symbols and constants form the "staging area" : + * they are considered to join "stable API" by v1.4.0. + * The proposal is written so that it can be made stable "as is", + * though it's still possible to suggest improvements. + * Staging is in fact last chance for changes, + * the API is locked once reaching "stable" status. + * ***************************************************************************************/ + +/* === Constants === */ + +/* all magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START \ + 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1 << ZSTD_BLOCKSIZELOG_MAX) + +/* === query limits === */ + +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ + +/* === frame size === */ + +/*! ZSTD_findFrameCompressedSize() : + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +/*************************************** + * Advanced compression API + ***************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supercedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { + ZSTD_fast = 1, + ZSTD_dfast = 2, + ZSTD_greedy = 3, + ZSTD_lazy = 4, + ZSTD_lazy2 = 5, + ZSTD_btlazy2 = 6, + ZSTD_btopt = 7, + ZSTD_btultra = 8, + ZSTD_btultra2 = 9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters */ + ZSTD_c_compressionLevel = 100, /* Update all compression parameters according to pre-defined cLevel table + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level sets all default values of other compression parameters */ + ZSTD_c_windowLog = + 101, /* Maximum allowed back-reference distance, expressed as power of 2. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such window size at decompression stage if using streaming. */ + ZSTD_c_hashLog = 102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog = 103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless when using "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog = 104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless when using "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch = 105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength = 106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy = 107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching = 160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. */ + ZSTD_c_ldmHashLog = 161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch = 162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog = 163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog = 164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag = + 200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag = 201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag = 202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * They return an error otherwise. */ + ZSTD_c_nbWorkers = + 400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives + * back control to caller, while compression work is performed in parallel, within worker + * threads. (note : a strong exception to this rule is when first invocation of + * ZSTD_compressStream2() sets ZSTD_e_end : in which case, ZSTD_compressStream2() delegates to + * ZSTD_compress2(), which is always a blocking call). More workers improve speed, but also + * increase memory usage. Default value is `0`, aka "single-threaded mode" : no worker is + * spawned, compression is performed inside Caller's thread, all invocations are blocking */ + ZSTD_c_jobSize = 401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of + * active threads. 0 means default, which is dynamically determined based on compression + * parameters. Job size must be a minimum of overlap size, or 1 MB, whichever is largest. The + * minimum size is automatically and transparently enforced */ + ZSTD_c_overlapLog = 402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new + * job. It helps preserve compression ratio, while each job is compressed in parallel. This + * value is enforced only when nbWorkers >= 1. Larger values increase compression ratio, but + * decrease speed. Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no + * overlap; 0:default default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1 = 500, + ZSTD_c_experimentalParam2 = 10, + ZSTD_c_experimentalParam3 = 1000, + ZSTD_c_experimentalParam4 = 1001 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overriden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +/*! ZSTD_CCtx_loadDictionary() : + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : + * Reference a prepared dictionary, to be used for all next compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supercede any compression parameter previously set within CCtx. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This removes any reference to any dictionary too. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using + * ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +typedef enum { + ZSTD_e_continue = + 0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush = 1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving + * compression. */ + ZSTD_e_end = 2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading + * mode) + * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize + * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal + * worker threads, flush whatever is available, and then immediately returns, just indicating that there is some data + * remaining to be flushed. The function nonetheless guarantees forward progress : it will return only after it reads or + * write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function + * delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( + ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); + +/* ============================== */ +/* Advanced decompression API */ +/* ============================== */ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax = + 100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass + * mode. By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_format + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1 = 1000 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_loadDictionary() : + * Create an internal DDict from dict buffer, + * to be used to decompress next frames. + * The dictionary remains valid for all future frames, until explicitly invalidated. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN 6 +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX - 1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT \ + 27 /* by default, the streaming decoder will refuse any frame \ + * requiring larger than (1<= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +/*************************************** + * Memory management + ***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from + * compressionLevel. ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). + * Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CCtx size estimation is only correct for single-threaded compression. */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from + * compressionLevel. ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with + * ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code + * if ZSTD_c_nbWorkers is >= 1. Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize */ +ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like + * ZSTD_createCDict(). ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, + * like ZSTD_createCDict_advanced(). Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream( + void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream( + void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, const void* dict, + size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize, const void* dict, + size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction)(void* opaque, size_t size); +typedef void (*ZSTD_freeFunction)(void* opaque, void* address); +typedef struct { + ZSTD_allocFunction customAlloc; + ZSTD_freeFunction customFree; + void* opaque; +} ZSTD_customMem; +static ZSTD_customMem const ZSTD_defaultCMem = {NULL, NULL, NULL}; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem); + +/*************************************** + * Advanced compression functions + ***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams( + int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component + * `ZSTD_compressionParameters`. All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, + * noDictID=0 */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range */ +ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * both values are optional, select `0` if unknown. */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams( + ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : + * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */ +ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const void* dict, size_t dictSize, ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams); + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + +/* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ +#define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParam_setParameter() : + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParam_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp); + +/*************************************** + * Advanced decompression functions + ***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < + * ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more + * precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive + * decompression. */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced( + ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 + +/*! ZSTD_DCtx_setFormat() : + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos); + +/******************************************************************** + * Advanced streaming functions + * Warning : most of these functions are now redundant with the Advanced API. + * Once Advanced API reaches "stable" status, + * redundant functions will be deprecated, and then at some point removed. + ********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, + unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use + ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + "0" also disables frame content size field. It may be enabled in the future. + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or + dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto + (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw + content) and ZSTD_dlm_byCopy.*/ +ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use + value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and + ZSTD_dlm_byCopy. */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + pledgedSrcSize must be correct. If srcSize is not known at init time, use + value ZSTD_CONTENTSIZE_UNKNOWN. */ + +/*! ZSTD_resetCStream() : + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + +/*===== Advanced Streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, + size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */ +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, + const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */ +ZSTDLIB_API size_t ZSTD_resetDStream( + ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ + +/********************************************************************* + * Buffer-less and synchronous inner streaming functions + * + * This is an advanced API, giving full control over buffer management, for users which need direct control over + *memory. But it's also a complex one, with several restrictions, documented below. Prefer normal streaming API for an + *easier experience. + ********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, + or ZSTD_compressBegin_advanced(), for finer parameter control. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, + see WindowLog). It remembers all previous contiguous blocks, plus one separated memory segment (which can itself + consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init + time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict( + ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, + unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize + must be correct. If srcSize is not known, use macro + ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, + unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_API size_t ZSTD_compressContinue( + ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/*- + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs + false information. As a consequence, check that values remain within valid application range. For example, do not + allocate memory blindly, check that `windowSize` is within expectation. Each application can set its own limits, + depending on local restrictions. For extended interoperability, it is recommended to support `windowSize` of at least 8 + MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can @return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume + more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means + "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameHeader; + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_API size_t ZSTD_getFrameHeader( + ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced( + ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); +ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, + unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == + ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { + ZSTDnit_frameHeader, + ZSTDnit_blockHeader, + ZSTDnit_block, + ZSTDnit_lastBlock, + ZSTDnit_checksum, + ZSTDnit_skippableFrame +} ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +/* ============================ */ +/** Block level API */ +/* ============================ */ + +/*! + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + + copyCCtx() and copyDCtx() can be used too + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, really consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + In which case, nothing is produced into `dst` ! + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTDLIB_API size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, + size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined(__cplusplus) +} +#endif diff --git a/src/lib/compress/zstd_1_3_8/zstd_common.c b/src/lib/compress/zstd_1_3_8/zstd_common.c new file mode 100644 index 0000000000000000000000000000000000000000..3c1ebd61fc603e531abfd36bccda0cc05ef4e409 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_common.c @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* + * Dependencies + ***************************************/ +#include /* malloc, calloc, free */ +#include /* memset */ +#include "error_private.h" +#include "zstd_internal.h" + +/*-**************************************** + * Version + ******************************************/ +unsigned ZSTD_versionNumber(void) +{ + return ZSTD_VERSION_NUMBER; +} + +const char* ZSTD_versionString(void) +{ + return ZSTD_VERSION_STRING; +} + +/*-**************************************** + * ZSTD Error Management + ******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ +/*! ZSTD_isError() : + * tells if a return value is an error code + * symbol is required for external callers */ +unsigned ZSTD_isError(size_t code) +{ + return ERR_isError(code); +} + +/*! ZSTD_getErrorName() : + * provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) +{ + return ERR_getErrorName(code); +} + +/*! ZSTD_getError() : + * convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) +{ + return ERR_getErrorCode(code); +} + +/*! ZSTD_getErrorString() : + * provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) +{ + return ERR_getErrorString(code); +} + +/*=************************************************************** + * Custom allocator + ****************************************************************/ +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return malloc(size); +} + +void* ZSTD_calloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + memset(ptr, 0, size); + return ptr; + } + return calloc(1, size); +} + +void ZSTD_free(void* ptr, ZSTD_customMem customMem) +{ + if (ptr != NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + free(ptr); + } +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_compress.c b/src/lib/compress/zstd_1_3_8/zstd_compress.c new file mode 100644 index 0000000000000000000000000000000000000000..cdaf822f7faacac24c30ca815aef1df52c9d42f4 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_compress.c @@ -0,0 +1,4697 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* + * Dependencies + ***************************************/ +#include /* INT_MAX */ +#include /* memset */ +#include "cpu.h" +#include "mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_compress_internal.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" + +/*-************************************* + * Helper functions + ***************************************/ +size_t ZSTD_compressBound(size_t srcSize) +{ + return ZSTD_COMPRESSBOUND(srcSize); +} + +/*-************************************* + * Context memory management + ***************************************/ +struct ZSTD_CDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictContentSize; + void* workspace; + size_t workspaceSize; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + { + size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init == 0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (0ULL - 1)); + if (!customMem.customAlloc ^ !customMem.customFree) + return NULL; + { + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) + return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)workspace; + if (workspaceSize <= sizeof(ZSTD_CCtx)) + return NULL; /* minimum size */ + if ((size_t)workspace & 7) + return NULL; /* must be 8-aligned */ + memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + cctx->staticSize = workspaceSize; + cctx->workSpace = (void*)(cctx + 1); + cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) + return NULL; + assert(((size_t)cctx->workSpace & (sizeof(void*) - 1)) == 0); /* ensure correct alignment */ + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workSpace; + cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; + { + void* const ptr = cctx->blockState.nextCBlock + 1; + cctx->entropyWorkspace = (U32*)ptr; + } + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_free(cctx->workSpace, cctx->customMem); + cctx->workSpace = NULL; + ZSTD_freeCDict(cctx->cdictLocal); + cctx->cdictLocal = NULL; +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->mtctx = NULL; +#endif +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx == NULL) + return 0; /* support free on NULL */ + if (cctx->staticSize) + return ERROR(memory_allocation); /* not compatible with static CCtx */ + ZSTD_freeCCtxContent(cctx); + ZSTD_free(cctx, cctx->customMem); + return 0; +} + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx == NULL) + return 0; /* support sizeof on NULL */ + return sizeof(*cctx) + cctx->workSpaceSize + ZSTD_sizeof_CDict(cctx->cdictLocal) + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) +{ + return &(ctx->seqStore); +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + memset(&cctxParams, 0, sizeof(cctxParams)); + cctxParams.cParams = cParams; + cctxParams.compressionLevel = + ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(cParams)); + cctxParams.fParams.contentSizeFlag = 1; + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced(ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if (!customMem.customAlloc ^ !customMem.customFree) + return NULL; + params = (ZSTD_CCtx_params*)ZSTD_calloc(sizeof(ZSTD_CCtx_params), customMem); + if (!params) { + return NULL; + } + params->customMem = customMem; + params->compressionLevel = ZSTD_CLEVEL_DEFAULT; + params->fParams.contentSizeFlag = 1; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { + return 0; + } + ZSTD_free(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) +{ + if (!cctxParams) { + return ERROR(GENERIC); + } + memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + if (!cctxParams) { + return ERROR(GENERIC); + } + CHECK_F(ZSTD_checkCParams(params.cParams)); + memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params.cParams; + cctxParams->fParams = params.fParams; + cctxParams->compressionLevel = + ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(params.cParams)); + return 0; +} + +/* ZSTD_assignParamsToCCtxParams() : + * params is presumed valid at this stage */ +static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(ZSTD_CCtx_params cctxParams, ZSTD_parameters params) +{ + ZSTD_CCtx_params ret = cctxParams; + ret.cParams = params.cParams; + ret.fParams = params.fParams; + ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(params.cParams)); + return ret; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = {0, 0, 0}; + + switch (param) { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = + ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = + ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + default: { + ZSTD_bounds const boundError = {ERROR(parameter_unsupported), 0, 0}; + return boundError; + } + } +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) + return 0; + if (value < bounds.lowerBound) + return 0; + if (value > bounds.upperBound) + return 0; + return 1; +} + +#define BOUNDCHECK(cParam, val) \ + { \ + if (!ZSTD_cParam_withinBounds(cParam, val)) { \ + return ERROR(parameter_outOfBound); \ + } \ + } + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch (param) { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + return ERROR(stage_wrong); + } + } + + switch (param) { + case ZSTD_c_format: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_compressionLevel: + if (cctx->cdict) + return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + if (cctx->cdict) + return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_forceMaxWindow: /* Force back-references to remain < windowSize, + * even when referencing into Dictionary content. + * default : 0 when using a CDict, 1 when using a Prefix */ + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_forceAttachDict: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_nbWorkers: + if ((value != 0) && cctx->staticSize) { + return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ + } + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + if (cctx->cdict) + return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + default: + return ERROR(parameter_unsupported); + } +} + +size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%i, %i)", (int)param, value); + switch (param) { + case ZSTD_c_format: + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel: { + int cLevel = value; + if (cLevel > ZSTD_maxCLevel()) + cLevel = ZSTD_maxCLevel(); + if (cLevel < ZSTD_minCLevel()) + cLevel = ZSTD_minCLevel(); + if (cLevel) { /* 0 : does not change current level */ + CCtxParams->compressionLevel = cLevel; + } + if (CCtxParams->compressionLevel >= 0) + return CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = value; + return value; + + case ZSTD_c_minMatch: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength: + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy: + if (value != 0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag: + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value != 0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag: + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag: /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value != 0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow: + CCtxParams->forceWindow = (value != 0); + return CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict: { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_nbWorkers: +#ifndef ZSTD_MULTITHREAD + if (value != 0) + return ERROR(parameter_unsupported); + return 0; +#else + return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); +#endif + + case ZSTD_c_jobSize: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value); +#endif + + case ZSTD_c_overlapLog: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value); +#endif + + case ZSTD_c_rsyncable: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value); +#endif + + case ZSTD_c_enableLongDistanceMatching: + CCtxParams->ldmParams.enableLdm = (value != 0); + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog: + if (value != 0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch: + if (value != 0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog: + if (value != 0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog: + if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) + return ERROR(parameter_outOfBound); + CCtxParams->ldmParams.hashRateLog = value; + return CCtxParams->ldmParams.hashRateLog; + + default: + return ERROR(parameter_unsupported); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParam_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch (param) { + case ZSTD_c_format: + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel: + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog: + *value = CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog: + *value = CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog: + *value = CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog: + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch: + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength: + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy: + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag: + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag: + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag: + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow: + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict: + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_nbWorkers: +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; +#endif + case ZSTD_c_overlapLog: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable: +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableLongDistanceMatching: + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog: + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch: + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog: + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog: + *value = CCtxParams->ldmParams.hashRateLog; + break; + default: + return ERROR(parameter_unsupported); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + if (cctx->cdict) + return ERROR(stage_wrong); + + cctx->requestedParams = *params; + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize + 1; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + if (cctx->staticSize) + return ERROR(memory_allocation); /* no malloc for static CCtx */ + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */ + if (dict == NULL || dictSize == 0) { /* no dictionary mode */ + cctx->cdictLocal = NULL; + cctx->cdict = NULL; + } else { + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne - 1, dictSize); + cctx->cdictLocal = + ZSTD_createCDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, cParams, cctx->customMem); + cctx->cdict = cctx->cdictLocal; + if (cctx->cdictLocal == NULL) + return ERROR(memory_allocation); + } + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced(cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced(cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + cctx->cdict = cdict; + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* exclusive */ + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + cctx->cdict = NULL; /* prefix discards any prior cdict */ + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ((reset == ZSTD_reset_session_only) || (reset == ZSTD_reset_session_and_parameters)) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ((reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters)) { + if (cctx->streamStage != zcss_init) + return ERROR(stage_wrong); + cctx->cdict = NULL; + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength, cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +#define CLAMP_TYPE(cParam, val, type) \ + { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)val < bounds.lowerBound) \ + val = (type)bounds.lowerBound; \ + else if ((int)val > bounds.upperBound) \ + val = (type)bounds.upperBound; \ + } +#define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength, cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy, cParams.strategy, ZSTD_strategy); + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_adjustCParams_internal() : + optimize `cPar` for a given input (`srcSize` and `dictSize`). + mostly downsizing to reduce memory consumption and initialization latency. + Both `srcSize` and `dictSize` are optional (use 0 if unknown). + Note : cPar is assumed validated. Use ZSTD_checkCParams() to ensure this condition. */ +static ZSTD_compressionParameters ZSTD_adjustCParams_internal( + ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + static const U64 minSrcSize = 513; /* (1<<9) + 1 */ + static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX - 1); + assert(ZSTD_checkCParams(cPar) == 0); + + if (dictSize && (srcSize + 1 < 2) /* srcSize unknown */) + srcSize = minSrcSize; /* presumed small when there is a dictionary */ + else if (srcSize == 0) + srcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* 0 == unknown : presumed large */ + + /* resize windowLog if input is small enough, to use less memory */ + if ((srcSize < maxWindowResize) && (dictSize < maxWindowResize)) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : ZSTD_highbit32(tSize - 1) + 1; + if (cPar.windowLog > srcLog) + cPar.windowLog = srcLog; + } + if (cPar.hashLog > cPar.windowLog + 1) + cPar.hashLog = cPar.windowLog + 1; + { + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cycleLog > cPar.windowLog) + cPar.chainLog -= (cycleLog - cPar.windowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + + return cPar; +} + +ZSTD_compressionParameters ZSTD_adjustCParams( + ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + if (CCtxParams->ldmParams.enableLdm) + cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + if (CCtxParams->cParams.windowLog) + cParams.windowLog = CCtxParams->cParams.windowLog; + if (CCtxParams->cParams.hashLog) + cParams.hashLog = CCtxParams->cParams.hashLog; + if (CCtxParams->cParams.chainLog) + cParams.chainLog = CCtxParams->cParams.chainLog; + if (CCtxParams->cParams.searchLog) + cParams.searchLog = CCtxParams->cParams.searchLog; + if (CCtxParams->cParams.minMatch) + cParams.minMatch = CCtxParams->cParams.minMatch; + if (CCtxParams->cParams.targetLength) + cParams.targetLength = CCtxParams->cParams.targetLength; + if (CCtxParams->cParams.strategy) + cParams.strategy = CCtxParams->cParams.strategy; + assert(!ZSTD_checkCParams(cParams)); + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize); +} + +static size_t ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, const U32 forCCtx) +{ + size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch == 3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + size_t const optPotentialSpace = ((MaxML + 1) + (MaxLL + 1) + (MaxOff + 1) + (1 << Litbits)) * sizeof(U32) + + (ZSTD_OPT_NUM + 1) * (sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + /* Estimate CCtx size is supported for single-threaded compression only. */ + if (params->nbWorkers > 0) { + return ERROR(GENERIC); + } + { + ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, 0, 0); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + U32 const divider = (cParams.minMatch == 3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11 * maxNbSeq; + size_t const entropySpace = HUF_WORKSPACE_SIZE; + size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); + size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); + + size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + matchStateSize + ldmSpace + ldmSeqSpace; + + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); + DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); + return sizeof(ZSTD_CCtx) + neededSpace; + } +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCCtxSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level = MIN(compressionLevel, 1); level <= compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) + memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + if (params->nbWorkers > 0) { + return ERROR(GENERIC); + } + { + size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; + size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; + size_t const streamingSize = inBuffSize + outBuffSize; + + return CCtxSize + streamingSize; + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level = MIN(compressionLevel, 1); level <= compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) + memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { + ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : cctx->inBuffPos - cctx->inToCompress; + if (buffered) + assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; + } +} + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); + } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which + case, report how many bytes are left to be flushed within output buffer */ +} + +static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) +{ + return (cParams1.hashLog == cParams2.hashLog) & (cParams1.chainLog == cParams2.chainLog) & + (cParams1.strategy == cParams2.strategy) /* opt parser space */ + & ((cParams1.minMatch == 3) == (cParams2.minMatch == 3)); /* hashlog3 space */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +/** The parameters are equivalent if ldm is not enabled in both sets or + * all the parameters are equivalent. */ +static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, ldmParams_t ldmParams2) +{ + return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || + (ldmParams1.enableLdm == ldmParams2.enableLdm && ldmParams1.hashLog == ldmParams2.hashLog && + ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && + ldmParams1.minMatchLength == ldmParams2.minMatchLength && + ldmParams1.hashRateLog == ldmParams2.hashRateLog); +} + +typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; + +/* ZSTD_sufficientBuff() : + * check internal buffers exist for streaming if buffPol == ZSTDb_buffered . + * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */ +static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, size_t maxNbLit1, ZSTD_buffered_policy_e buffPol2, + ZSTD_compressionParameters cParams2, U64 pledgedSrcSize) +{ + size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); + size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); + size_t const maxNbSeq2 = blockSize2 / ((cParams2.minMatch == 3) ? 3 : 4); + size_t const maxNbLit2 = blockSize2; + size_t const neededBufferSize2 = (buffPol2 == ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; + DEBUGLOG( + 4, "ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u", (U32)neededBufferSize2, (U32)bufferSize1); + DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbSeq2=%u <= maxNbSeq1=%u", (U32)maxNbSeq2, (U32)maxNbSeq1); + DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbLit2=%u <= maxNbLit1=%u", (U32)maxNbLit2, (U32)maxNbLit1); + return (maxNbLit2 <= maxNbLit1) & (maxNbSeq2 <= maxNbSeq1) & (neededBufferSize2 <= bufferSize1); +} + +/** Equivalence for resetCCtx purposes */ +static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, ZSTD_CCtx_params params2, size_t buffSize1, size_t maxNbSeq1, + size_t maxNbLit1, ZSTD_buffered_policy_e buffPol2, U64 pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); + if (!ZSTD_equivalentCParams(params1.cParams, params2.cParams)) { + DEBUGLOG(4, "ZSTD_equivalentCParams() == 0"); + return 0; + } + if (!ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams)) { + DEBUGLOG(4, "ZSTD_equivalentLdmParams() == 0"); + return 0; + } + if (!ZSTD_sufficientBuff(buffSize1, maxNbSeq1, maxNbLit1, buffPol2, params2.cParams, pledgedSrcSize)) { + DEBUGLOG(4, "ZSTD_sufficientBuff() == 0"); + return 0; + } + return 1; +} + +static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->nextToUpdate3 = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/*! ZSTD_continueCCtx() : + * reuse CCtx without reset (note : requires no dictionary) */ +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) +{ + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); + + cctx->blockSize = + blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ + cctx->appliedParams = params; + cctx->blockState.matchState.cParams = params.cParams; + cctx->pledgedSrcSizePlusOne = pledgedSrcSize + 1; + cctx->consumedSrcSize = 0; + cctx->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + cctx->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG( + 4, "pledged content size : %u ; flag : %u", (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); + cctx->stage = ZSTDcs_init; + cctx->dictID = 0; + if (params.ldmParams.enableLdm) + ZSTD_window_clear(&cctx->ldmState.window); + ZSTD_referenceExternalSequences(cctx, NULL, 0); + ZSTD_invalidateMatchState(&cctx->blockState.matchState); + ZSTD_reset_compressedBlockState(cctx->blockState.prevCBlock); + XXH64_reset(&cctx->xxhState, 0); + return 0; +} + +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; + +static void* ZSTD_reset_matchState(ZSTD_matchState_t* ms, void* ptr, const ZSTD_compressionParameters* cParams, + ZSTD_compResetPolicy_e const crp, U32 const forCCtx) +{ + size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch == 3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + assert(((size_t)ptr & 3) == 0); + + ms->hashLog3 = hashLog3; + memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_invalidateMatchState(ms); + + /* opt parser space */ + if (forCCtx && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ptr; + ms->opt.litLengthFreq = ms->opt.litFreq + (1 << Litbits); + ms->opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL + 1); + ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML + 1); + ptr = ms->opt.offCodeFreq + (MaxOff + 1); + ms->opt.matchTable = (ZSTD_match_t*)ptr; + ptr = ms->opt.matchTable + ZSTD_OPT_NUM + 1; + ms->opt.priceTable = (ZSTD_optimal_t*)ptr; + ptr = ms->opt.priceTable + ZSTD_OPT_NUM + 1; + } + + /* table Space */ + DEBUGLOG(4, "reset table : %u", crp != ZSTDcrp_noMemset); + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + if (crp != ZSTDcrp_noMemset) + memset(ptr, 0, tableSpace); /* reset tables only */ + ms->hashTable = (U32*)(ptr); + ms->chainTable = ms->hashTable + hSize; + ms->hashTable3 = ms->chainTable + chainSize; + ptr = ms->hashTable3 + h3Size; + + ms->cParams = *cParams; + + assert(((size_t)ptr & 3) == 0); + return ptr; +} + +#define ZSTD_WORKSPACETOOLARGE_FACTOR \ + 3 /* define "workspace is too large" as this number of times larger than needed \ + */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION \ + 128 /* when workspace is continuously too large \ + * during at least this number of times, \ + * context's memory usage is considered wasteful, \ + * because it's sized to handle a worst case scenario which rarely happens. \ + * In which case, resize it down to free some memory */ + +/*! ZSTD_resetCCtx_internal() : + note : `params` are assumed fully validated at this stage */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_CCtx_params params, U64 pledgedSrcSize, + ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", (U32)pledgedSrcSize, params.cParams.windowLog); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + + if (crp == ZSTDcrp_continue) { + if (ZSTD_equivalentParams(zc->appliedParams, + params, + zc->inBuffSize, + zc->seqStore.maxNbSeq, + zc->seqStore.maxNbLit, + zbuff, + pledgedSrcSize)) { + DEBUGLOG(4, + "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", + zc->appliedParams.cParams.windowLog, + zc->blockSize); + zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ + if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) + return ZSTD_continueCCtx(zc, params, pledgedSrcSize); + } + } + DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); + + if (params.ldmParams.enableLdm) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + zc->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength); + } + + { + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + U32 const divider = (params.cParams.minMatch == 3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11 * maxNbSeq; + size_t const buffOutSize = (zbuff == ZSTDb_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered) ? windowSize + blockSize : 0; + size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + void* ptr; /* used to partition workSpace */ + + /* Check if workSpace is large enough, alloc a new one if needed */ + { + size_t const entropySpace = HUF_WORKSPACE_SIZE; + size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const bufferSpace = buffInSize + buffOutSize; + size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); + size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); + + size_t const neededSpace = + entropySpace + blockStateSpace + ldmSpace + ldmSeqSpace + matchStateSize + tokenSpace + bufferSpace; + + int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; + int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; + int const workSpaceWasteful = + workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); + zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration + 1 : 0; + + DEBUGLOG(4, + "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", + neededSpace >> 10, + matchStateSize >> 10, + bufferSpace >> 10); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (workSpaceTooSmall || workSpaceWasteful) { + DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", zc->workSpaceSize >> 10, neededSpace >> 10); + /* static cctx : no resize, error out */ + if (zc->staticSize) + return ERROR(memory_allocation); + + zc->workSpaceSize = 0; + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); + if (zc->workSpace == NULL) + return ERROR(memory_allocation); + zc->workSpaceSize = neededSpace; + zc->workSpaceOversizedDuration = 0; + + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ + assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; + zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; + ptr = zc->blockState.nextCBlock + 1; + zc->entropyWorkspace = (U32*)ptr; + } + } + + /* init params */ + zc->appliedParams = params; + zc->blockState.matchState.cParams = params.cParams; + zc->pledgedSrcSizePlusOne = pledgedSrcSize + 1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, + "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, + zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32; + + /* ldm hash table */ + /* initialize bucketOffsets table later for pointer alignment */ + if (params.ldmParams.enableLdm) { + size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + zc->ldmState.hashTable = (ldmEntry_t*)ptr; + ptr = zc->ldmState.hashTable + ldmHSize; + zc->ldmSequences = (rawSeq*)ptr; + ptr = zc->ldmSequences + maxNbLdmSeq; + zc->maxNbLdmSequences = maxNbLdmSeq; + + memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); + } + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + + ptr = ZSTD_reset_matchState(&zc->blockState.matchState, ptr, ¶ms.cParams, crp, /* forCCtx */ 1); + + /* sequences storage */ + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; + zc->seqStore.llCode = (BYTE*)ptr; + zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; + zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; + zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.maxNbLit = blockSize; + ptr = zc->seqStore.litStart + blockSize + WILDCOPY_OVERLENGTH; + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); + memset(ptr, 0, ldmBucketSize); + zc->ldmState.bucketOffsets = (BYTE*)ptr; + ptr = zc->ldmState.bucketOffsets + ldmBucketSize; + ZSTD_window_clear(&zc->ldmState.window); + } + ZSTD_referenceExternalSequences(zc, NULL, 0); + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ptr; + zc->outBuffSize = buffOutSize; + zc->outBuff = zc->inBuff + buffInSize; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; i++) + cctx->blockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX + 1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + return (pledgedSrcSize <= cutoff || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN || + params.attachDictPref == ZSTD_dictForceAttach) && + params.attachDictPref != ZSTD_dictForceCopy && !params.forceWindow; /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t ZSTD_resetCCtx_byAttachingCDict( + ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) +{ + { + const ZSTD_compressionParameters* cdict_cParams = &cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); + params.cParams.windowLog = windowLog; + ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + } + + { + const U32 cdictEnd = (U32)(cdict->matchState.window.nextSrc - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } + } + + cctx->dictID = cdict->dictID; + + /* copy block state */ + memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static size_t ZSTD_resetCCtx_byCopyingCDict( + ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters* cdict_cParams = &cdict->matchState.cParams; + + DEBUGLOG(4, "copying dictionary into context"); + + { + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_noMemset, zbuff); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + /* copy tables */ + { + size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + size_t const tableSpace = (chainSize + hSize) * sizeof(U32); + assert((U32*)cctx->blockState.matchState.chainTable == + (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ + assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); + assert((U32*)cdict->matchState.chainTable == + (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ + assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); + memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + tableSpace); /* presumes all tables follow each other */ + } + + /* Zero the hashTable3, since the cdict never fills it */ + { + size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3; + assert(cdict->matchState.hashLog3 == 0); + memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + /* copy dictionary offsets */ + { + ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->nextToUpdate3 = srcMatchState->nextToUpdate3; + dstMatchState->loadedDictEnd = srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + + /* copy block state */ + memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict( + ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, ZSTD_frameParameters fParams, + U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + if (srcCCtx->stage != ZSTDcs_init) + return ERROR(stage_wrong); + + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { + ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + params.fParams = fParams; + ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + /* copy tables */ + { + size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) + ? 0 + : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + size_t const h3Size = (size_t)1 << srcCCtx->blockState.matchState.hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + assert((U32*)dstCCtx->blockState.matchState.chainTable == + (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ + assert( + (U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); + memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + tableSpace); /* presumes all tables follow each other */ + } + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->nextToUpdate3 = srcMatchState->nextToUpdate3; + dstMatchState->loadedDictEnd = srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + + /* copy block state */ + memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". + * @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = {1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/}; + ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize > 0); + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered == 1); + if (pledgedSrcSize == 0) + pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff); +} + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void ZSTD_reduceTable_internal( + U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + assert((size & (ZSTD_ROWSIZE - 1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U << 31)); /* can be casted to int */ + for (rowNb = 0; rowNb < nbRows; rowNb++) { + int column; + for (column = 0; column < ZSTD_ROWSIZE; column++) { + if (preserveMark) { + U32 const adder = (table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) ? reducerValue : 0; + table[cellNb] += adder; + } + if (table[cellNb] < reducerValue) + table[cellNb] = 0; + else + table[cellNb] -= reducerValue; + cellNb++; + } + } +} + +static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) +{ + ZSTD_reduceTable_internal(table, size, reducerValue, 0); +} + +static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) +{ + ZSTD_reduceTable_internal(table, size, reducerValue, 1); +} + +/*! ZSTD_reduceIndex() : + * rescale all indexes to avoid future overflow (indexes are U32) */ +static void ZSTD_reduceIndex(ZSTD_CCtx* zc, const U32 reducerValue) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + { + U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (zc->appliedParams.cParams.strategy != ZSTD_fast) { + U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog; + if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + +/*-******************************************************* + * Block entropic compression + *********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +static size_t ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw) << 1) + (U32)(srcSize << 3); + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) + return ERROR(dstSize_tooSmall); + MEM_writeLE24(dst, cBlockHeader24); + memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +static size_t ZSTD_noCompressLiterals(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); + + if (srcSize + flSize > dstCapacity) + return ERROR(dstSize_tooSmall); + + switch (flSize) { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize << 3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1 << 2) + (srcSize << 4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3 << 2) + (srcSize << 4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +static size_t ZSTD_compressRleLiteralsBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize > 31) + (srcSize > 4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch (flSize) { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize << 3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1 << 2) + (srcSize << 4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3 << 2) + (srcSize << 4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + return flSize + 1; +} + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat >= ZSTD_btultra) ? (U32)(strat)-1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + return (srcSize >> minlog) + 2; +} + +static size_t ZSTD_compressLiterals(ZSTD_hufCTables_t const* prevHuf, ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, void* workspace, size_t wkspSize, const int bmi2) +{ + size_t const minGain = ZSTD_minGain(srcSize, strategy); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5, "ZSTD_compressLiterals (disableLiteralCompression=%i)", disableLiteralCompression); + + /* Prepare nextEntropy assuming reusing the existing table */ + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* small ? don't even attempt compression (speed opt) */ +#define COMPRESS_LITERALS_SIZE_MIN 63 + { + size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + if (dstCapacity < lhSize + 1) + return ERROR(dstSize_tooSmall); /* not enough space for compression */ + { + HUF_repeat repeat = prevHuf->repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) + singleStream = 1; + cLitSize = singleStream ? HUF_compress1X_repeat(ostart + lhSize, + dstCapacity - lhSize, + src, + srcSize, + 255, + 11, + workspace, + wkspSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, + preferRepeat, + bmi2) + : HUF_compress4X_repeat(ostart + lhSize, + dstCapacity - lhSize, + src, + srcSize, + 255, + 11, + workspace, + wkspSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, + preferRepeat, + bmi2); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + hType = set_repeat; + } + } + + if ((cLitSize == 0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize == 1) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch (lhSize) { + case 3: /* 2 - 2 - 10 - 10 */ + { + U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { + U32 const lhc = hType + (2 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { + U32 const lhc = hType + (3 << 2) + ((U32)srcSize << 4) + ((U32)cLitSize << 22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + return lhSize + cLitSize; +} + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u = 0; u < nbSeq; u++) { + U32 const llv = sequences[u].litLength; + U32 const mlv = sequences[u].matchLength; + llCodeTable[u] = (BYTE)ZSTD_LLcode(llv); + ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); + mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv); + } + if (seqStorePtr->longLengthID == 1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthID == 2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabiltyLog256[256] = { + 0, + 2048, + 1792, + 1642, + 1536, + 1453, + 1386, + 1329, + 1280, + 1236, + 1197, + 1162, + 1130, + 1100, + 1073, + 1047, + 1024, + 1001, + 980, + 960, + 941, + 923, + 906, + 889, + 874, + 859, + 844, + 830, + 817, + 804, + 791, + 779, + 768, + 756, + 745, + 734, + 724, + 714, + 704, + 694, + 685, + 676, + 667, + 658, + 650, + 642, + 633, + 626, + 618, + 610, + 603, + 595, + 588, + 581, + 574, + 567, + 561, + 554, + 548, + 542, + 535, + 529, + 523, + 517, + 512, + 506, + 500, + 495, + 489, + 484, + 478, + 473, + 468, + 463, + 458, + 453, + 448, + 443, + 438, + 434, + 429, + 424, + 420, + 415, + 411, + 407, + 402, + 398, + 394, + 390, + 386, + 382, + 377, + 373, + 370, + 366, + 362, + 358, + 354, + 350, + 347, + 343, + 339, + 336, + 332, + 329, + 325, + 322, + 318, + 315, + 311, + 308, + 305, + 302, + 298, + 295, + 292, + 289, + 286, + 282, + 279, + 276, + 273, + 270, + 267, + 264, + 261, + 258, + 256, + 253, + 250, + 247, + 244, + 241, + 239, + 236, + 233, + 230, + 228, + 225, + 222, + 220, + 217, + 215, + 212, + 209, + 207, + 204, + 202, + 199, + 197, + 194, + 192, + 190, + 187, + 185, + 182, + 180, + 178, + 175, + 173, + 171, + 168, + 166, + 164, + 162, + 159, + 157, + 155, + 153, + 151, + 149, + 146, + 144, + 142, + 140, + 138, + 136, + 134, + 132, + 130, + 128, + 126, + 123, + 121, + 119, + 117, + 115, + 114, + 112, + 110, + 108, + 106, + 104, + 102, + 100, + 98, + 96, + 94, + 93, + 91, + 89, + 87, + 85, + 83, + 82, + 80, + 78, + 76, + 74, + 73, + 71, + 69, + 67, + 66, + 64, + 62, + 61, + 59, + 57, + 55, + 54, + 52, + 50, + 49, + 47, + 46, + 44, + 42, + 41, + 39, + 37, + 36, + 34, + 33, + 31, + 30, + 28, + 26, + 25, + 23, + 22, + 20, + 19, + 17, + 16, + 14, + 13, + 11, + 10, + 8, + 7, + 5, + 4, + 2, + 1, +}; + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabiltyLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabiltyLog256[norm256]; + } + return cost >> 8; +} + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) +{ + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +static size_t ZSTD_fseBitCost(FSE_CTable const* ctable, unsigned const* count, unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +typedef enum { ZSTD_defaultDisallowed = 0, ZSTD_defaultAllowed = 1 } ZSTD_defaultPolicy_e; + +MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType(FSE_repeat* repeatMode, unsigned const* count, + unsigned const max, size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = + (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ((*repeatMode == FSE_repeat_valid) && (nbSeq < staticFse_nbSeq_max)) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ((nbSeq < dynamicFse_nbSeq_min) || (mostFrequent < (nbSeq >> (defaultNormLog - 1)))) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = + isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, + "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, + (unsigned)repeatCost, + (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, FSE_CTable* nextCTable, U32 FSELog, + symbolEncodingType_e type, unsigned* count, U32 max, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, + U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, void* workspace, + size_t workspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max)); + if (dstCapacity == 0) + return ERROR(dstSize_tooSmall); + *op = codeTable[0]; + return 1; + case set_repeat: + memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + CHECK_F(FSE_buildCTable_wksp( + nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be + pre-calculated */ + return 0; + case set_compressed: { + S16 norm[MaxSeq + 1]; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq - 1]] > 1) { + count[codeTable[nbSeq - 1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + { + size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) + return NCountSize; + CHECK_F(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + return NCountSize; + } + } + default: + return assert(0), ERROR(GENERIC); + } +} + +FORCE_INLINE_TEMPLATE size_t ZSTD_encodeSequences_body(void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, + BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, + size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ + DEBUGLOG(6, + "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq - 1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq - 1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq - 1]); + BIT_addBits(&blockStream, sequences[nbSeq - 1].litLength, LL_bits[llCodeTable[nbSeq - 1]]); + if (MEM_32bits()) + BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq - 1].matchLength, ML_bits[mlCodeTable[nbSeq - 1]]); + if (MEM_32bits()) + BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq - 1]; + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq - 1].offset >> extraBits, ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq - 1].offset, ofCodeTable[nbSeq - 1]); + } + BIT_flushBits(&blockStream); + + { + size_t n; + for (n = nbSeq - 2; n < nbSeq; n--) { /* intentional underflow */ + BYTE const llCode = llCodeTable[n]; + BYTE const ofCode = ofCodeTable[n]; + BYTE const mlCode = mlCodeTable[n]; + U32 const llBits = LL_bits[llCode]; + U32 const ofBits = ofCode; + U32 const mlBits = ML_bits[mlCode]; + DEBUGLOG(6, + "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", + (unsigned)sequences[n].litLength, + (unsigned)sequences[n].matchLength + MINMATCH, + (unsigned)sequences[n].offset); + /* 32b*/ /* 64b*/ + /* (7)*/ /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ + FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ + if (MEM_32bits()) + BIT_flushBits(&blockStream); /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ + if (MEM_32bits() || (ofBits + mlBits + llBits >= 64 - 7 - (LLFSELog + MLFSELog + OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits + mlBits) > 24)) + BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits + mlBits + llBits > 56)) + BIT_flushBits(&blockStream); + if (longOffsets) { + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN - 1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } + } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { + size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize == 0) + return ERROR(dstSize_tooSmall); /* not enough space */ + return streamSize; + } +} + +static size_t ZSTD_encodeSequences_default(void* dst, size_t dstCapacity, FSE_CTable const* CTable_MatchLength, + BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, + dstCapacity, + CTable_MatchLength, + mlCodeTable, + CTable_OffsetBits, + ofCodeTable, + CTable_LitLength, + llCodeTable, + sequences, + nbSeq, + longOffsets); +} + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t ZSTD_encodeSequences_bmi2(void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, + BYTE const* ofCodeTable, FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, + size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, + dstCapacity, + CTable_MatchLength, + mlCodeTable, + CTable_OffsetBits, + ofCodeTable, + CTable_LitLength, + llCodeTable, + sequences, + nbSeq, + longOffsets); +} + +#endif + +static size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity, FSE_CTable const* CTable_MatchLength, + BYTE const* mlCodeTable, FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, size_t nbSeq, int longOffsets, + int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, + dstCapacity, + CTable_MatchLength, + mlCodeTable, + CTable_OffsetBits, + ofCodeTable, + CTable_LitLength, + llCodeTable, + sequences, + nbSeq, + longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, + dstCapacity, + CTable_MatchLength, + mlCodeTable, + CTable_OffsetBits, + ofCodeTable, + CTable_LitLength, + llCodeTable, + sequences, + nbSeq, + longOffsets); +} + +/* ZSTD_compressSequences_internal(): + * actually compresses both literals and sequences */ +MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, + void* workspace, size_t wkspSize, const int bmi2) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned count[MaxSeq + 1]; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* seqHead; + BYTE* lastNCount = NULL; + + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1 << MAX(MLFSELog, LLFSELog))); + DEBUGLOG(5, "ZSTD_compressSequences_internal"); + + /* Compress literals */ + { + const BYTE* const literals = seqStorePtr->litStart; + size_t const litSize = seqStorePtr->lit - literals; + int const disableLiteralCompression = + (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + size_t const cSize = ZSTD_compressLiterals(&prevEntropy->huf, + &nextEntropy->huf, + cctxParams->cParams.strategy, + disableLiteralCompression, + op, + dstCapacity, + literals, + litSize, + workspace, + wkspSize, + bmi2); + if (ZSTD_isError(cSize)) + return cSize; + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + if ((oend - op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/) + return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq >> 8) + 0x80), op[1] = (BYTE)nbSeq, op += 2; + else + op[0] = 0xFF, MEM_writeLE16(op + 1, (U16)(nbSeq - LONGNBSEQ)), op += 3; + if (nbSeq == 0) { + /* Copy the old tables over as if we repeated them */ + memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + /* build CTable for Literal Lengths */ + { + unsigned max = MaxLL; + size_t const mostFrequent = + HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, + count, + max, + mostFrequent, + nbSeq, + LLFSELog, + prevEntropy->fse.litlengthCTable, + LL_defaultNorm, + LL_defaultNormLog, + ZSTD_defaultAllowed, + strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(LLtype < set_compressed && + nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { + size_t const countSize = ZSTD_buildCTable(op, + oend - op, + CTable_LitLength, + LLFSELog, + (symbolEncodingType_e)LLtype, + count, + max, + llCodeTable, + nbSeq, + LL_defaultNorm, + LL_defaultNormLog, + MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + workspace, + wkspSize); + if (ZSTD_isError(countSize)) + return countSize; + if (LLtype == set_compressed) + lastNCount = op; + op += countSize; + } + } + /* build CTable for Offsets */ + { + unsigned max = MaxOff; + size_t const mostFrequent = + HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, + count, + max, + mostFrequent, + nbSeq, + OffFSELog, + prevEntropy->fse.offcodeCTable, + OF_defaultNorm, + OF_defaultNormLog, + defaultPolicy, + strategy); + assert(!( + Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { + size_t const countSize = ZSTD_buildCTable(op, + oend - op, + CTable_OffsetBits, + OffFSELog, + (symbolEncodingType_e)Offtype, + count, + max, + ofCodeTable, + nbSeq, + OF_defaultNorm, + OF_defaultNormLog, + DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + workspace, + wkspSize); + if (ZSTD_isError(countSize)) + return countSize; + if (Offtype == set_compressed) + lastNCount = op; + op += countSize; + } + } + /* build CTable for MatchLengths */ + { + unsigned max = MaxML; + size_t const mostFrequent = + HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend - op)); + nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, + count, + max, + mostFrequent, + nbSeq, + MLFSELog, + prevEntropy->fse.matchlengthCTable, + ML_defaultNorm, + ML_defaultNormLog, + ZSTD_defaultAllowed, + strategy); + assert(!(MLtype < set_compressed && + nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { + size_t const countSize = ZSTD_buildCTable(op, + oend - op, + CTable_MatchLength, + MLFSELog, + (symbolEncodingType_e)MLtype, + count, + max, + mlCodeTable, + nbSeq, + ML_defaultNorm, + ML_defaultNormLog, + MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + workspace, + wkspSize); + if (ZSTD_isError(countSize)) + return countSize; + if (MLtype == set_compressed) + lastNCount = op; + op += countSize; + } + } + + *seqHead = (BYTE)((LLtype << 6) + (Offtype << 4) + (MLtype << 2)); + + { + size_t const bitstreamSize = ZSTD_encodeSequences(op, + oend - op, + CTable_MatchLength, + mlCodeTable, + CTable_OffsetBits, + ofCodeTable, + CTable_LitLength, + llCodeTable, + sequences, + nbSeq, + longOffsets, + bmi2); + if (ZSTD_isError(bitstreamSize)) + return bitstreamSize; + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() recieves a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastNCount && (op - lastNCount) < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(op - lastNCount == 3); + DEBUGLOG(5, + "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return op - ostart; +} + +MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, + size_t srcSize, void* workspace, size_t wkspSize, int bmi2) +{ + size_t const cSize = ZSTD_compressSequences_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, workspace, wkspSize, bmi2); + if (cSize == 0) + return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) + return 0; /* block not compressed */ + if (ZSTD_isError(cSize)) + return cSize; + + /* Check compressibility */ + { + size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) + return 0; /* block not compressed */ + } + + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[3][ZSTD_STRATEGY_MAX + 1] = { + {ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2}, + {ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, + ZSTD_compressBlock_btultra_extDict}, + {ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_compressBlock_doubleFast_dictMatchState, + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState}}; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, const BYTE* anchor, size_t lastLLSize) +{ + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + size_t cSize; + DEBUGLOG(5, + "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, + (unsigned)ms->window.dictLimit, + (unsigned)ms->nextToUpdate); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + + if (srcSize < MIN_CBLOCK_SIZE + ZSTD_blockHeaderSize + 1) { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + cSize = 0; + goto out; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + ms->opt.symbolCosts = + &zc->blockState.prevCBlock->entropy; /* required for optimal parser to read stats from dictionary */ + + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart - base); + if (sizeof(ptrdiff_t) == 8) + assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (current > ms->nextToUpdate + 384) + ms->nextToUpdate = current - MIN(192, (U32)(current - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { + ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(!zc->appliedParams.ldmParams.enableLdm); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm) { + rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0}; + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + CHECK_F(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams, src, srcSize)); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else { /* not long range mode */ + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { + const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } + } + + /* encode sequences and literals */ + cSize = ZSTD_compressSequences(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, + dstCapacity, + srcSize, + zc->entropyWorkspace, + HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + +out: + if (!ZSTD_isError(cSize) && cSize != 0) { + /* confirm repcodes and entropy tables when emitting a compressed block */ + ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; + zc->blockState.prevCBlock = zc->blockState.nextCBlock; + zc->blockState.nextCBlock = tmp; + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/*! ZSTD_compress_frameChunk() : + * Compress a chunk of data into one or multiple blocks. + * All blocks will be terminated, all input will be consumed. + * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. + * Frame is supposed already started (header already produced) + * @return : compressed size, or an error code + */ +static size_t ZSTD_compress_frameChunk( + ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + assert(cctx->appliedParams.cParams.windowLog <= 31); + + DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) + return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (remaining < blockSize) + blockSize = remaining; + + if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) { + U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(cctx, correction); + if (ms->nextToUpdate < correction) + ms->nextToUpdate = 0; + else + ms->nextToUpdate -= correction; + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } + ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + if (ms->nextToUpdate < ms->window.lowLimit) + ms->nextToUpdate = ms->window.lowLimit; + + { + size_t cSize = ZSTD_compressBlock_internal( + cctx, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, ip, blockSize); + if (ZSTD_isError(cSize)) + return cSize; + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + if (ZSTD_isError(cSize)) + return cSize; + } else { + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed) << 1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader24); + cSize += ZSTD_blockHeaderSize; + } + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", (unsigned)cSize); + } + } + + if (lastFrameChunk && (op > ostart)) + cctx->stage = ZSTDcs_ending; + return op - ostart; +} + +static size_t ZSTD_writeFrameHeader( + void* dst, size_t dstCapacity, ZSTD_CCtx_params params, U64 pledgedSrcSize, U32 dictID) +{ + BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID > 0) + (dictID >= 256) + (dictID >= 65536); /* 0-3 */ + U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params.fParams.checksumFlag > 0; + U32 const windowSize = (U32)1 << params.cParams.windowLog; + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag + ? (pledgedSrcSize >= 256) + (pledgedSrcSize >= 65536 + 256) + (pledgedSrcSize >= 0xFFFFFFFFU) + : 0; /* 0-3 */ + BYTE const frameHeaderDecriptionByte = + (BYTE)(dictIDSizeCode + (checksumFlag << 2) + (singleSegment << 5) + (fcsCode << 6)); + size_t pos = 0; + + assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + if (dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX) + return ERROR(dstSize_tooSmall); + DEBUGLOG(4, + "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params.fParams.noDictIDFlag, + (unsigned)dictID, + (unsigned)dictIDSizeCode); + + if (params.format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDecriptionByte; + if (!singleSegment) + op[pos++] = windowLogByte; + switch (dictIDSizeCode) { + default: + assert(0); /* impossible */ + case 0: + break; + case 1: + op[pos] = (BYTE)(dictID); + pos++; + break; + case 2: + MEM_writeLE16(op + pos, (U16)dictID); + pos += 2; + break; + case 3: + MEM_writeLE32(op + pos, dictID); + pos += 4; + break; + } + switch (fcsCode) { + default: + assert(0); /* impossible */ + case 0: + if (singleSegment) + op[pos++] = (BYTE)(pledgedSrcSize); + break; + case 1: + MEM_writeLE16(op + pos, (U16)(pledgedSrcSize - 256)); + pos += 2; + break; + case 2: + MEM_writeLE32(op + pos, (U32)(pledgedSrcSize)); + pos += 4; + break; + case 3: + MEM_writeLE64(op + pos, (U64)(pledgedSrcSize)); + pos += 8; + break; + } + return pos; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapcity` is too small (stage != ZSTDcs_init) + return ERROR(stage_wrong); + if (cctx->appliedParams.ldmParams.enableLdm) + return ERROR(parameter_unsupported); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + return 0; +} + +static size_t ZSTD_compressContinue_internal( + ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", cctx->stage, (unsigned)srcSize); + if (cctx->stage == ZSTDcs_created) + return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ + + if (frame && (cctx->stage == ZSTDcs_init)) { + fhSize = + ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->pledgedSrcSizePlusOne - 1, cctx->dictID); + if (ZSTD_isError(fhSize)) + return fhSize; + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) + return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize)) { + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + } + + if (!frame) { + /* overflow check and correction for block mode */ + if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) { + U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); + U32 const correction = + ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(cctx, correction); + if (ms->nextToUpdate < correction) + ms->nextToUpdate = 0; + else + ms->nextToUpdate -= correction; + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { + size_t const cSize = frame ? ZSTD_compress_frameChunk(cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) + : ZSTD_compressBlock_internal(cctx, dst, dstCapacity, src, srcSize); + if (ZSTD_isError(cSize)) + return cSize; + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + if (cctx->consumedSrcSize + 1 > cctx->pledgedSrcSizePlusOne) { + DEBUGLOG(4, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne - 1, + (unsigned)cctx->consumedSrcSize); + return ERROR(srcSize_wrong); + } + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN(ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + if (srcSize > blockSizeMax) + return ERROR(srcSize_wrong); + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const void* src, + size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) +{ + const BYTE* const ip = (const BYTE*)src; + const BYTE* const iend = ip + srcSize; + + ZSTD_window_update(&ms->window, src, srcSize); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + + /* Assert that we the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + if (srcSize <= HASH_READ_SIZE) + return 0; + + switch (params->cParams.strategy) { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, dtlm); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (srcSize >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(ms, iend - HASH_READ_SIZE); + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + if (srcSize >= HASH_READ_SIZE) + ZSTD_updateTree(ms, iend - HASH_READ_SIZE, iend); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + +/* Dictionaries that assign zero probability to symbols that show up causes problems + when FSE encoding. Refuse dictionaries that assign zero probability to symbols + that we may encounter during compression. + NOTE: This behavior is not standard and could be improved in the future. */ +static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) + return ERROR(dictionary_corrupted); + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) + return ERROR(dictionary_corrupted); + } + return 0; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed > 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff + 1]; + unsigned offcodeMaxValue = MaxOff; + size_t dictID; + + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1 << MAX(MLFSELog, LLFSELog))); + assert(dictSize > 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictPtr += 4; /* skip magic number */ + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); + dictPtr += 4; + + { + unsigned maxSymbolValue = 255; + size_t const hufHeaderSize = + HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, dictEnd - dictPtr); + if (HUF_isError(hufHeaderSize)) + return ERROR(dictionary_corrupted); + if (maxSymbolValue < 255) + return ERROR(dictionary_corrupted); + dictPtr += hufHeaderSize; + } + + { + unsigned offcodeLog; + size_t const offcodeHeaderSize = + FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(offcodeHeaderSize)) + return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) + return ERROR(dictionary_corrupted); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + /* fill all offset symbols to avoid garbage at end of table */ + CHECK_E(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE), + dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { + short matchlengthNCount[MaxML + 1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = + FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(matchlengthHeaderSize)) + return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) + return ERROR(dictionary_corrupted); + /* Every match length code must have non-zero probability */ + CHECK_F(ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + CHECK_E(FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, + matchlengthNCount, + matchlengthMaxValue, + matchlengthLog, + workspace, + HUF_WORKSPACE_SIZE), + dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { + short litlengthNCount[MaxLL + 1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = + FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(litlengthHeaderSize)) + return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) + return ERROR(dictionary_corrupted); + /* Every literal length code must have non-zero probability */ + CHECK_F(ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + CHECK_E(FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, + litlengthNCount, + litlengthMaxValue, + litlengthLog, + workspace, + HUF_WORKSPACE_SIZE), + dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr + 12 > dictEnd) + return ERROR(dictionary_corrupted); + bs->rep[0] = MEM_readLE32(dictPtr + 0); + bs->rep[1] = MEM_readLE32(dictPtr + 4); + bs->rep[2] = MEM_readLE32(dictPtr + 8); + dictPtr += 12; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable */ + CHECK_F(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + /* All repCodes must be <= dictContentSize and != 0*/ + { + U32 u; + for (u = 0; u < 3; u++) { + if (bs->rep[u] == 0) + return ERROR(dictionary_corrupted); + if (bs->rep[u] > dictContentSize) + return ERROR(dictionary_corrupted); + } + } + + bs->entropy.huf.repeatMode = HUF_repeat_valid; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; + CHECK_F(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); + return dictID; + } +} + +/** ZSTD_compress_insertDictionary() : + * @return : dictID, or an error code */ +static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict == NULL) || (dictSize <= 8)) + return 0; + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + } + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_wrong); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary(bs, ms, params, dict, dictSize, dtlm, workspace); +} + +/*! ZSTD_compressBegin_internal() : + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (cdict && cdict->dictContentSize > 0) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff)); + { + size_t const dictID = ZSTD_compress_insertDictionary(cctx->blockState.prevCBlock, + &cctx->blockState.matchState, + ¶ms, + dict, + dictSize, + dictContentType, + dtlm, + cctx->entropyWorkspace); + if (ZSTD_isError(dictID)) + return dictID; + assert(dictID <= (size_t)(U32)-1); + cctx->dictID = (U32)dictID; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); + /* compression parameters verification and optimization */ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compressBegin_internal( + cctx, dict, dictSize, dictContentType, dtlm, cdict, params, pledgedSrcSize, ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : + * @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + return ZSTD_compressBegin_advanced_internal( + cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL /*cdict*/, cctxParams, pledgedSrcSize); +} + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, + dict, + dictSize, + ZSTD_dct_auto, + ZSTD_dtlm_fast, + NULL, + cctxParams, + ZSTD_CONTENTSIZE_UNKNOWN, + ZSTDb_not_buffered); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); +} + +/*! ZSTD_writeEpilogue() : + * Ends a frame. + * @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + if (cctx->stage == ZSTDcs_created) + return ERROR(stage_wrong); /* init missing */ + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); + if (ZSTD_isError(fhSize)) + return fhSize; + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw) << 1) + 0; + if (dstCapacity < 4) + return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32)XXH64_digest(&cctx->xxhState); + if (dstCapacity < 4) + return ERROR(dstSize_tooSmall); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op - ostart; +} + +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = + ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); + if (ZSTD_isError(cSize)) + return cSize; + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity - cSize); + if (ZSTD_isError(endResult)) + return endResult; + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize + 1) { + DEBUGLOG(4, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne - 1, + (unsigned)cctx->consumedSrcSize); + return ERROR(srcSize_wrong); + } + } + return cSize + endResult; +} + +static size_t ZSTD_compress_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, ZSTD_parameters params) +{ + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + DEBUGLOG(4, "ZSTD_compress_internal"); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); +} + +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced"); + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compress_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, ZSTD_CCtx_params params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + CHECK_F(ZSTD_compressBegin_internal( + cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered)); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0); + ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + assert(params.fParams.contentSizeFlag == 1); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + int compressionLevel, int* zstd_version) +{ + if (NULL != zstd_version) { + *zstd_version = OB_ZSTD_LIB_VERSION_138; + } + // fprintf(stderr, __FILE__ ": ytest 1_3_8 compress\n"); + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + size_t result; + ZSTD_CCtx ctxBody; + ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); + result = + ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel, NULL /*not check zstd_verion*/); + ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ + return result; +} + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict == NULL) + return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); +} + +static size_t ZSTD_initCDict_internal(ZSTD_CDict* cdict, const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(cParams)); + cdict->matchState.cParams = cParams; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictBuffer = NULL; + cdict->dictContent = dictBuffer; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); + cdict->dictBuffer = internalBuffer; + cdict->dictContent = internalBuffer; + if (!internalBuffer) + return ERROR(memory_allocation); + memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + { + void* const end = ZSTD_reset_matchState(&cdict->matchState, + (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, + &cParams, + ZSTDcrp_continue, + /* forCCtx */ 0); + assert(end == (char*)cdict->workspace + cdict->workspaceSize); + (void)end; + } + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is <= 8 bytes. + */ + { + ZSTD_CCtx_params params; + memset(¶ms, 0, sizeof(params)); + params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + params.cParams = cParams; + { + size_t const dictID = ZSTD_compress_insertDictionary(&cdict->cBlockState, + &cdict->matchState, + ¶ms, + cdict->dictContent, + cdict->dictContentSize, + dictContentType, + ZSTD_dtlm_full, + cdict->workspace); + if (ZSTD_isError(dictID)) + return dictID; + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) +{ + DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) + return NULL; + + { + ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + void* const workspace = ZSTD_malloc(workspaceSize, customMem); + + if (!cdict || !workspace) { + ZSTD_free(cdict, customMem); + ZSTD_free(workspace, customMem); + return NULL; + } + cdict->customMem = customMem; + cdict->workspace = workspace; + cdict->workspaceSize = workspaceSize; + if (ZSTD_isError(ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, cParams))) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict == NULL) + return 0; /* support free on NULL */ + { + ZSTD_customMem const cMem = cdict->customMem; + ZSTD_free(cdict->workspace, cMem); + ZSTD_free(cdict->dictBuffer, cMem); + ZSTD_free(cdict, cMem); + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) +{ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + size_t const neededSize = + sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) + HUF_WORKSPACE_SIZE + matchStateSize; + ZSTD_CDict* const cdict = (ZSTD_CDict*)workspace; + void* ptr; + if ((size_t)workspace & 7) + return NULL; /* 8-aligned */ + DEBUGLOG(4, + "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, + (unsigned)neededSize, + (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) + return NULL; + + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(cdict + 1, dict, dictSize); + dict = cdict + 1; + ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; + } else { + ptr = cdict + 1; + } + cdict->workspace = ptr; + cdict->workspaceSize = HUF_WORKSPACE_SIZE + matchStateSize; + + if (ZSTD_isError(ZSTD_initCDict_internal(cdict, dict, dictSize, ZSTD_dlm_byRef, dictContentType, cParams))) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/* ZSTD_compressBegin_usingCDict_advanced() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + if (cdict == NULL) + return ERROR(dictionary_wrong); + { + ZSTD_CCtx_params params = cctx->requestedParams; + params.cParams = ZSTD_getCParamsFromCDict(cdict); + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + params.cParams.windowLog = MAX(params.cParams.windowLog, limitedSrcLog); + } + params.fParams = fParams; + return ZSTD_compressBegin_internal( + cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, cdict, params, pledgedSrcSize, ZSTDb_not_buffered); + } +} + +/* ZSTD_compressBegin_usingCDict() : + * pledgedSrcSize=0 means "unknown" + * if pledgedSrcSize>0, it will enable contentSizeFlag */ +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = {0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/}; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); + return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + CHECK_F(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict( + ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = {1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/}; + return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + +/* ****************************************************************** + * Streaming + ********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) +{ + return ZSTD_BLOCKSIZE_MAX; +} + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */; +} + +static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, const void* const dict, size_t const dictSize, + ZSTD_dictContentType_e const dictContentType, const ZSTD_CDict* const cdict, ZSTD_CCtx_params params, + unsigned long long const pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_resetCStream_internal"); + /* Finalize the compression parameters */ + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + CHECK_F(ZSTD_compressBegin_internal( + cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, params, pledgedSrcSize, ZSTDb_buffered)); + + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + cctx->inBuffTarget = + cctx->blockSize + + (cctx->blockSize == pledgedSrcSize); /* for small input: avoid automatic flush on reaching end of block, since it + would require to add a 3-bytes null block to end frame */ + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + return 0; /* ready to go */ +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params params = zcs->requestedParams; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + if (pledgedSrcSize == 0) + pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + params.fParams.contentSizeFlag = 1; + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (dict && dictSize >= 8) { + DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize); + if (zcs->staticSize) { /* static CCtx : never uses malloc */ + /* incompatible with internal cdict creation */ + return ERROR(memory_allocation); + } + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = + ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params.cParams, zcs->customMem); + zcs->cdict = zcs->cdictLocal; + if (zcs->cdictLocal == NULL) + return ERROR(memory_allocation); + } else { + if (cdict) { + params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */ + } + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; + zcs->cdict = cdict; + } + + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced( + ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + if (!cdict) + return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */ + { + ZSTD_CCtx_params params = zcs->requestedParams; + params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.fParams = fParams; + return ZSTD_initCStream_internal(zcs, NULL, 0, cdict, params, pledgedSrcSize); + } +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = {0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */}; + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + return ZSTD_initCStream_usingCDict_advanced( + zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ +} + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced( + ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, + "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", + (unsigned)pledgedSrcSize, + params.fParams.contentSizeFlag); + CHECK_F(ZSTD_checkCParams(params.cParams)); + if ((pledgedSrcSize == 0) && (params.fParams.contentSizeFlag == 0)) + pledgedSrcSize = + ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now + specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize); +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + U64 const pledgedSrcSize = + (pss == 0) + ? ZSTD_CONTENTSIZE_UNKNOWN + : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" + **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ + ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); + return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN); +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize == 0) + hintInSize = cctx->blockSize; + return hintInSize; +} + +static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length) + memcpy(dst, src, length); + return length; +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * non-static, because can be called from zstdmt_compress.c + * @return : hint size for next input */ +size_t ZSTD_compressStream_generic( + ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective const flushMode) +{ + const char* const istart = (const char*)input->src; + const char* const iend = istart + input->size; + const char* ip = istart + input->pos; + char* const ostart = (char*)output->dst; + char* const oend = ostart + output->size; + char* op = ostart + output->pos; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + while (someMoreWork) { + switch (zcs->streamStage) { + case zcss_init: + /* call ZSTD_initCStream() first ! */ + return ERROR(init_missing); + + case zcss_load: + if ((flushMode == ZSTD_e_end) && ((size_t)(oend - op) >= ZSTD_compressBound(iend - ip)) /* enough dstCapacity */ + && (zcs->inBuffPos == 0)) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd(zcs, op, oend - op, ip, iend - ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + if (ZSTD_isError(cSize)) + return cSize; + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; + break; + } + /* complete loading into inBuffer */ + { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend - ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ((flushMode == ZSTD_e_continue) && (zcs->inBuffPos < zcs->inBuffTarget)) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; + break; + } + if ((flushMode == ZSTD_e_flush) && (zcs->inBuffPos == zcs->inToCompress)) { + /* empty */ + someMoreWork = 0; + break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { + void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend - op; + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip == iend); + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = lastBlock ? ZSTD_compressEnd(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) + : ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) + return cSize; + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + /* fall-through */ + case zcss_flush: + DEBUGLOG(5, "flush stage"); + { + size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend - op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG( + 5, "toFlush: %u into %u ==> flushed: %u", (unsigned)toFlush, (unsigned)(oend - op), (unsigned)flushed); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush != flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op == oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) + return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); + } +#endif + return ZSTD_nextInputSizeHint(cctx); +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + CHECK_F(ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue)); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +size_t ZSTD_compressStream2(ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + if (output->pos > output->size) + return ERROR(GENERIC); + if (input->pos > input->size) + return ERROR(GENERIC); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict == NULL || cctx->cdict == NULL); /* only one can be set */ + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) + cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ + params.cParams = + ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne - 1, 0 /*dictSize*/); + +#ifdef ZSTD_MULTITHREAD + if ((cctx->pledgedSrcSizePlusOne - 1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); + if (cctx->mtctx == NULL) + return ERROR(memory_allocation); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + CHECK_F(ZSTDMT_initCStream_internal(cctx->mtctx, + prefixDict.dict, + prefixDict.dictSize, + ZSTD_dct_rawContent, + cctx->cdict, + params, + cctx->pledgedSrcSizePlusOne - 1)); + cctx->streamStage = zcss_load; + cctx->appliedParams.nbWorkers = params.nbWorkers; + } else +#endif + { + CHECK_F(ZSTD_resetCStream_internal(cctx, + prefixDict.dict, + prefixDict.dictSize, + prefixDict.dictContentType, + cctx->cdict, + params, + cctx->pledgedSrcSizePlusOne - 1)); + assert(cctx->streamStage == zcss_load); + assert(cctx->appliedParams.nbWorkers == 0); + } + } + /* end of transparent initialization stage */ + + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + if (cctx->cParamsChanged) { + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); + cctx->cParamsChanged = 0; + } + { + size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + if (ZSTD_isError(flushMin) || (endOp == ZSTD_e_end && flushMin == 0)) { /* compression completed */ + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + } + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + return flushMin; + } + } +#endif + CHECK_F(ZSTD_compressStream_generic(cctx, output, input, endOp)); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, + size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output = {dst, dstCapacity, *dstPos}; + ZSTD_inBuffer input = {src, srcSize, *srcPos}; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + { + size_t oPos = 0; + size_t iPos = 0; + size_t const result = + ZSTD_compressStream2_simpleArgs(cctx, dst, dstCapacity, &oPos, src, srcSize, &iPos, ZSTD_e_end); + if (ZSTD_isError(result)) + return result; + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + return ERROR(dstSize_tooSmall); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = {NULL, 0, 0}; + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = {NULL, 0, 0}; + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + CHECK_F(remainingToFlush); + if (zcs->appliedParams.nbWorkers > 0) + return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { + size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 +int ZSTD_maxCLevel(void) +{ + return ZSTD_MAX_CLEVEL; +} +int ZSTD_minCLevel(void) +{ + return (int)-ZSTD_TARGETLENGTH_MAX; +} + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL + 1] = { + { + /* "default" - guarantees a monotonically increasing memory budget */ + /* W, C, H, S, L, TL, strat */ + {19, 12, 13, 1, 6, 1, ZSTD_fast}, /* base for negative levels */ + {19, 13, 14, 1, 7, 0, ZSTD_fast}, /* level 1 */ + {20, 15, 16, 1, 6, 0, ZSTD_fast}, /* level 2 */ + {21, 16, 17, 1, 5, 1, ZSTD_dfast}, /* level 3 */ + {21, 18, 18, 1, 5, 1, ZSTD_dfast}, /* level 4 */ + {21, 18, 19, 2, 5, 2, ZSTD_greedy}, /* level 5 */ + {21, 19, 19, 3, 5, 4, ZSTD_greedy}, /* level 6 */ + {21, 19, 19, 3, 5, 8, ZSTD_lazy}, /* level 7 */ + {21, 19, 19, 3, 5, 16, ZSTD_lazy2}, /* level 8 */ + {21, 19, 20, 4, 5, 16, ZSTD_lazy2}, /* level 9 */ + {22, 20, 21, 4, 5, 16, ZSTD_lazy2}, /* level 10 */ + {22, 21, 22, 4, 5, 16, ZSTD_lazy2}, /* level 11 */ + {22, 21, 22, 5, 5, 16, ZSTD_lazy2}, /* level 12 */ + {22, 21, 22, 5, 5, 32, ZSTD_btlazy2}, /* level 13 */ + {22, 22, 23, 5, 5, 32, ZSTD_btlazy2}, /* level 14 */ + {22, 23, 23, 6, 5, 32, ZSTD_btlazy2}, /* level 15 */ + {22, 22, 22, 5, 5, 48, ZSTD_btopt}, /* level 16 */ + {23, 23, 22, 5, 4, 64, ZSTD_btopt}, /* level 17 */ + {23, 23, 22, 6, 3, 64, ZSTD_btultra}, /* level 18 */ + {23, 24, 22, 7, 3, 256, ZSTD_btultra2}, /* level 19 */ + {25, 25, 23, 7, 3, 256, ZSTD_btultra2}, /* level 20 */ + {26, 26, 24, 7, 3, 512, ZSTD_btultra2}, /* level 21 */ + {27, 27, 25, 9, 3, 999, ZSTD_btultra2}, /* level 22 */ + }, + { + /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + {18, 12, 13, 1, 5, 1, ZSTD_fast}, /* base for negative levels */ + {18, 13, 14, 1, 6, 0, ZSTD_fast}, /* level 1 */ + {18, 14, 14, 1, 5, 1, ZSTD_dfast}, /* level 2 */ + {18, 16, 16, 1, 4, 1, ZSTD_dfast}, /* level 3 */ + {18, 16, 17, 2, 5, 2, ZSTD_greedy}, /* level 4.*/ + {18, 18, 18, 3, 5, 2, ZSTD_greedy}, /* level 5.*/ + {18, 18, 19, 3, 5, 4, ZSTD_lazy}, /* level 6.*/ + {18, 18, 19, 4, 4, 4, ZSTD_lazy}, /* level 7 */ + {18, 18, 19, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ + {18, 18, 19, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ + {18, 18, 19, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ + {18, 18, 19, 5, 4, 12, ZSTD_btlazy2}, /* level 11.*/ + {18, 19, 19, 7, 4, 12, ZSTD_btlazy2}, /* level 12.*/ + {18, 18, 19, 4, 4, 16, ZSTD_btopt}, /* level 13 */ + {18, 18, 19, 4, 3, 32, ZSTD_btopt}, /* level 14.*/ + {18, 18, 19, 6, 3, 128, ZSTD_btopt}, /* level 15.*/ + {18, 19, 19, 6, 3, 128, ZSTD_btultra}, /* level 16.*/ + {18, 19, 19, 8, 3, 256, ZSTD_btultra}, /* level 17.*/ + {18, 19, 19, 6, 3, 128, ZSTD_btultra2}, /* level 18.*/ + {18, 19, 19, 8, 3, 256, ZSTD_btultra2}, /* level 19.*/ + {18, 19, 19, 10, 3, 512, ZSTD_btultra2}, /* level 20.*/ + {18, 19, 19, 12, 3, 512, ZSTD_btultra2}, /* level 21.*/ + {18, 19, 19, 13, 3, 999, ZSTD_btultra2}, /* level 22.*/ + }, + { + /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + {17, 12, 12, 1, 5, 1, ZSTD_fast}, /* base for negative levels */ + {17, 12, 13, 1, 6, 0, ZSTD_fast}, /* level 1 */ + {17, 13, 15, 1, 5, 0, ZSTD_fast}, /* level 2 */ + {17, 15, 16, 2, 5, 1, ZSTD_dfast}, /* level 3 */ + {17, 17, 17, 2, 4, 1, ZSTD_dfast}, /* level 4 */ + {17, 16, 17, 3, 4, 2, ZSTD_greedy}, /* level 5 */ + {17, 17, 17, 3, 4, 4, ZSTD_lazy}, /* level 6 */ + {17, 17, 17, 3, 4, 8, ZSTD_lazy2}, /* level 7 */ + {17, 17, 17, 4, 4, 8, ZSTD_lazy2}, /* level 8 */ + {17, 17, 17, 5, 4, 8, ZSTD_lazy2}, /* level 9 */ + {17, 17, 17, 6, 4, 8, ZSTD_lazy2}, /* level 10 */ + {17, 17, 17, 5, 4, 8, ZSTD_btlazy2}, /* level 11 */ + {17, 18, 17, 7, 4, 12, ZSTD_btlazy2}, /* level 12 */ + {17, 18, 17, 3, 4, 12, ZSTD_btopt}, /* level 13.*/ + {17, 18, 17, 4, 3, 32, ZSTD_btopt}, /* level 14.*/ + {17, 18, 17, 6, 3, 256, ZSTD_btopt}, /* level 15.*/ + {17, 18, 17, 6, 3, 128, ZSTD_btultra}, /* level 16.*/ + {17, 18, 17, 8, 3, 256, ZSTD_btultra}, /* level 17.*/ + {17, 18, 17, 10, 3, 512, ZSTD_btultra}, /* level 18.*/ + {17, 18, 17, 5, 3, 256, ZSTD_btultra2}, /* level 19.*/ + {17, 18, 17, 7, 3, 512, ZSTD_btultra2}, /* level 20.*/ + {17, 18, 17, 9, 3, 512, ZSTD_btultra2}, /* level 21.*/ + {17, 18, 17, 11, 3, 999, ZSTD_btultra2}, /* level 22.*/ + }, + { + /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + {14, 12, 13, 1, 5, 1, ZSTD_fast}, /* base for negative levels */ + {14, 14, 15, 1, 5, 0, ZSTD_fast}, /* level 1 */ + {14, 14, 15, 1, 4, 0, ZSTD_fast}, /* level 2 */ + {14, 14, 15, 2, 4, 1, ZSTD_dfast}, /* level 3 */ + {14, 14, 14, 4, 4, 2, ZSTD_greedy}, /* level 4 */ + {14, 14, 14, 3, 4, 4, ZSTD_lazy}, /* level 5.*/ + {14, 14, 14, 4, 4, 8, ZSTD_lazy2}, /* level 6 */ + {14, 14, 14, 6, 4, 8, ZSTD_lazy2}, /* level 7 */ + {14, 14, 14, 8, 4, 8, ZSTD_lazy2}, /* level 8.*/ + {14, 15, 14, 5, 4, 8, ZSTD_btlazy2}, /* level 9.*/ + {14, 15, 14, 9, 4, 8, ZSTD_btlazy2}, /* level 10.*/ + {14, 15, 14, 3, 4, 12, ZSTD_btopt}, /* level 11.*/ + {14, 15, 14, 4, 3, 24, ZSTD_btopt}, /* level 12.*/ + {14, 15, 14, 5, 3, 32, ZSTD_btultra}, /* level 13.*/ + {14, 15, 15, 6, 3, 64, ZSTD_btultra}, /* level 14.*/ + {14, 15, 15, 7, 3, 256, ZSTD_btultra}, /* level 15.*/ + {14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + {14, 15, 15, 6, 3, 128, ZSTD_btultra2}, /* level 17.*/ + {14, 15, 15, 7, 3, 256, ZSTD_btultra2}, /* level 18.*/ + {14, 15, 15, 8, 3, 256, ZSTD_btultra2}, /* level 19.*/ + {14, 15, 15, 8, 3, 512, ZSTD_btultra2}, /* level 20.*/ + {14, 15, 15, 9, 3, 512, ZSTD_btultra2}, /* level 21.*/ + {14, 15, 15, 10, 3, 999, ZSTD_btultra2}, /* level 22.*/ + }, +}; + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + size_t const addedSize = srcSizeHint ? 0 : 500; + U64 const rSize = srcSizeHint + dictSize ? srcSizeHint + dictSize + addedSize : (U64)-1; + U32 const tableID = + (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + int row = compressionLevel; + DEBUGLOG(5, "ZSTD_getCParams (cLevel=%i)", compressionLevel); + if (compressionLevel == 0) + row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + if (compressionLevel < 0) + row = 0; /* entry 0 is baseline for fast mode */ + if (compressionLevel > ZSTD_MAX_CLEVEL) + row = ZSTD_MAX_CLEVEL; + { + ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + if (compressionLevel < 0) + cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); + } +} + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). + * All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_compress_internal.h b/src/lib/compress/zstd_1_3_8/zstd_compress_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..427012ec828487369ad91bfea17d337768ad83ce --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_compress_internal.h @@ -0,0 +1,1141 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* + * Dependencies + ***************************************/ +#include "zstd_internal.h" +#ifdef ZSTD_MULTITHREAD +#include "zstdmt_compress.h" +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +/*-************************************* + * Constants + ***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK \ + 1 /* For btlazy2 strategy, index 1 now means "unsorted". \ + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. \ + It's not a big deal though : candidate will just be sorted again. \ + Additionnally, candidate position 1 will be lost. \ + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. \ + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy \ + Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + +/*-************************************* + * Context memory management + ***************************************/ +typedef enum { ZSTDcs_created = 0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init = 0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + U32 CTable[HUF_CTABLE_SIZE_U32(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +typedef struct { + U32 off; + U32 len; +} ZSTD_match_t; + +typedef struct { + int price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + +typedef enum { zop_dynamic = 0, zop_predef } ZSTD_OptPrice_e; + +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice; /* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more data */ +} ZSTD_window_t; + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 nextToUpdate3; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + U64 hashPower; /* Used to compute the rolling hash. + * Depends on ldmParams.minMatchLength */ +} ldmState_t; + +typedef struct { + U32 enableLdm; /* 1 if enable long distance matching */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + U32 offset; + U32 litLength; + U32 matchLength; +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The position where reading stopped. <= size. */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 32, + 33, + 33, + 34, + 34, + 35, + 35, + 36, + 36, + 36, + 36, + 37, + 37, + 37, + 37, + 38, + 38, + 38, + 38, + 38, + 38, + 38, + 38, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 39, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 41, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42, + 42}; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/*! ZSTD_storeSeq() : + * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. + * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). + * `mlBase` : matchLength - MINMATCH + */ +MEM_STATIC void ZSTD_storeSeq( + seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase) +{ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start == NULL) + g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { + U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, + "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", + pos, + (U32)litLength, + (U32)mlBase + MINMATCH, + (U32)offsetCode); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength > 0xFFFF) { + assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ + seqStorePtr->longLengthID = 1; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offset = offsetCode + 1; + + /* match Length */ + if (mlBase > 0xFFFF) { + assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ + seqStorePtr->longLengthID = 2; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].matchLength = (U16)mlBase; + + seqStorePtr->sequences++; +} + +/*-************************************* + * Match length counter + ***************************************/ +static unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +#if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)(r >> 3); +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_ctzll((U64)val) >> 3); +#else + static const int DeBruijnBytePos[64] = {0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 0, + 3, + 1, + 3, + 1, + 4, + 2, + 7, + 0, + 2, + 3, + 6, + 1, + 5, + 3, + 5, + 1, + 3, + 4, + 4, + 2, + 5, + 6, + 7, + 7, + 0, + 1, + 2, + 3, + 3, + 4, + 6, + 2, + 6, + 5, + 5, + 3, + 4, + 5, + 6, + 7, + 1, + 2, + 4, + 6, + 4, + 4, + 5, + 7, + 2, + 6, + 5, + 7, + 6, + 7, + 7}; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +#endif + } else { /* 32 bits */ +#if defined(_MSC_VER) + unsigned long r = 0; + _BitScanForward(&r, (U32)val); + return (unsigned)(r >> 3); +#elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +#else + static const int DeBruijnBytePos[32] = { + 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1}; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +#endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +#if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64(&r, val); + return (unsigned)(r >> 3); +#elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_clzll(val) >> 3); +#else + unsigned r; + const unsigned n32 = sizeof(size_t) * 4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val >> n32)) { + r = 4; + } else { + r = 0; + val >>= n32; + } + if (!(val >> 16)) { + r += 2; + val >>= 8; + } else { + val >>= 24; + } + r += (!val); + return r; +#endif + } else { /* 32 bits */ +#if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse(&r, (unsigned long)val); + return (unsigned)(r >> 3); +#elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +#else + unsigned r; + if (!(val >> 16)) { + r = 2; + val >>= 8; + } else { + r = 0; + val >>= 24; + } + r += (!val); + return r; +#endif + } + } +} + +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t) - 1); + + if (pIn < pInLoopLimit) { + { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) + return ZSTD_NbCommonBytes(diff); + } + pIn += sizeof(size_t); + pMatch += sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn += sizeof(size_t); + pMatch += sizeof(size_t); + continue; + } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } + } + if (MEM_64bits() && (pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { + pIn += 4; + pMatch += 4; + } + if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { + pIn += 2; + pMatch += 2; + } + if ((pIn < pInLimit) && (*pMatch == *pIn)) + pIn++; + return (size_t)(pIn - pStart); +} + +/** ZSTD_count_2segments() : + * can count match length with `ip` & `match` in 2 different segments. + * convention : on reaching mEnd, match count continue starting from iStart + */ +MEM_STATIC size_t ZSTD_count_2segments( + const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) +{ + const BYTE* const vEnd = MIN(ip + (mEnd - match), iEnd); + size_t const matchLength = ZSTD_count(ip, match, vEnd); + if (match + matchLength != mEnd) + return matchLength; + DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength); + DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match); + DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip); + DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart); + DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip + matchLength, iStart, iEnd)); + return matchLength + ZSTD_count(ip + matchLength, iStart, iEnd); +} + +/*-************************************* + * Hashes + ***************************************/ +static const U32 prime3bytes = 506832829U; +static U32 ZSTD_hash3(U32 u, U32 h) +{ + return ((u << (32 - 24)) * prime3bytes) >> (32 - h); +} +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) +{ + return ZSTD_hash3(MEM_readLE32(ptr), h); +} /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) +{ + return (u * prime4bytes) >> (32 - h); +} +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) +{ + return ZSTD_hash4(MEM_read32(ptr), h); +} + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) +{ + return (size_t)(((u << (64 - 40)) * prime5bytes) >> (64 - h)); +} +static size_t ZSTD_hash5Ptr(const void* p, U32 h) +{ + return ZSTD_hash5(MEM_readLE64(p), h); +} + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) +{ + return (size_t)(((u << (64 - 48)) * prime6bytes) >> (64 - h)); +} +static size_t ZSTD_hash6Ptr(const void* p, U32 h) +{ + return ZSTD_hash6(MEM_readLE64(p), h); +} + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) +{ + return (size_t)(((u << (64 - 56)) * prime7bytes) >> (64 - h)); +} +static size_t ZSTD_hash7Ptr(const void* p, U32 h) +{ + return ZSTD_hash7(MEM_readLE64(p), h); +} + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) +{ + return (size_t)(((u)*prime8bytes) >> (64 - h)); +} +static size_t ZSTD_hash8Ptr(const void* p, U32 h) +{ + return ZSTD_hash8(MEM_readLE64(p), h); +} + +MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch (mls) { + default: + case 4: + return ZSTD_hash4Ptr(p, hBits); + case 5: + return ZSTD_hash5Ptr(p, hBits); + case 6: + return ZSTD_hash6Ptr(p, hBits); + case 7: + return ZSTD_hash7Ptr(p, hBits); + case 8: + return ZSTD_hash8Ptr(p, hBits); + } +} + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) + power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* + * Round buffer management + ***************************************/ +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + (((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t* ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? ZSTD_extDict + : ms->dictMatchState != NULL ? ZSTD_dictMatchState + : ZSTD_noDict; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, void const* srcEnd) +{ + U32 const current = (U32)((BYTE const*)srcEnd - window.base); + return current > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + * NOTE: (maxDist & cycleMask) must be zero. + */ +MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const newCurrent = (current & cycleMask) + maxDist; + U32 const correction = current - newCurrent; + assert((maxDist & cycleMask) == 0); + assert(current > newCurrent); + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1 << 28); + + window->base += correction; + window->dictBase += correction; + window->lowLimit -= correction; + window->dictLimit -= correction; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * This allows a simple check that index >= lowLimit to see if index is valid. + * This must be called before a block compression call, with srcEnd as the block + * source end. + * + * If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit. + * This is because dictionaries are allowed to be referenced as long as the last + * byte of the dictionary is in the window, but once they are out of range, + * they cannot be referenced. If loadedDictEndPtr is NULL, we use + * loadedDictEnd == 0. + * + * In normal dict mode, the dict is between lowLimit and dictLimit. In + * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary + * is below them. forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window, void const* srcEnd, U32 maxDist, + U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base); + U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u", (unsigned)blockEndIdx, (unsigned)maxDist); + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) + window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, + "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, + (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + if (loadedDictEndPtr) + *loadedDictEndPtr = 0; + if (dictMatchStatePtr) + *dictMatchStatePtr = NULL; + } +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, void const* src, size_t srcSize) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + /* Check if blocks follow each other */ + if (src != window->nextSrc) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + // ms->nextToUpdate = window->dictLimit; + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) + window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ((ip + srcSize > window->dictBase + window->lowLimit) & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/* debug functions */ +#if (DEBUGLEVEL >= 2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u = 0, sum = 0; u <= max; u++) + sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u = 0; u <= max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u])); + } +} + +#endif + +#if defined(__cplusplus) +} +#endif + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_compressStream_generic() : + * Private use only. To be called from zstdmt_compress.c in single-thread mode. */ +size_t ZSTD_compressStream_generic( + ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective const flushMode); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, ZSTD_CCtx_params params); + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapcity` is too small ( /* memcpy, memmove, memset */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) +#include "zstd_legacy.h" +#endif + +/*-******************************************************* + * Types + *********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + +static size_t ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) + return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { + U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + CHECK_E(ZSTD_loadDEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted); + ddict->entropyPresent = 1; + return 0; +} + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) + dictSize = 0; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) + return ERROR(memory_allocation); + memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + CHECK_F(ZSTD_loadEntropy_intoDDict(ddict, dictContentType)); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) + return NULL; + + { + ZSTD_DDict* const ddict = (ZSTD_DDict*)ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) + return NULL; + ddict->cMem = customMem; + { + size_t const initResult = ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } + } + return ddict; + } +} + +/*! ZSTD_createDDict() : + * Create a digested dictionary, to start decompression without startup delay. + * `dict` content is copied inside DDict. + * Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = {NULL, NULL, NULL}; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = {NULL, NULL, NULL}; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + +const ZSTD_DDict* ZSTD_initStaticDDict(void* sBuffer, size_t sBufferSize, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) + return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) + return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(ddict + 1, dict, dictSize); /* local copy */ + dict = ddict + 1; + } + if (ZSTD_isError(ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef, dictContentType))) + return NULL; + return ddict; +} + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict == NULL) + return 0; /* support free on NULL */ + { + ZSTD_customMem const cMem = ddict->cMem; + ZSTD_free(ddict->dictBuffer, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict == NULL) + return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0); +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict == NULL) + return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_ddict.h b/src/lib/compress/zstd_1_3_8/zstd_ddict.h new file mode 100644 index 0000000000000000000000000000000000000000..ff51cb24d3e2228e5da0b6507468f6c58ac91100 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_ddict.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* size_t */ +#include "zstd.h" /* ZSTD_DDict, and several public functions */ + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +#endif /* ZSTD_DDICT_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_decompress.c b/src/lib/compress/zstd_1_3_8/zstd_decompress.c new file mode 100644 index 0000000000000000000000000000000000000000..41ac8e349ec3ab1e4bc55c37d0c7bd259a856dec --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_decompress.c @@ -0,0 +1,1811 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* *************************************************************** + * Tuning parameters + *****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +#define ZSTD_HEAPMODE 1 +#endif + +/*! + * LEGACY_SUPPORT : + * if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) + */ +#ifndef ZSTD_LEGACY_SUPPORT +#define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +#define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +#define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* memcpy, memmove, memset */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) +#include "zstd_legacy.h" +#endif + +/*-************************************************************* + * Context management + ***************************************************************/ +size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx) +{ + if (dctx == NULL) + return 0; /* support sizeof NULL */ + return sizeof(*dctx) + ZSTD_sizeof_DDict(dctx->ddictLocal) + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) +{ + return sizeof(ZSTD_DCtx); +} + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = + (format == ZSTD_f_zstd1_magicless) ? ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE : ZSTD_FRAMEHEADERSIZE_PREFIX; + ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert((format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless)); + return startingInputLength; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->format = + ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */ + dctx->staticSize = 0; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; + dctx->noForwardProgress = 0; + dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*)workspace; + + if ((size_t)workspace & 7) + return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) + return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx + 1); + return dctx; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) + return NULL; + + { + ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); + if (!dctx) + return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx == NULL) + return 0; /* support free on NULL */ + if (dctx->staticSize) + return ERROR(memory_allocation); /* not compatible with static DCtx */ + { + ZSTD_customMem const cMem = dctx->customMem; + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + ZSTD_free(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + ZSTD_free(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) + return 0; + { + U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) + return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) + return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) + return 1; +#endif + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + if (srcSize < minInputSize) + return ERROR(srcSize_wrong); + + { + BYTE const fhd = ((const BYTE*)src)[minInputSize - 1]; + U32 const dictID = fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + memset(zfhPtr, + 0, + sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be + read only if return value is zero, since they are 2 different signals */ + if (srcSize < minInputSize) + return minInputSize; + if (src == NULL) + return ERROR(GENERIC); /* invalid parameter */ + + if ((format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER)) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char*)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { + size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) + return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { + BYTE const fhdByte = ip[minInputSize - 1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte & 3; + U32 const checksumFlag = (fhdByte >> 2) & 1; + U32 const singleSegment = (fhdByte >> 5) & 1; + U32 const fcsID = fhdByte >> 6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + if ((fhdByte & 0x08) != 0) + return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */ + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) + return ERROR(frameParameter_windowTooLarge); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte & 7); + } + switch (dictIDSizeCode) { + default: + assert(0); /* impossible */ + case 0: + break; + case 1: + dictID = ip[pos]; + pos++; + break; + case 2: + dictID = MEM_readLE16(ip + pos); + pos += 2; + break; + case 3: + dictID = MEM_readLE32(ip + pos); + pos += 4; + break; + } + switch (fcsID) { + default: + assert(0); /* impossible */ + case 0: + if (singleSegment) + frameContentSize = ip[pos]; + break; + case 1: + frameContentSize = MEM_readLE16(ip + pos) + 256; + break; + case 2: + frameContentSize = MEM_readLE32(ip + pos); + break; + case 3: + frameContentSize = MEM_readLE64(ip + pos); + break; + } + if (singleSegment) + windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned)MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { + ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } + } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ERROR(srcSize_wrong); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + if ((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32) + return ERROR(frameParameter_unsupported); + + return skippableHeaderSize + sizeU32; +} + +/** ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) + return skippableSize; + if (srcSize < skippableSize) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE*)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) + return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { + size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE*)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) + return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) + return result; /* invalid header */ + if (result > 0) + return ERROR(srcSize_wrong); /* headerSize too small */ + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) + return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) + XXH64_reset(&dctx->xxhState, 0); + return 0; +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); +#endif + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && + (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + return readSkippableFrameSize(src, srcSize); + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { + size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ret; + if (ret > 0) + return ERROR(srcSize_wrong); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return cBlockSize; + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ERROR(srcSize_wrong); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + + if (blockProperties.lastBlock) + break; + } + + if (zfh.checksumFlag) { /* Final frame content checksum */ + if (remainingSize < 4) + return ERROR(srcSize_wrong); + ip += 4; + } + + return ip - ipstart; + } +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + if (dst == NULL) { + if (srcSize == 0) + return 0; + return ERROR(dstBuffer_null); + } + if (srcSize > dstCapacity) + return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, BYTE b, size_t regenSize) +{ + if (dst == NULL) { + if (regenSize == 0) + return 0; + return ERROR(dstBuffer_null); + } + if (regenSize > dstCapacity) + return ERROR(dstSize_tooSmall); + memset(dst, b, regenSize); + return regenSize; +} + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void** srcPtr, size_t* srcSizePtr) +{ + const BYTE* ip = (const BYTE*)(*srcPtr); + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + if (remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN + ZSTD_blockHeaderSize) + return ERROR(srcSize_wrong); + + /* Frame Header */ + { + size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); + if (ZSTD_isError(frameHeaderSize)) + return frameHeaderSize; + if (remainingSrcSize < frameHeaderSize + ZSTD_blockHeaderSize) + return ERROR(srcSize_wrong); + CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); + ip += frameHeaderSize; + remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSrcSize) + return ERROR(srcSize_wrong); + + switch (blockProperties.blockType) { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend - op, ip, cBlockSize, /* frame */ 1); + break; + case bt_raw: + decodedSize = ZSTD_copyRawBlock(op, oend - op, ip, cBlockSize); + break; + case bt_rle: + decodedSize = ZSTD_setRleBlock(op, oend - op, *ip, blockProperties.origSize); + break; + case bt_reserved: + default: + return ERROR(corruption_detected); + } + + if (ZSTD_isError(decodedSize)) + return decodedSize; + if (dctx->fParams.checksumFlag) + XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) + break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + if ((U64)(op - ostart) != dctx->fParams.frameContentSize) { + return ERROR(corruption_detected); + } + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + if (remainingSrcSize < 4) + return ERROR(checksum_wrong); + checkRead = MEM_readLE32(ip); + if (checkRead != checkCalc) + return ERROR(checksum_wrong); + ip += 4; + remainingSrcSize -= 4; + } + + /* Allow caller to get size read */ + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return op - ostart; +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict == NULL || ddict == NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) + return frameSize; + /* legacy support is not compatible with static dctx */ + if (dctx->staticSize) + return ERROR(memory_allocation); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) + return decodedSize; + + assert(decodedSize <= -dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(4, "reading magic number %08X (expecting %08X)", (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) + return skippableSize; + if (srcSize < skippableSize) + return ERROR(srcSize_wrong); + + src = (const BYTE*)src + skippableSize; + srcSize -= skippableSize; + continue; + } + } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + CHECK_F(ZSTD_decompressBegin_usingDDict(dctx, ddict)); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); + } + ZSTD_checkContinuity(dctx, dst); + + { + const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize); + if ((ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) && (moreThan1Frame == 1)) { + /* at least one frame successfully completed, + * but following bytes are garbage : + * it's more likely to be a srcSize error, + * specifying more bytes than compressed size of frame(s). + * This error message replaces ERROR(prefix_unknown), + * which would be confusing, as the first header is actually correct. + * Note that one could be unlucky, it might be a corruption error instead, + * happening right at the place where we expect zstd magic bytes. + * But this is _much_ less likely than a srcSize field error. */ + return ERROR(srcSize_wrong); + } + if (ZSTD_isError(res)) + return res; + assert(res <= dstCapacity); + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) + return ERROR(srcSize_wrong); /* input not entirely consumed */ + + return (BYTE*)dst - (BYTE*)dststart; +} + +size_t ZSTD_decompress_usingDict( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + +size_t ZSTD_decompressDCtx( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int* zstd_version) +{ + if (NULL != zstd_version) { + *zstd_version = OB_ZSTD_LIB_VERSION_138; + } + // fprintf(stderr, __FILE__ ": ytest 1_3_8 decompress\n"); + return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE >= 1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (dctx == NULL) + return ERROR(memory_allocation); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize, NULL /*not check zstd_verion*/); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + +/*-************************************** + * Advanced Streaming Decompression API + * Bufferless and synchronous + ****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) +{ + return dctx->expected; +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) +{ + switch (dctx->stage) { + default: /* should not happen */ + assert(0); + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + if (srcSize != dctx->expected) + return ERROR(srcSize_wrong); /* not allowed */ + if (dstCapacity) + ZSTD_checkContinuity(dctx, dst); + + switch (dctx->stage) { + case ZSTDds_getFrameHeaderSize: + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = + ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } + } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) + return dctx->headerSize; + memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: { + blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) + return cBlockSize; + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { + size_t rSize; + switch (dctx->bType) { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + break; + case bt_raw: + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle: + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + break; + case bt_reserved: /* should never happen */ + default: + return ERROR(corruption_detected); + } + if (ZSTD_isError(rSize)) + return rSize; + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->fParams.checksumFlag) + XXH64_update(&dctx->xxhState, dst, rSize); + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + if (dctx->decodedSize != dctx->fParams.frameContentSize) { + return ERROR(corruption_detected); + } + } + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG( + 4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + if (check32 != h32) + return ERROR(checksum_wrong); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = + MEM_readLE32(dctx->headerBuffer + + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + return ERROR(GENERIC); /* some compiler require default to do something */ + } +} + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + if (dictSize <= 8) + return ERROR(dictionary_corrupted); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT( + offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT( + offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT( + sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { + void* const workspace = + &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, workspace, workspaceSize); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, workspace, workspaceSize); +#endif + if (HUF_isError(hSize)) + return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { + short offcodeNCount[MaxOff + 1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = + FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(offcodeHeaderSize)) + return ERROR(dictionary_corrupted); + if (offcodeMaxValue > MaxOff) + return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) + return ERROR(dictionary_corrupted); + ZSTD_buildFSETable(entropy->OFTable, offcodeNCount, offcodeMaxValue, OF_base, OF_bits, offcodeLog); + dictPtr += offcodeHeaderSize; + } + + { + short matchlengthNCount[MaxML + 1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = + FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(matchlengthHeaderSize)) + return ERROR(dictionary_corrupted); + if (matchlengthMaxValue > MaxML) + return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) + return ERROR(dictionary_corrupted); + ZSTD_buildFSETable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, ML_base, ML_bits, matchlengthLog); + dictPtr += matchlengthHeaderSize; + } + + { + short litlengthNCount[MaxLL + 1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = + FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd - dictPtr); + if (FSE_isError(litlengthHeaderSize)) + return ERROR(dictionary_corrupted); + if (litlengthMaxValue > MaxLL) + return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) + return ERROR(dictionary_corrupted); + ZSTD_buildFSETable(entropy->LLTable, litlengthNCount, litlengthMaxValue, LL_base, LL_bits, litlengthLog); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr + 12 > dictEnd) + return ERROR(dictionary_corrupted); + { + int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr + 12)); + for (i = 0; i < 3; i++) { + U32 const rep = MEM_readLE32(dictPtr); + dictPtr += 4; + if (rep == 0 || rep >= dictContentSize) + return ERROR(dictionary_corrupted); + entropy->rep[i] = rep; + } + } + + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) + return ZSTD_refDictContent(dctx, dict, dictSize); + { + U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } + } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { + size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + if (ZSTD_isError(eSize)) + return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + CHECK_F(ZSTD_decompressBegin(dctx)); + if (dict && dictSize) + CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); + return 0; +} + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); + } + CHECK_F(ZSTD_decompressBegin(dctx)); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) + return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) + return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompresse frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = {0, 0, 0, ZSTD_frame, 0, 0, 0}; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) + return 0; + return zfp.dictID; +} + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a pre-digested Dictionary + * Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, NULL, 0, ddict); +} + +/*===================================== + * Streaming decompression + *====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_advanced(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) +{ + return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; +} +size_t ZSTD_DStreamOutSize(void) +{ + return ZSTD_BLOCKSIZE_MAX; +} + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) +{ + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + ZSTD_freeDDict(dctx->ddictLocal); + if (dict && dictSize >= 8) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + if (dctx->ddictLocal == NULL) + return ERROR(memory_allocation); + } else { + dctx->ddictLocal = NULL; + } + dctx->ddict = dctx->ddictLocal; + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced( + ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType); +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + zds->streamStage = zdss_init; + zds->noForwardProgress = 0; + CHECK_F(ZSTD_DCtx_loadDictionary(zds, dict, dictSize)); + return ZSTD_FRAMEHEADERSIZE_PREFIX; +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + size_t const initResult = ZSTD_initDStream(dctx); + dctx->ddict = ddict; + return initResult; +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + dctx->streamStage = zdss_loadHeader; + dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0; + dctx->legacyVersion = 0; + dctx->hostageByte = 0; + return ZSTD_FRAMEHEADERSIZE_PREFIX; +} + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + dctx->ddict = ddict; + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + if (maxWindowSize < min) + return ERROR(parameter_outOfBound); + if (maxWindowSize > max) + return ERROR(parameter_outOfBound); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = {0, 0, 0}; + switch (dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) + return 0; + if (value < bounds.lowerBound) + return 0; + if (value > bounds.upperBound) + return 0; + return 1; +} + +#define CHECK_DBOUNDS(p, v) \ + { \ + if (!ZSTD_dParam_withinBounds(p, v)) \ + return ERROR(parameter_outOfBound); \ + } + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + switch (dParam) { + case ZSTD_d_windowLogMax: + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + default:; + } + return ERROR(parameter_unsupported); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ((reset == ZSTD_reset_session_only) || (reset == ZSTD_reset_session_and_parameters)) { + (void)ZSTD_initDStream(dctx); + } + if ((reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters)) { + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + } + return 0; +} + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t)neededSize; + if ((unsigned long long)minRBSize != neededSize) + return ERROR(frameParameter_windowTooLarge); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = + 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) + return err; + if (err > 0) + return ERROR(srcSize_wrong); + if (zfh.windowSize > windowSizeMax) + return ERROR(frameParameter_windowTooLarge); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + +/* ***** Decompression ***** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + if (input->pos > input->size) { /* forbidden */ + DEBUGLOG(5, "in: pos: %u vs size: %u", (U32)input->pos, (U32)input->size); + return ERROR(srcSize_wrong); + } + if (output->pos > output->size) { /* forbidden */ + DEBUGLOG(5, "out: pos: %u vs size: %u", (U32)output->pos, (U32)output->size); + return ERROR(dstSize_tooSmall); + } + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + + while (someMoreWork) { + switch (zds->streamStage) { + case zdss_init: + DEBUGLOG(5, "stage zdss_init => transparent reset "); + ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + /* fall-through */ + + case zdss_loadHeader: + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (zds->legacyVersion) { + /* legacy support is incompatible with static dctx */ + if (zds->staticSize) + return ERROR(memory_allocation); + { + size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint == 0) + zds->streamStage = zdss_init; + return hint; + } + } +#endif + { + size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + DEBUGLOG(5, "header size : %u", (U32)hSize); + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend - istart); + if (legacyVersion) { + const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL; + size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + /* legacy support is incompatible with static dctx */ + if (zds->staticSize) + return ERROR(memory_allocation); + CHECK_F(ZSTD_initLegacyStream( + &zds->legacyContext, zds->previousLegacyVersion, legacyVersion, dict, dictSize)); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { + size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint == 0) + zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } + } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend - ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); + zds->lhSize = hSize; + ip += toLoad; + break; + } + } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ + && (U64)(size_t)(oend - op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend - istart); + if (cSize <= (size_t)(iend - istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend - op, istart, cSize, zds->ddict); + if (ZSTD_isError(decompressedSize)) + return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + ip = istart + cSize; + op += decompressedSize; + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == + ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, + "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >> 10), + (U32)(zds->maxWindowSize >> 10)); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) + return ERROR(frameParameter_windowTooLarge); + + /* Adapt buffer sizes to frame header instructions */ + { + size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = + ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); + if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx)) + return ERROR(memory_allocation); + } else { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + if (zds->inBuff == NULL) + return ERROR(memory_allocation); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } + } + zds->streamStage = zdss_read; + /* fall-through */ + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize == 0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend - ip) >= neededInSize) { /* decode directly from src */ + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, + (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, + neededInSize); + if (ZSTD_isError(decodedSize)) + return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) + break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + break; + } + } + if (ip == iend) { + someMoreWork = 0; + break; + } /* no more input */ + zds->streamStage = zdss_load; + /* fall-through */ + + case zdss_load: { + size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend - ip)); + } else { + if (toLoad > zds->inBuffSize - zds->inPos) + return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend - ip); + } + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { + someMoreWork = 0; + break; + } /* not enough input, wait for more */ + + /* decode loaded input */ + { + size_t const decodedSize = ZSTD_decompressContinue( + zds, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) + return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + break; + } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + } + } + zds->streamStage = zdss_flush; + /* fall-through */ + + case zdss_flush: { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend - op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ((zds->outBuffSize < zds->fParams.frameContentSize) && + (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize)) { + DEBUGLOG(5, + "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } + } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + return ERROR(GENERIC); /* some compiler require default to do something */ + } + } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + if ((ip == istart) && (op == ostart)) { /* no forward progress */ + zds->noForwardProgress++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + if (op == oend) + return ERROR(dstSize_tooSmall); + if (ip == iend) + return ERROR(srcSize_wrong); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { + size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output + is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte = 1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += + ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output = {dst, dstCapacity, *dstPos}; + ZSTD_inBuffer input = {src, srcSize, *srcPos}; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_decompress_block.c b/src/lib/compress/zstd_1_3_8/zstd_decompress_block.c new file mode 100644 index 0000000000000000000000000000000000000000..f4e154bcb67b254321c2bb06e20d3ef992b843da --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_decompress_block.c @@ -0,0 +1,1472 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* memcpy, memmove, memset */ +#include "compiler.h" /* prefetch */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" + +/*_******************************************************* + * Macros + **********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + +/*_******************************************************* + * Memory operations + **********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) +{ + memcpy(dst, src, 4); +} + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + if (srcSize < ZSTD_blockHeaderSize) + return ERROR(srcSize_wrong); + { + U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) + return 1; + if (bpPtr->blockType == bt_reserved) + return ERROR(corruption_detected); + return cSize; + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize); +/*! ZSTD_decodeLiteralsBlock() : + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) + return ERROR(corruption_detected); + + { + const BYTE* const istart = (const BYTE*)src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch (litEncType) { + case set_repeat: + if (dctx->litEntropy == 0) + return ERROR(dictionary_corrupted); + /* fall-through */ + + case set_compressed: + if (srcSize < 5) + return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { + size_t lhSize, litSize, litCSize; + U32 singleStream = 0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + switch (lhlCode) { + case 0: + case 1: + default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) + return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) + return ERROR(corruption_detected); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType == set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr, dctx->bmi2); + } else { + hufSuccess = HUF_decompress4X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart + lhSize, litCSize, dctx->HUFptr, dctx->bmi2); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp(dctx->entropy.hufTable, + dctx->litBuffer, + litSize, + istart + lhSize, + litCSize, + dctx->workspace, + sizeof(dctx->workspace)); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(dctx->entropy.hufTable, + dctx->litBuffer, + litSize, + istart + lhSize, + litCSize, + dctx->workspace, + sizeof(dctx->workspace), + dctx->bmi2); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(dctx->entropy.hufTable, + dctx->litBuffer, + litSize, + istart + lhSize, + litCSize, + dctx->workspace, + sizeof(dctx->workspace), + dctx->bmi2); + } + } + + if (HUF_isError(hufSuccess)) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType == set_compressed) + dctx->HUFptr = dctx->entropy.hufTable; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: { + size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch (lhlCode) { + case 0: + case 2: + default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize + litSize + WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize + lhSize > srcSize) + return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart + lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize + litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart + lhSize; + dctx->litSize = litSize; + return lhSize + litSize; + } + + case set_rle: { + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch (lhlCode) { + case 0: + case 2: + default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize < 4) + return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) + return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize + 1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1 << LL_DEFAULTNORMLOG) + 1] = { + {1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + {0, 0, 4, 0}, + {16, 0, 4, 0}, + {32, 0, 5, 1}, + {0, 0, 5, 3}, + {0, 0, 5, 4}, + {0, 0, 5, 6}, + {0, 0, 5, 7}, + {0, 0, 5, 9}, + {0, 0, 5, 10}, + {0, 0, 5, 12}, + {0, 0, 6, 14}, + {0, 1, 5, 16}, + {0, 1, 5, 20}, + {0, 1, 5, 22}, + {0, 2, 5, 28}, + {0, 3, 5, 32}, + {0, 4, 5, 48}, + {32, 6, 5, 64}, + {0, 7, 5, 128}, + {0, 8, 6, 256}, + {0, 10, 6, 1024}, + {0, 12, 6, 4096}, + {32, 0, 4, 0}, + {0, 0, 4, 1}, + {0, 0, 5, 2}, + {32, 0, 5, 4}, + {0, 0, 5, 5}, + {32, 0, 5, 7}, + {0, 0, 5, 8}, + {32, 0, 5, 10}, + {0, 0, 5, 11}, + {0, 0, 6, 13}, + {32, 1, 5, 16}, + {0, 1, 5, 18}, + {32, 1, 5, 22}, + {0, 2, 5, 24}, + {32, 3, 5, 32}, + {0, 3, 5, 40}, + {0, 6, 4, 64}, + {16, 6, 4, 64}, + {32, 7, 5, 128}, + {0, 9, 6, 512}, + {0, 11, 6, 2048}, + {48, 0, 4, 0}, + {16, 0, 4, 1}, + {32, 0, 5, 2}, + {32, 0, 5, 3}, + {32, 0, 5, 5}, + {32, 0, 5, 6}, + {32, 0, 5, 8}, + {32, 0, 5, 9}, + {32, 0, 5, 11}, + {32, 0, 5, 12}, + {0, 0, 6, 15}, + {32, 1, 5, 18}, + {32, 1, 5, 20}, + {32, 2, 5, 24}, + {32, 2, 5, 28}, + {32, 3, 5, 40}, + {32, 4, 5, 48}, + {0, 16, 6, 65536}, + {0, 15, 6, 32768}, + {0, 14, 6, 16384}, + {0, 13, 6, 8192}, +}; /* LL_defaultDTable */ + +/* Default FSE distribution table for Offset Codes */ +static const ZSTD_seqSymbol OF_defaultDTable[(1 << OF_DEFAULTNORMLOG) + 1] = { + {1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + {0, 0, 5, 0}, + {0, 6, 4, 61}, + {0, 9, 5, 509}, + {0, 15, 5, 32765}, + {0, 21, 5, 2097149}, + {0, 3, 5, 5}, + {0, 7, 4, 125}, + {0, 12, 5, 4093}, + {0, 18, 5, 262141}, + {0, 23, 5, 8388605}, + {0, 5, 5, 29}, + {0, 8, 4, 253}, + {0, 14, 5, 16381}, + {0, 20, 5, 1048573}, + {0, 2, 5, 1}, + {16, 7, 4, 125}, + {0, 11, 5, 2045}, + {0, 17, 5, 131069}, + {0, 22, 5, 4194301}, + {0, 4, 5, 13}, + {16, 8, 4, 253}, + {0, 13, 5, 8189}, + {0, 19, 5, 524285}, + {0, 1, 5, 1}, + {16, 6, 4, 61}, + {0, 10, 5, 1021}, + {0, 16, 5, 65533}, + {0, 28, 5, 268435453}, + {0, 27, 5, 134217725}, + {0, 26, 5, 67108861}, + {0, 25, 5, 33554429}, + {0, 24, 5, 16777213}, +}; /* OF_defaultDTable */ + +/* Default FSE distribution table for Match Lengths */ +static const ZSTD_seqSymbol ML_defaultDTable[(1 << ML_DEFAULTNORMLOG) + 1] = { + {1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */ + /* nextState, nbAddBits, nbBits, baseVal */ + {0, 0, 6, 3}, + {0, 0, 4, 4}, + {32, 0, 5, 5}, + {0, 0, 5, 6}, + {0, 0, 5, 8}, + {0, 0, 5, 9}, + {0, 0, 5, 11}, + {0, 0, 6, 13}, + {0, 0, 6, 16}, + {0, 0, 6, 19}, + {0, 0, 6, 22}, + {0, 0, 6, 25}, + {0, 0, 6, 28}, + {0, 0, 6, 31}, + {0, 0, 6, 34}, + {0, 1, 6, 37}, + {0, 1, 6, 41}, + {0, 2, 6, 47}, + {0, 3, 6, 59}, + {0, 4, 6, 83}, + {0, 7, 6, 131}, + {0, 9, 6, 515}, + {16, 0, 4, 4}, + {0, 0, 4, 5}, + {32, 0, 5, 6}, + {0, 0, 5, 7}, + {32, 0, 5, 9}, + {0, 0, 5, 10}, + {0, 0, 6, 12}, + {0, 0, 6, 15}, + {0, 0, 6, 18}, + {0, 0, 6, 21}, + {0, 0, 6, 24}, + {0, 0, 6, 27}, + {0, 0, 6, 30}, + {0, 0, 6, 33}, + {0, 1, 6, 35}, + {0, 1, 6, 39}, + {0, 2, 6, 43}, + {0, 3, 6, 51}, + {0, 4, 6, 67}, + {0, 5, 6, 99}, + {0, 8, 6, 259}, + {32, 0, 4, 4}, + {48, 0, 4, 4}, + {16, 0, 4, 5}, + {32, 0, 5, 7}, + {32, 0, 5, 8}, + {32, 0, 5, 10}, + {32, 0, 5, 11}, + {0, 0, 6, 14}, + {0, 0, 6, 17}, + {0, 0, 6, 20}, + {0, 0, 6, 23}, + {0, 0, 6, 26}, + {0, 0, 6, 29}, + {0, 0, 6, 32}, + {0, 16, 6, 65539}, + {0, 15, 6, 32771}, + {0, 14, 6, 16387}, + {0, 13, 6, 8195}, + {0, 12, 6, 4099}, + {0, 11, 6, 2051}, + {0, 10, 6, 1027}, +}; /* ML_defaultDTable */ + +static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U32 nbAddBits) +{ + void* ptr = dt; + ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr; + ZSTD_seqSymbol* const cell = dt + 1; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->baseValue = baseValue; +} + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, unsigned tableLog) +{ + ZSTD_seqSymbol* const tableDecode = dt + 1; + U16 symbolNext[MaxSeq + 1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize - 1; + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + + /* Init, lay down lowprob symbols */ + { + ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { + S16 const largeLimit = (S16)(1 << (tableLog - 1)); + U32 s; + for (s = 0; s < maxSV1; s++) { + if (normalizedCounter[s] == -1) { + tableDecode[highThreshold--].baseValue = s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) + DTableH.fastMode = 0; + symbolNext[s] = normalizedCounter[s]; + } + } + } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { + U32 const tableMask = tableSize - 1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s = 0; s < maxSV1; s++) { + int i; + for (i = 0; i < normalizedCounter[s]; i++) { + tableDecode[position].baseValue = s; + position = (position + step) & tableMask; + while (position > highThreshold) + position = (position + step) & tableMask; /* lowprob area */ + } + } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u = 0; u < tableSize; u++) { + U32 const symbol = tableDecode[u].baseValue; + U32 const nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32(nextState)); + tableDecode[u].nextState = (U16)((nextState << tableDecode[u].nbBits) - tableSize); + assert(nbAdditionalBits[symbol] < 255); + tableDecode[u].nbAdditionalBits = (BYTE)nbAdditionalBits[symbol]; + tableDecode[u].baseValue = baseValue[symbol]; + } + } +} + +/*! ZSTD_buildSeqTable() : + * @return : nb bytes read from src, + * or an error code if it fails */ +static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr, + symbolEncodingType_e type, unsigned max, U32 maxLog, const void* src, size_t srcSize, const U32* baseValue, + const U32* nbAdditionalBits, const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable, int ddictIsCold, int nbSeq) +{ + switch (type) { + case set_rle: + if (!srcSize) + return ERROR(srcSize_wrong); + if ((*(const BYTE*)src) > max) + return ERROR(corruption_detected); + { + U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U32 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic: + *DTablePtr = defaultTable; + return 0; + case set_repeat: + if (!flagRepeatTable) + return ERROR(corruption_detected); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed: { + unsigned tableLog; + S16 norm[MaxSeq + 1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSE_isError(headerSize)) + return ERROR(corruption_detected); + if (tableLog > maxLog) + return ERROR(corruption_detected); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); + *DTablePtr = DTableSpace; + return headerSize; + } + default: /* impossible */ + assert(0); + return ERROR(GENERIC); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) + return ERROR(srcSize_wrong); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr = 0; + if (srcSize != 1) + return ERROR(srcSize_wrong); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + if (ip + 2 > iend) + return ERROR(srcSize_wrong); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip += 2; + } else { + if (ip >= iend) + return ERROR(srcSize_wrong); + nbSeq = ((nbSeq - 0x80) << 8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + if (ip + 4 > iend) + return ERROR(srcSize_wrong); /* minimum possible size */ + { + symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { + size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, + &dctx->LLTptr, + LLtype, + MaxLL, + LLFSELog, + ip, + iend - ip, + LL_base, + LL_bits, + LL_defaultDTable, + dctx->fseEntropy, + dctx->ddictIsCold, + nbSeq); + if (ZSTD_isError(llhSize)) + return ERROR(corruption_detected); + ip += llhSize; + } + + { + size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, + &dctx->OFTptr, + OFtype, + MaxOff, + OffFSELog, + ip, + iend - ip, + OF_base, + OF_bits, + OF_defaultDTable, + dctx->fseEntropy, + dctx->ddictIsCold, + nbSeq); + if (ZSTD_isError(ofhSize)) + return ERROR(corruption_detected); + ip += ofhSize; + } + + { + size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, + &dctx->MLTptr, + MLtype, + MaxML, + MLFSELog, + ip, + iend - ip, + ML_base, + ML_bits, + ML_defaultDTable, + dctx->fseEntropy, + dctx->ddictIsCold, + nbSeq); + if (ZSTD_isError(mlhSize)) + return ERROR(corruption_detected); + ip += mlhSize; + } + } + + return ip - istart; +} + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; + const BYTE* match; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; + const BYTE* prefixStart; + const BYTE* dictEnd; + size_t pos; +} seqState_t; + +/* ZSTD_execSequenceLast7(): + * exceptional case : decompress a match starting within last 7 bytes of output buffer. + * requires more careful checks, to ensure there is no overflow. + * performance does not matter though. + * note : this case is supposed to be never generated "naturally" by reference encoder, + * since in most cases it needs at least 8 bytes to look for a match. + * but it's allowed by the specification. */ +FORCE_NOINLINE +size_t ZSTD_execSequenceLast7(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, + const BYTE* const litLimit, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd > oend) + return ERROR(dstSize_tooSmall); /* last match must fit within dstBuffer */ + if (iLitEnd > litLimit) + return ERROR(corruption_detected); /* try to read beyond literal buffer */ + + /* copy literals */ + while (op < oLitEnd) + *op++ = *(*litPtr)++; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) + return ERROR(corruption_detected); + match = dictEnd - (base - match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { + size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } + } + while (op < oMatchEnd) + *op++ = *match++; + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd > oend) + return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) + return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd > oend_w) + return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op + 8, + (*litPtr) + 8, + sequence.litLength - + 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + if (sequence.offset > (size_t)(oLitEnd - virtualStart)) + return ERROR(corruption_detected); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { + size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) + op[i] = match[i]; + return sequenceLength; + } + } + } + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ + static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op + 4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; + match += 8; + + if (oMatchEnd > oend - (16 - MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) + *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequenceLong(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, + const BYTE* const litLimit, const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = sequence.match; + + /* check */ + if (oMatchEnd > oend) + return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) + return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd > oend_w) + return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + if (sequence.litLength > 8) + ZSTD_wildcopy(op + 8, + (*litPtr) + 8, + sequence.litLength - + 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - dictStart)) + return ERROR(corruption_detected); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { + size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) + op[i] = match[i]; + return sequenceLength; + } + } + } + assert(op <= oend_w); + assert(sequence.matchLength >= MINMATCH); + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ + static const int dec64table[] = {8, 8, 8, 7, 8, 9, 10, 11}; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op + 4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; + match += 8; + + if (oMatchEnd > oend - (16 - MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) + *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + +static void ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +{ + ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset = 1 } ZSTD_longOffset_e; + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +FORCE_INLINE_TEMPLATE seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; + U32 const totalBits = llBits + mlBits + ofBits; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; + + /* sequence */ + { + size_t offset; + if (!ofBits) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + if (extraBits) + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits /*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofBits <= 1) { + offset += (llBase == 0); + if (offset) { + size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { /* offset == 0 */ + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = mlBase + ((mlBits > 0) ? BIT_readBitsFast(&seqState->DStream, mlBits /*>0*/) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits + llBits >= STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64 - (LLFSELog + MLFSELog + OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16 + LLFSELog + MLFSELog + OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = llBase + ((llBits > 0) ? BIT_readBitsFast(&seqState->DStream, llBits /*>0*/) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + /* ANS state update */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + +FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body"); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { + U32 i; + for (i = 0; i < ZSTD_REP_NUM; i++) + seqState.prevOffset[i] = dctx->entropy.rep[i]; + } + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq;) { + nbSeq--; + { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + if (ZSTD_isError(oneSeqSize)) + return oneSeqSize; + op += oneSeqSize; + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + if (nbSeq) + return ERROR(corruption_detected); + /* save reps for next block */ + { + U32 i; + for (i = 0; i < ZSTD_REP_NUM; i++) + dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); + } + } + + /* last literal segment */ + { + size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend - op)) + return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op - ostart; +} + +static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, + size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +FORCE_INLINE_TEMPLATE seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) +{ + seq_t seq; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; + U32 const totalBits = llBits + mlBits + ofBits; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; + + /* sequence */ + { + size_t offset; + if (!ofBits) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets) { + U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32 - 1); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + if (MEM_32bits() || extraBits) + BIT_reloadDStream(&seqState->DStream); + if (extraBits) + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofBits <= 1) { + offset += (llBase == 0); + if (offset) { + size_t temp = (offset == 3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = mlBase + ((mlBits > 0) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits + llBits >= STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64 - (LLFSELog + MLFSELog + OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16 + LLFSELog + MLFSELog + OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = llBase + ((llBits > 0) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + { + size_t const pos = seqState->pos + seq.litLength; + const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; + seq.match = matchBase + pos - + seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only + * happen when input is corrupted. No consequence though : no memory access will occur, + * overly large offset will be detected in ZSTD_execSequenceLong() */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + +FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 4 +#define STORED_SEQS_MASK (STORED_SEQS - 1) +#define ADVANCED_SEQS 4 + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + dctx->fseEntropy = 1; + { + int i; + for (i = 0; i < ZSTD_REP_NUM; i++) + seqState.prevOffset[i] = dctx->entropy.rep[i]; + } + seqState.prefixStart = prefixStart; + seqState.pos = (size_t)(op - prefixStart); + seqState.dictEnd = dictEnd; + assert(iend >= ip); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend - ip), corruption_detected); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb = 0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb < seqAdvance); seqNb++) { + sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, isLongOffset); + PREFETCH_L1(sequences[seqNb].match); + PREFETCH_L1(sequences[seqNb].match + sequences[seqNb].matchLength - + 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + if (seqNb < seqAdvance) + return ERROR(corruption_detected); + + /* decode and decompress */ + for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) { + seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequenceLong(op, + oend, + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], + &litPtr, + litEnd, + prefixStart, + dictStart, + dictEnd); + if (ZSTD_isError(oneSeqSize)) + return oneSeqSize; + PREFETCH_L1(sequence.match); + PREFETCH_L1(sequence.match + sequence.matchLength - + 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + if (seqNb < nbSeq) + return ERROR(corruption_detected); + + /* finish queue */ + seqNb -= seqAdvance; + for (; seqNb < nbSeq; seqNb++) { + size_t const oneSeqSize = ZSTD_execSequenceLong( + op, oend, sequences[seqNb & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); + if (ZSTD_isError(oneSeqSize)) + return oneSeqSize; + op += oneSeqSize; + } + + /* save reps for next block */ + { + U32 i; + for (i = 0; i < ZSTD_REP_NUM; i++) + dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); + } + } + + /* last literal segment */ + { + size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend - op)) + return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op - ostart; +} + +static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, + size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static TARGET_ATTRIBUTE("bmi2") size_t ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static TARGET_ATTRIBUTE("bmi2") size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, + size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, + size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes + * mearning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, + size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) + total += 1; + } + + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + + return total; +} +#endif + +size_t ZSTD_decompressBlock_internal( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const int frame) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * In block mode, window size is not known, so we have to be conservative. + * (note: but it could be evaluated from current-lowLimit) + */ + ZSTD_longOffset_e const isLongOffset = + (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + if (srcSize >= ZSTD_BLOCKSIZE_MAX) + return ERROR(srcSize_wrong); + + /* Decode literals section */ + { + size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) + return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) + return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (!usePrefetchDecoder && (!frame || (dctx->fParams.windowSize > (1 << 24))) && + (nbSeq > ADVANCED_SEQS)) { /* could probably use a larger nbSeq limit */ + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (shareLongOffsets >= minShare); + } +#endif + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } +} + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_decompress_block.h b/src/lib/compress/zstd_1_3_8/zstd_decompress_block.h new file mode 100644 index 0000000000000000000000000000000000000000..ef37e742b5e614e4a7a3759661dcd89b653242d7 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_decompress_block.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* size_t */ +#include "zstd.h" /* DCtx, and some public functions */ +#include "zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal( + ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const int frame); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, unsigned tableLog); + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_decompress_internal.h b/src/lib/compress/zstd_1_3_8/zstd_decompress_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..ae7f523eb4879889ffe5f59b53a096f8117b9561 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_decompress_internal.h @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + +#ifndef ZSTD_DECOMPRESS_INTERNAL_H +#define ZSTD_DECOMPRESS_INTERNAL_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "mem.h" /* BYTE, U16, U32 */ +#include "zstd_internal.h" /* ZSTD_seqSymbol */ + +/*-******************************************************* + * Constants + *********************************************************/ +static const U32 LL_base[MaxLL + 1] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 18, + 20, + 22, + 24, + 28, + 32, + 40, + 48, + 64, + 0x80, + 0x100, + 0x200, + 0x400, + 0x800, + 0x1000, + 0x2000, + 0x4000, + 0x8000, + 0x10000}; + +static const U32 OF_base[MaxOff + 1] = {0, + 1, + 1, + 5, + 0xD, + 0x1D, + 0x3D, + 0x7D, + 0xFD, + 0x1FD, + 0x3FD, + 0x7FD, + 0xFFD, + 0x1FFD, + 0x3FFD, + 0x7FFD, + 0xFFFD, + 0x1FFFD, + 0x3FFFD, + 0x7FFFD, + 0xFFFFD, + 0x1FFFFD, + 0x3FFFFD, + 0x7FFFFD, + 0xFFFFFD, + 0x1FFFFFD, + 0x3FFFFFD, + 0x7FFFFFD, + 0xFFFFFFD, + 0x1FFFFFFD, + 0x3FFFFFFD, + 0x7FFFFFFD}; + +static const U32 OF_bits[MaxOff + 1] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31}; + +static const U32 ML_base[MaxML + 1] = {3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 37, + 39, + 41, + 43, + 47, + 51, + 59, + 67, + 83, + 99, + 0x83, + 0x103, + 0x203, + 0x403, + 0x803, + 0x1003, + 0x2003, + 0x4003, + 0x8003, + 0x10003}; + +/*-******************************************************* + * Decompression types + *********************************************************/ +typedef struct { + U32 fastMode; + U32 tableLog; +} ZSTD_seqSymbol_header; + +typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; +} ZSTD_seqSymbol; + +#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building + hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least + HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; +} ZSTD_entropyDTables_t; + +typedef enum { + ZSTDds_getFrameHeaderSize, + ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, + ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, + ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, + ZSTDds_skipFrame +} ZSTD_dStage; + +typedef enum { zdss_init = 0, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +struct ZSTD_DCtx_s { + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block + decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context + lifetime. */ + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; + int noForwardProgress; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst); + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_double_fast.c b/src/lib/compress/zstd_1_3_8/zstd_double_fast.c new file mode 100644 index 0000000000000000000000000000000000000000..4499a55d05236dbc517f843a47cdd613f53fd6ef --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_double_fast.c @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const current = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = current + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = current + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */, ZSTD_dictMode_e const dictMode) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const U32* const dictHashLong = dictMode == ZSTD_dictMatchState ? dms->hashTable : NULL; + const U32* const dictHashSmall = dictMode == ZSTD_dictMatchState ? dms->chainTable : NULL; + const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? dms->window.dictLimit : 0; + const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? dictBase + dictStartIndex : NULL; + const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? prefixLowestIndex - (U32)(dictEnd - dictBase) : 0; + const U32 dictHBitsL = dictMode == ZSTD_dictMatchState ? dictCParams->hashLog : hBitsL; + const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? dictCParams->chainLog : hBitsS; + const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); + + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + + /* init */ + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const maxRep = (U32)(ip - prefixLowest); + if (offset_2 > maxRep) + offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) + offsetSaved = offset_1, offset_1 = 0; + } + if (dictMode == ZSTD_dictMatchState) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const current = (U32)(ip - base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = current + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState && repIndex < prefixLowestIndex) + ? dictBase + (repIndex - dictIndexDelta) + : base + repIndex; + hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + + /* check dictMatchState repcode */ + if (dictMode == ZSTD_dictMatchState && + ((U32)((prefixLowestIndex - 1) - repIndex) >= 3 /* intentional underflow */) && + (MEM_read32(repMatch) == MEM_read32(ip + 1))) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + goto _match_stored; + } + + /* check noDict repcode */ + if (dictMode == ZSTD_noDict && ((offset_1 > 0) & (MEM_read32(ip + 1 - offset_1) == MEM_read32(ip + 1)))) { + mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip + 8, matchLong + 8, iend) + 8; + offset = (U32)(ip - matchLong); + while (((ip > anchor) & (matchLong > prefixLowest)) && (ip[-1] == matchLong[-1])) { + ip--; + matchLong--; + mLength++; + } /* catch up */ + goto _match_found; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictHashLong[dictHL]; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip + 8, dictMatchL + 8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(current - dictMatchIndexL - dictIndexDelta); + while (((ip > anchor) & (dictMatchL > dictStart)) && (ip[-1] == dictMatchL[-1])) { + ip--; + dictMatchL--; + mLength++; + } /* catch up */ + goto _match_found; + } + } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictHashSmall[dictHS]; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } + + ip += ((ip - anchor) >> kSearchStrength) + 1; + continue; + + _search_next_long : + + { + size_t const hl3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); + size_t const dictHLNext = ZSTD_hashPtr(ip + 1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = current + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip + 1)) { + mLength = ZSTD_count(ip + 9, matchL3 + 8, iend) + 8; + ip++; + offset = (U32)(ip - matchL3); + while (((ip > anchor) & (matchL3 > prefixLowest)) && (ip[-1] == matchL3[-1])) { + ip--; + matchL3--; + mLength++; + } /* catch up */ + goto _match_found; + } + } else if (dictMode == ZSTD_dictMatchState) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip + 1)) { + mLength = ZSTD_count_2segments(ip + 1 + 8, dictMatchL3 + 8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip > anchor) & (dictMatchL3 > dictStart)) && (ip[-1] == dictMatchL3[-1])) { + ip--; + dictMatchL3--; + mLength++; + } /* catch up */ + goto _match_found; + } + } + } + + /* if no long +1 match, explore the short match we found */ + if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(current - matchIndexS); + while (((ip > anchor) & (match > dictStart)) && (ip[-1] == match[-1])) { + ip--; + match--; + mLength++; + } /* catch up */ + } else { + mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; + offset = (U32)(ip - match); + while (((ip > anchor) & (match > prefixLowest)) && (ip[-1] == match[-1])) { + ip--; + match--; + mLength++; + } /* catch up */ + } + + /* fall-through */ + + _match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + + _match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashLong[ZSTD_hashPtr(base + current + 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(base + current + 2, hBitsS, mls)] = + current + 2; /* here because current+2 could be > iend-8 */ + hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); + + /* check immediate repcode */ + if (dictMode == ZSTD_dictMatchState) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip - base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState && repIndex2 < prefixLowestIndex + ? dictBase - dictIndexDelta + repIndex2 + : base + repIndex2; + if (((U32)((prefixLowestIndex - 1) - (U32)repIndex2) >= 3 /* intentional overflow */) && + (MEM_read32(repMatch2) == MEM_read32(ip))) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip + 4, repMatch2 + 4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; + offset_2 = offset_1; + offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2 - MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ((ip <= ilimit) && ((offset_2 > 0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)))) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; + U32 const tmpOff = offset_2; + offset_2 = offset_1; + offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength - MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + case 5: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + case 6: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + case 7: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + } +} + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + case 5: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + case 6: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + case 7: + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + } +} + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + prefixStartIndex; + const U32 dictStartIndex = ms->window.lowLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 current = (U32)(ip - base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + + if ((((U32)((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (repIndex > dictStartIndex)) && + (MEM_read32(repMatch) == MEM_read32(ip + 1))) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip + 8, matchLong + 8, iend, matchEnd, prefixStart) + 8; + offset = current - matchLongIndex; + while (((ip > anchor) & (matchLong > lowMatchPtr)) && (ip[-1] == matchLong[-1])) { + ip--; + matchLong--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip + 1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = current + 1; + if ((matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip + 1))) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip + 9, match3 + 8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = current + 1 - matchIndex3; + while (((ip > anchor) & (match3 > lowMatchPtr)) && (ip[-1] == match3[-1])) { + ip--; + match3--; + mLength++; + } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, prefixStart) + 4; + offset = current - matchIndex; + while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { + ip--; + match--; + mLength++; + } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + + } else { + ip += ((ip - anchor) >> kSearchStrength) + 1; + continue; + } + } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashSmall[ZSTD_hashPtr(base + current + 2, hBitsS, mls)] = current + 2; + hashLong[ZSTD_hashPtr(base + current + 2, hBitsL, 8)] = current + 2; + hashSmall[ZSTD_hashPtr(ip - 2, hBitsS, mls)] = (U32)(ip - 2 - base); + hashLong[ZSTD_hashPtr(ip - 2, hBitsL, 8)] = (U32)(ip - 2 - base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip - base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ((((U32)((prefixStartIndex - 1) - repIndex2) >= + 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (repIndex2 > dictStartIndex)) && + (MEM_read32(repMatch2) == MEM_read32(ip))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip + 4, repMatch2 + 4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; + offset_2 = offset_1; + offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2 - MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + case 5: + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + case 6: + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + case 7: + return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + } +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_double_fast.h b/src/lib/compress/zstd_1_3_8/zstd_double_fast.h new file mode 100644 index 0000000000000000000000000000000000000000..988b17dfdd35d7030ce13aea38baf5b048cfc85a --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_double_fast.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include "mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_errors.h b/src/lib/compress/zstd_1_3_8/zstd_errors.h new file mode 100644 index 0000000000000000000000000000000000000000..8305854d8676c3bf4133675cd3e254f4c29baa41 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_errors.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined(__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBILITY +#if defined(__GNUC__) && (__GNUC__ >= 4) +#define ZSTDERRORLIB_VISIBILITY __attribute__((visibility("default"))) +#else +#define ZSTDERRORLIB_VISIBILITY +#endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT == 1) +#define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT == 1) +#define ZSTDERRORLIB_API \ + __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a \ + function pointer load from the IAT and an indirect jump.*/ +#else +#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall = 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_maxCode = + 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString( + ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_fast.c b/src/lib/compress/zstd_1_3_8/zstd_fast.c new file mode 100644 index 0000000000000000000000000000000000000000..417f38083b63ab50d3e9291e5c955628688a6104 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_fast.c @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_fast.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for (; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const current = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = current; + if (dtlm == ZSTD_dtlm_fast) + continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { + U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = current + p; + } + } + } + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, ZSTD_dictMode_e const dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ? dms->hashTable : NULL; + const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? dms->window.dictLimit : 0; + const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? dictBase + dictStartIndex : NULL; + const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? prefixStartIndex - (U32)(dictEnd - dictBase) : 0; + const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); + const U32 dictHLog = dictMode == ZSTD_dictMatchState ? dictCParams->hashLog : hlog; + + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + + /* otherwise, we would get index underflow when translating a dict index + * into a local index */ + assert(dictMode != ZSTD_dictMatchState || prefixStartIndex >= (U32)(dictEnd - dictBase)); + + /* init */ + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const maxRep = (U32)(ip - prefixStart); + if (offset_2 > maxRep) + offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) + offsetSaved = offset_1, offset_1 = 0; + } + if (dictMode == ZSTD_dictMatchState) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hlog, mls); + U32 const current = (U32)(ip - base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + const U32 repIndex = current + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState && repIndex < prefixStartIndex) + ? dictBase + (repIndex - dictIndexDelta) + : base + repIndex; + hashTable[h] = current; /* update hash table */ + + if ((dictMode == ZSTD_dictMatchState) && + ((U32)((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + } else if (dictMode == ZSTD_noDict && ((offset_1 > 0) & (MEM_read32(ip + 1 - offset_1) == MEM_read32(ip + 1)))) { + mLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + } else if ((matchIndex <= prefixStartIndex)) { + if (dictMode == ZSTD_dictMatchState) { + size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); + U32 const dictMatchIndex = dictHashTable[dictHash]; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex <= dictStartIndex || MEM_read32(dictMatch) != MEM_read32(ip)) { + assert(stepSize >= 1); + ip += ((ip - anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a dict match */ + U32 const offset = (U32)(current - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip > anchor) & (dictMatch > dictStart)) && (ip[-1] == dictMatch[-1])) { + ip--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + } + } else { + assert(stepSize >= 1); + ip += ((ip - anchor) >> kSearchStrength) + stepSize; + continue; + } + } else if (MEM_read32(match) != MEM_read32(ip)) { + /* it's not a match, and we're not going to check the dictionary */ + assert(stepSize >= 1); + ip += ((ip - anchor) >> kSearchStrength) + stepSize; + continue; + } else { + /* found a regular match */ + U32 const offset = (U32)(ip - match); + mLength = ZSTD_count(ip + 4, match + 4, iend) + 4; + while (((ip > anchor) & (match > prefixStart)) && (ip[-1] == match[-1])) { + ip--; + match--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + assert(base + current + 2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base + current + 2, hlog, mls)] = + current + 2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip - 2, hlog, mls)] = (U32)(ip - 2 - base); + + /* check immediate repcode */ + if (dictMode == ZSTD_dictMatchState) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip - base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = + repIndex2 < prefixStartIndex ? dictBase - dictIndexDelta + repIndex2 : base + repIndex2; + if (((U32)((prefixStartIndex - 1) - (U32)repIndex2) >= 3 /* intentional overflow */) && + (MEM_read32(repMatch2) == MEM_read32(ip))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip + 4, repMatch2 + 4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; + offset_2 = offset_1; + offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2 - MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ((ip <= ilimit) && ((offset_2 > 0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)))) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; + U32 const tmpOff = offset_2; + offset_2 = offset_1; + offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip - base); + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength - MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32 const mls = cParams->minMatch; + assert(ms->dictMatchState == NULL); + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); + case 5: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); + case 6: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); + case 7: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); + } +} + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32 const mls = cParams->minMatch; + assert(ms->dictMatchState != NULL); + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + case 5: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + case 6: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + case 7: + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + } +} + +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 dictStartIndex = ms->window.lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1 = rep[0], offset_2 = rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hlog, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 current = (U32)(ip - base); + const U32 repIndex = current + 1 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashTable[h] = current; /* update hash table */ + assert(offset_1 <= current + 1); /* check repIndex */ + + if ((((U32)((prefixStartIndex - 1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && + (MEM_read32(repMatch) == MEM_read32(ip + 1))) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, 0, mLength - MINMATCH); + } else { + if ((matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip))) { + assert(stepSize >= 1); + ip += ((ip - anchor) >> kSearchStrength) + stepSize; + continue; + } + { + const BYTE* matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip + 4, match + 4, iend, matchEnd, prefixStart) + 4; + while (((ip > anchor) & (match > lowMatchPtr)) && (ip[-1] == match[-1])) { + ip--; + match--; + mLength++; + } /* catch up */ + offset = current - matchIndex; + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip - anchor, anchor, offset + ZSTD_REP_MOVE, mLength - MINMATCH); + } + } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base + current + 2, hlog, mls)] = current + 2; + hashTable[ZSTD_hashPtr(ip - 2, hlog, mls)] = (U32)(ip - 2 - base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip - base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ((((U32)((prefixStartIndex - 1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip + 4, repMatch2 + 4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; + offset_2 = offset_1; + offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2 - MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32 const mls = cParams->minMatch; + switch (mls) { + default: /* includes case 3 */ + case 4: + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4); + case 5: + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5); + case 6: + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6); + case 7: + return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7); + } +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_fast.h b/src/lib/compress/zstd_1_3_8/zstd_fast.h new file mode 100644 index 0000000000000000000000000000000000000000..10ea9abf11e83809027d459e0a72c441cb25ea4c --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_fast.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include "mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_internal.h b/src/lib/compress/zstd_1_3_8/zstd_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..54b2aec739fdc28fe6ea3a09725c6e31f03f2d5c --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_internal.h @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* + * Dependencies + ***************************************/ +#include "compiler.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ + +#if defined(__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + +/*-************************************* + * shared macros + ***************************************/ +#undef MIN +#undef MAX +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define CHECK_F(f) \ + { \ + size_t const errcod = f; \ + if (ERR_isError(errcod)) \ + return errcod; \ + } /* check and Forward error code */ +#define CHECK_E(f, e) \ + { \ + size_t const errcod = f; \ + if (ERR_isError(errcod)) \ + return ERROR(e); \ + } /* check and send Error code */ + +/*-************************************* + * Common constants + ***************************************/ +#define ZSTD_OPT_NUM (1 << 12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM - 1) +static const U32 repStartValue[ZSTD_REP_NUM] = {1, 4, 8}; + +#define KB *(1 << 10) +#define MB *(1 << 20) +#define GB *(1U << 30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTD_fcs_fieldSize[4] = {0, 2, 4, 8}; +static const size_t ZSTD_did_fieldSize[4] = {0, 1, 2, 4}; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE \ + 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE \ + (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define MaxLit ((1 << Litbits) - 1) +#define MaxML 52 +#define MaxLL 35 +#define DefaultMaxOff 28 +#define MaxOff 31 +#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ +#define MLFSELog 9 +#define LLFSELog 9 +#define OffFSELog 8 +#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog) + +static const U32 LL_bits[MaxLL + 1] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; +static const S16 LL_defaultNorm[MaxLL + 1] = { + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1}; +#define LL_DEFAULTNORMLOG 6 /* for static allocation */ +static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG; + +static const U32 ML_bits[MaxML + 1] = {0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16}; +static const S16 ML_defaultNorm[MaxML + 1] = {1, + 4, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + -1, + -1, + -1, + -1, + -1, + -1, + -1}; +#define ML_DEFAULTNORMLOG 6 /* for static allocation */ +static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG; + +static const S16 OF_defaultNorm[DefaultMaxOff + 1] = { + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}; +#define OF_DEFAULTNORMLOG 5 /* for static allocation */ +static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; + +/*-******************************************* + * Shared functions to include for inlining + *********************************************/ +static void ZSTD_copy8(void* dst, const void* src) +{ + memcpy(dst, src, 8); +} +#define COPY8(d, s) \ + { \ + ZSTD_copy8(d, s); \ + d += 8; \ + s += 8; \ + } + +/*! ZSTD_wildcopy() : + * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ +#define WILDCOPY_OVERLENGTH 8 +MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + do + COPY8(op, ip) + while (op < oend); +} + +MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, + void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = (BYTE*)dstEnd; + do + COPY8(op, ip) + while (op < oend); +} + +/*-******************************************* + * Private declarations + *********************************************/ +typedef struct seqDef_s { + U32 offset; + U16 litLength; + U16 matchLength; +} seqDef; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; + BYTE* litStart; + BYTE* lit; + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ + U32 longLengthPos; +} seqStore_t; + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +void ZSTD_seqToCodes( + const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + +/* custom memory allocation functions */ +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem); +void* ZSTD_calloc(size_t size, ZSTD_customMem customMem); +void ZSTD_free(void* ptr, ZSTD_customMem customMem); + +MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + { +#if defined(_MSC_VER) /* Visual */ + unsigned long r = 0; + _BitScanReverse(&r, val); + return (unsigned)r; +#elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +#else /* Software version */ + static const U32 DeBruijnClz[32] = {0, + 9, + 1, + 10, + 13, + 21, + 2, + 29, + 11, + 14, + 16, + 18, + 22, + 25, + 3, + 30, + 8, + 12, + 20, + 28, + 15, + 17, + 24, + 7, + 19, + 27, + 23, + 6, + 26, + 5, + 4, + 31}; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; +#endif + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes( + ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_lazy.c b/src/lib/compress/zstd_1_3_8/zstd_lazy.c new file mode 100644 index 0000000000000000000000000000000000000000..ae969180412e75a27c426378e81bf0005d3d3201 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_lazy.c @@ -0,0 +1,1105 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" + +/*-************************************* + * Binary Tree search + ***************************************/ + +static void ZSTD_updateDUBT(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend, U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for (; idx < target; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2 * (idx & btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : current >= btlow == (current - btmask) + * doesn't fail */ +static void ZSTD_insertDUBT1( + ZSTD_matchState_t* ms, U32 current, const BYTE* inputEnd, U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (current >= dictLimit) ? base + current : dictBase + current; + const BYTE* const iend = (current >= dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2 * (current & btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = + *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while + *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = ms->window.lowLimit; + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", current, dictLimit, windowLow); + assert(current >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2 * (matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < current); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ((dictMode != ZSTD_extDict) || (matchIndex + matchLength >= dictLimit) /* both in current segment*/ + || (current < dictLimit) /* both in extDict */) { + const BYTE* const mBase = + ((dictMode != ZSTD_extDict) || (matchIndex + matchLength >= dictLimit)) ? base : dictBase; + assert((matchIndex + matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (current < dictLimit)); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip + matchLength, match + matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); + if (matchIndex + matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG( + 8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", current, matchIndex, (U32)matchLength); + + if (ip + matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { + smallerPtr = &dummy32; + break; + } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr + 1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { + largerPtr = &dummy32; + break; + } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; +} + +static size_t ZSTD_DUBT_findBetterDictMatch(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, size_t bestLength, U32 nbCompares, U32 const mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32* const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const current = (U32)(ip - base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller = 0, commonLengthLarger = 0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + while (nbCompares-- && (dictMatchIndex > dictLowLimit)) { + U32* const nextPtr = dictBt + 2 * (dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex + matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ((4 * (int)(matchLength - bestLength)) > + (int)(ZSTD_highbit32(current - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1))) { + DEBUGLOG(9, + "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u " + "(dictMatchIndex %u, matchIndex %u)", + current, + (U32)bestLength, + (U32)matchLength, + (U32)*offsetPtr, + ZSTD_REP_MOVE + current - matchIndex, + dictMatchIndex, + matchIndex); + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + } + if (ip + matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's + larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { + break; + } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { + break; + } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); + (void)mIndex; + DEBUGLOG(8, + "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + current, + (U32)bestLength, + (U32)*offsetPtr, + mIndex); + } + return bestLength; +} + +static size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, U32 const mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const current = (U32)(ip - base); + U32 const windowLow = ms->window.lowLimit; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2 * (matchIndex & btMask); + U32* unsortedMark = bt + 2 * (matchIndex & btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", current); + assert(ip <= iend - 8); /* required for h calculation */ + + /* reach end of unsorted candidates list */ + while ((matchIndex > unsortLimit) && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) && (nbCandidates > 1)) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", matchIndex); + *unsortedMark = + previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2 * (matchIndex & btMask); + unsortedMark = bt + 2 * (matchIndex & btMask) + 1; + nbCandidates--; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ((matchIndex > unsortLimit) && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK)) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2 * (matchIndex & btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { + size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2 * (current & btMask); + U32* largerPtr = bt + 2 * (current & btMask) + 1; + U32 matchEndIdx = current + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2 * (matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex + matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip + matchLength, match + matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); + if (matchIndex + matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ((4 * (int)(matchLength - bestLength)) > + (int)(ZSTD_highbit32(current - matchIndex + 1) - ZSTD_highbit32((U32)offsetPtr[0] + 1))) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + if (ip + matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { + smallerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { + largerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; + + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, bestLength, nbCompares, mls, dictMode); + } + + assert(matchEndIdx > current + 8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); + (void)mIndex; + DEBUGLOG(8, + "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + current, + (U32)bestLength, + (U32)*offsetPtr, + mIndex); + } + return bestLength; + } +} + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, const U32 mls /* template */, const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); +} + +static size_t ZSTD_BtFindBestMatch_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); + case 7: + case 6: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + +static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7: + case 6: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + +static size_t ZSTD_BtFindBestMatch_extDict_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7: + case 6: + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + } +} + +/* ********************************* + * Hash Chain + ***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams, const BYTE* ip, U32 const mls) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while (idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); +} + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch_generic(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize - 1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 lowLimit = ms->window.lowLimit; + const U32 current = (U32)(ip - base); + const U32 minChain = current > chainSize ? current - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml = 4 - 1; + + /* HC4 match finder */ + U32 matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + + for (; (matchIndex > lowLimit) & (nbAttempts > 0); nbAttempts--) { + size_t currentMl = 0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match + 4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip + 4, match + 4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; + if (ip + currentMl == iLimit) + break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) + break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + if (dictMode == ZSTD_dictMatchState) { + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for (; (matchIndex > dmsLowestIndex) & (nbAttempts > 0); nbAttempts--) { + size_t currentMl = 0; + const BYTE* const match = dmsBase + matchIndex; + assert(match + 4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip + 4, match + 4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = current - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; + if (ip + currentMl == iLimit) + break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) + break; + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); + case 5: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict); + case 7: + case 6: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict); + } +} + +static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); + case 5: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState); + case 7: + case 6: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState); + } +} + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS( + ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) +{ + switch (ms->cParams.minMatch) { + default: /* includes case 3 */ + case 4: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); + case 5: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict); + case 7: + case 6: + return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict); + } +} + +/* ******************************* + * Common parser - lazy strategy + *********************************/ +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const U32 searchMethod, const U32 depth, ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + + typedef size_t (*searchMax_f)(ZSTD_matchState_t * ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + searchMax_f const searchMax = dictMode == ZSTD_dictMatchState + ? (searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS + : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) + : (searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS); + U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset = 0; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = dictMode == ZSTD_dictMatchState ? dms->window.dictLimit : 0; + const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dictLowest = dictMode == ZSTD_dictMatchState ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? prefixLowestIndex - (U32)(dictEnd - dictBase) : 0; + const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest); + + /* init */ + ip += (dictAndPrefixLength == 0); + ms->nextToUpdate3 = ms->nextToUpdate; + if (dictMode == ZSTD_noDict) { + U32 const maxRep = (U32)(ip - prefixLowest); + if (offset_2 > maxRep) + savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) + savedOffset = offset_1, offset_1 = 0; + } + if (dictMode == ZSTD_dictMatchState) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength = 0; + size_t offset = 0; + const BYTE* start = ip + 1; + + /* check repCode */ + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = (dictMode == ZSTD_dictMatchState && repIndex < prefixLowestIndex) + ? dictBase + (repIndex - dictIndexDelta) + : base + repIndex; + if (((U32)((prefixLowestIndex - 1) - repIndex) >= 3 /* intentional underflow */) && + (MEM_read32(repMatch) == MEM_read32(ip + 1))) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixLowest) + 4; + if (depth == 0) + goto _storeSequence; + } + } + if (dictMode == ZSTD_noDict && ((offset_1 > 0) & (MEM_read32(ip + 1 - offset_1) == MEM_read32(ip + 1)))) { + matchLength = ZSTD_count(ip + 1 + 4, ip + 1 + 4 - offset_1, iend) + 4; + if (depth == 0) + goto _storeSequence; + } + + /* first search (depth 0) */ + { + size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset = offsetFound; + } + + if (matchLength < 4) { + ip += ((ip - anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth >= 1) + while (ip < ilimit) { + ip++; + if ((dictMode == ZSTD_noDict) && (offset) && ((offset_1 > 0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip + 4, ip + 4 - offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = + repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; + if (((U32)((prefixLowestIndex - 1) - repIndex) >= 3 /* intentional underflow */) && + (MEM_read32(repMatch) == MEM_read32(ip))) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + } + { + size_t offset2 = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } + } + + /* let's find an even better one */ + if ((depth == 2) && (ip < ilimit)) { + ip++; + if ((dictMode == ZSTD_noDict) && (offset) && + ((offset_1 > 0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip + 4, ip + 4 - offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + if (dictMode == ZSTD_dictMatchState) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = + repIndex < prefixLowestIndex ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; + if (((U32)((prefixLowestIndex - 1) - repIndex) >= 3 /* intentional underflow */) && + (MEM_read32(repMatch) == MEM_read32(ip))) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + } + { + size_t offset2 = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } + } + } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. + * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which + * overflows the pointer, which is undefined behavior. + */ + /* catch up */ + if (offset) { + if (dictMode == ZSTD_noDict) { + while (((start > anchor) & (start - (offset - ZSTD_REP_MOVE) > prefixLowest)) && + (start[-1] == (start - (offset - ZSTD_REP_MOVE))[-1])) /* only search for offset within prefix */ + { + start--; + matchLength++; + } + } + if (dictMode == ZSTD_dictMatchState) { + U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = + (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) { + start--; + match--; + matchLength++; + } /* catch up */ + } + offset_2 = offset_1; + offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + /* store sequence */ + _storeSequence : { + size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength - MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + if (dictMode == ZSTD_dictMatchState) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip - base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = dictMode == ZSTD_dictMatchState && repIndex < prefixLowestIndex + ? dictBase - dictIndexDelta + repIndex + : base + repIndex; + if (((U32)((prefixLowestIndex - 1) - (U32)repIndex) >= 3 /* intentional overflow */) && + (MEM_read32(repMatch) == MEM_read32(ip))) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repEnd2, prefixLowest) + 4; + offset = offset_2; + offset_2 = offset_1; + offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength - MINMATCH); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while (((ip <= ilimit) & (offset_2 > 0)) && (MEM_read32(ip) == MEM_read32(ip - offset_2))) { + /* store sequence */ + matchLength = ZSTD_count(ip + 4, ip + 4 - offset_2, iend) + 4; + offset = offset_2; + offset_2 = offset_1; + offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength - MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } + + /* Save reps for next block */ + rep[0] = offset_1 ? offset_1 : savedOffset; + rep[1] = offset_2 ? offset_2 : savedOffset; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_dictMatchState); +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const U32 searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const U32 lowestIndex = ms->window.lowLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + + typedef size_t (*searchMax_f)(ZSTD_matchState_t * ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); + searchMax_f searchMax = + searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + + U32 offset_1 = rep[0], offset_2 = rep[1]; + + /* init */ + ms->nextToUpdate3 = ms->nextToUpdate; + ip += (ip == prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength = 0; + size_t offset = 0; + const BYTE* start = ip + 1; + U32 current = (U32)(ip - base); + + /* check repCode */ + { + const U32 repIndex = (U32)(current + 1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip + 1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip + 1 + 4, repMatch + 4, iend, repEnd, prefixStart) + 4; + if (depth == 0) + goto _storeSequence; + } + } + + /* first search (depth 0) */ + { + size_t offsetFound = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset = offsetFound; + } + + if (matchLength < 4) { + ip += ((ip - anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth >= 1) + while (ip < ilimit) { + ip++; + current++; + /* check repCode */ + if (offset) { + const U32 repIndex = (U32)(current - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength * 3 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } + } + + /* search match, depth 1 */ + { + size_t offset2 = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } + } + + /* let's find an even better one */ + if ((depth == 2) && (ip < ilimit)) { + ip++; + current++; + /* check repCode */ + if (offset) { + const U32 repIndex = (U32)(current - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } + } + + /* search match, depth 2 */ + { + size_t offset2 = 999999999; + size_t const ml2 = searchMax(ms, ip, iend, &offset2); + int const gain2 = (int)(ml2 * 4 - ZSTD_highbit32((U32)offset2 + 1)); /* raw approx */ + int const gain1 = (int)(matchLength * 4 - ZSTD_highbit32((U32)offset + 1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } + } + } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + U32 const matchIndex = (U32)((start - base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start > anchor) && (match > mStart) && (start[-1] == match[-1])) { + start--; + match--; + matchLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ + _storeSequence : { + size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength - MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repIndex = (U32)((ip - base) - offset_2); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit - 1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip + 4, repMatch + 4, iend, repEnd, prefixStart) + 4; + offset = offset_2; + offset_2 = offset_1; + offset_1 = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength - MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } + } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 1, 2); +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_lazy.h b/src/lib/compress/zstd_1_3_8/zstd_lazy.h new file mode 100644 index 0000000000000000000000000000000000000000..d32cb541d9c665fd2d8a0cd6cb99872a71ea59f0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_lazy.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); + +void ZSTD_preserveUnsortedMark(U32* const table, U32 const size, + U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */ + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_ldm.c b/src/lib/compress/zstd_1_3_8/zstd_ldm.c new file mode 100644 index 0000000000000000000000000000000000000000..23848f43a7a2f8b6f72086151907bcca72b74cb0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_ldm.c @@ -0,0 +1,547 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#include "zstd_ldm.h" + +#include "debug.h" +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 +#define LDM_HASH_CHAR_OFFSET 10 + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) + params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) + params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (cParams->strategy >= ZSTD_btopt) { + /* Get out of the way of the optimal parser */ + U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength); + assert(minMatch >= ZSTD_LDM_MINMATCH_MIN); + assert(minMatch <= ZSTD_LDM_MINMATCH_MAX); + params->minMatchLength = minMatch; + } + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog ? 0 : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + return params.enableLdm ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm ? (maxChunkSize / params.minMatchLength) : 0; +} + +/** ZSTD_ldm_getSmallHash() : + * numBits should be <= 32 + * If numBits==0, returns 0. + * @return : the most significant numBits of value. */ +static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits) +{ + assert(numBits <= 32); + return numBits == 0 ? 0 : (U32)(value >> (64 - numBits)); +} + +/** ZSTD_ldm_getChecksum() : + * numBitsToDiscard should be <= 32 + * @return : the next most significant 32 bits after numBitsToDiscard */ +static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard) +{ + assert(numBitsToDiscard <= 32); + return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF; +} + +/** ZSTD_ldm_getTag() ; + * Given the hash, returns the most significant numTagBits bits + * after (32 + hbits) bits. + * + * If there are not enough bits remaining, return the last + * numTagBits bits. */ +static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits) +{ + assert(numTagBits < 32 && hbits <= 32); + if (32 - hbits < numTagBits) { + return hash & (((U32)1 << numTagBits) - 1); + } else { + return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1); + } +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket(ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry( + ldmState_t* ldmState, size_t const hash, const ldmEntry_t entry, ldmParams_t const ldmParams) +{ + BYTE* const bucketOffsets = ldmState->bucketOffsets; + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry; + bucketOffsets[hash]++; + bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1; +} + +/** ZSTD_ldm_makeEntryAndInsertByTag() : + * + * Gets the small hash, checksum, and tag from the rollingHash. + * + * If the tag matches (1 << ldmParams.hashRateLog)-1, then + * creates an ldmEntry from the offset, and inserts it into the hash table. + * + * hBits is the length of the small hash, which is the most significant hBits + * of rollingHash. The checksum is the next 32 most significant bits, followed + * by ldmParams.hashRateLog bits that make up the tag. */ +static void ZSTD_ldm_makeEntryAndInsertByTag( + ldmState_t* ldmState, U64 const rollingHash, U32 const hBits, U32 const offset, ldmParams_t const ldmParams) +{ + U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog); + U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1; + if (tag == tagMask) { + U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits); + U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); + ldmEntry_t entry; + entry.offset = offset; + entry.checksum = checksum; + ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams); + } +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch(const BYTE* pIn, const BYTE* pAnchor, const BYTE* pMatch, const BYTE* pBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch (ms->cParams.strategy) { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +/** ZSTD_ldm_fillLdmHashTable() : + * + * Fills hashTable from (lastHashed + 1) to iend (non-inclusive). + * lastHash is the rolling hash that corresponds to lastHashed. + * + * Returns the rolling hash corresponding to position iend-1. */ +static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state, U64 lastHash, const BYTE* lastHashed, const BYTE* iend, + const BYTE* base, U32 hBits, ldmParams_t const ldmParams) +{ + U64 rollingHash = lastHash; + const BYTE* cur = lastHashed + 1; + + while (cur < iend) { + rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1], cur[ldmParams.minMatchLength - 1], state->hashPower); + ZSTD_ldm_makeEntryAndInsertByTag(state, rollingHash, hBits, (U32)(cur - base), ldmParams); + ++cur; + } + return rollingHash; +} + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const current = (U32)(anchor - ms->window.base); + if (current > ms->nextToUpdate + 1024) { + ms->nextToUpdate = current - MIN(512, current - ms->nextToUpdate - 1024); + } +} + +static size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U64 const hashPower = ldmState->hashPower; + U32 const hBits = params->hashLog - params->bucketSizeLog; + U32 const ldmBucketSize = 1U << params->bucketSizeLog; + U32 const hashRateLog = params->hashRateLog; + U32 const ldmTagMask = (1U << params->hashRateLog) - 1; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - MAX(minMatchLength, HASH_READ_SIZE); + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash */ + BYTE const* lastHashed = NULL; + U64 rollingHash = 0; + + while (ip <= ilimit) { + size_t mLength; + U32 const current = (U32)(ip - base); + size_t forwardMatchLength = 0, backwardMatchLength = 0; + ldmEntry_t* bestEntry = NULL; + if (ip != istart) { + rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0], lastHashed[minMatchLength], hashPower); + } else { + rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength); + } + lastHashed = ip; + + /* Do not insert and do not look for a match */ + if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) { + ip++; + continue; + } + + /* Get the best entry and compute the match lengths */ + { + ldmEntry_t* const bucket = ZSTD_ldm_getBucket(ldmState, ZSTD_ldm_getSmallHash(rollingHash, hBits), *params); + ldmEntry_t* cur; + size_t bestMatchLength = 0; + U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); + + for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) { + size_t curForwardMatchLength, curBackwardMatchLength, curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = cur->offset < dictLimit ? dictStart : lowPrefixPtr; + + curForwardMatchLength = ZSTD_count_2segments(ip, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch, lowMatchPtr); + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(ip, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch(ip, anchor, pMatch, lowPrefixPtr); + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + } + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + } + + /* No match found -- continue searching */ + if (bestEntry == NULL) { + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits, current, *params); + ip++; + continue; + } + + /* Match found */ + mLength = forwardMatchLength + backwardMatchLength; + ip -= backwardMatchLength; + + { + /* Store the sequence: + * ip = current - backwardMatchLength + * The match is at (bestEntry->offset - backwardMatchLength) + */ + U32 const matchIndex = bestEntry->offset; + U32 const offset = current - matchIndex; + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(ip - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table */ + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits, (U32)(lastHashed - base), *params); + + assert(ip + backwardMatchLength == lastHashed); + + /* Fill the hash table from lastHashed+1 to ip+mLength*/ + /* Heuristic: don't need to fill the entire table at end of block */ + if (ip + mLength <= ilimit) { + rollingHash = ZSTD_ldm_fillLdmHashTable(ldmState, rollingHash, lastHashed, ip + mLength, base, hBits, *params); + lastHashed = ip + mLength - 1; + } + ip += mLength; + anchor = ip; + } + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) + table[u].offset = 0; + else + table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximmum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const* const chunkEnd = (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow(&ldmState->window, /* cycleLog */ 0, maxDist, src); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * the offset against maxDist directly. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, NULL, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal(ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the lits */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, (U32)(iend - ip), minMatch); + int i; + /* End signal */ + if (sequence.offset == 0) + break; + + assert(sequence.offset <= (1U << cParams->windowLog)); + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength); + { + size_t const newLitLength = blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i - 1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq( + seqStore, newLitLength, ip - newLitLength, sequence.offset + ZSTD_REP_MOVE, sequence.matchLength - MINMATCH); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/src/lib/compress/zstd_1_3_8/zstd_ldm.h b/src/lib/compress/zstd_1_3_8/zstd_ldm.h new file mode 100644 index 0000000000000000000000000000000000000000..f9f6df210660e650b36b124615970e91400425b9 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_ldm.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include "zstd.h" /* ZSTD_CCtx, size_t */ + +/*-************************************* + * Long distance matching + ***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_compressionParameters const* cParams); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_opt.c b/src/lib/compress/zstd_1_3_8/zstd_opt.c new file mode 100644 index 0000000000000000000000000000000000000000..8ee4d30bf9a44b70d3dfc95702f73e08990b9c74 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_opt.c @@ -0,0 +1,1307 @@ +/* + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ +#define ZSTD_MAX_PRICE (1 << 30) + +#define ZSTD_PREDEF_THRESHOLD \ + 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined \ + distributions */ + +/*-************************************* + * Price functions for optimal parser + ***************************************/ + +#if 0 /* approximation at bit level */ +#define BITCOST_ACCURACY 0 +#define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +#define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy */ +#define BITCOST_ACCURACY 8 +#define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +#define WEIGHT(stat, opt) ((void)opt, ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +#define BITCOST_ACCURACY 8 +#define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +#define WEIGHT(stat, opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat + 1) * BITCOST_MULTIPLIER); +} + +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL >= 2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(U32 price) +{ + return (double)price / (BITCOST_MULTIPLIER * 8); +} +#endif + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + +/* ZSTD_downscaleStat() : + * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) + * return the resulting sum of elements */ +static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) +{ + U32 s, sum = 0; + DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex + 1); + assert(ZSTD_FREQ_DIV + malus > 0 && ZSTD_FREQ_DIV + malus < 31); + for (s = 0; s < lastEltIndex + 1; s++) { + table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV + malus)); + sum += table[s]; + } + return sum; +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * or init from zero, using src for literals stats, or flat 1 for match symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void ZSTD_rescaleFreqs(optState_t* const optPtr, const BYTE* const src, size_t const srcSize, int const optLevel) +{ + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* first block : init */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ + DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + /* huffman table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + { + unsigned lit; + for (lit = 0; lit <= MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog - bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } + } + + { + unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll = 0; ll <= MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog - bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } + } + + { + unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml = 0; ml <= MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog - bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } + } + + { + unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of = 0; of <= MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog - bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } + } + + } else { /* not a dictionary */ + + assert(optPtr->litFreq != NULL); + { + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + } + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + + { + unsigned ll; + for (ll = 0; ll <= MaxLL; ll++) + optPtr->litLengthFreq[ll] = 1; + } + optPtr->litLengthSum = MaxLL + 1; + + { + unsigned ml; + for (ml = 0; ml <= MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML + 1; + + { + unsigned of; + for (of = 0; of <= MaxOff; of++) + optPtr->offCodeFreq[of] = 1; + } + optPtr->offCodeSum = MaxOff + 1; + } + + } else { /* new block : re-use previous statistics, scaled down */ + + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); + optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); + optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost( + const BYTE* const literals, U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + if (litLength == 0) + return 0; + if (optPtr->priceType == zop_predef) + return (litLength * 6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { + U32 price = litLength * optPtr->litSumBasePrice; + U32 u; + for (u = 0; u < litLength; u++) { + assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= + optPtr->litSumBasePrice); /* literal cost should never be negative */ + price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + + /* dynamic statistics */ + { + U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + optPtr->litLengthSumBasePrice - + WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_litLengthContribution() : + * @return ( cost(litlength) - cost(0) ) + * this value can then be added to rawLiteralsCost() + * to provide a cost which is directly comparable to a match ending at same position */ +static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + if (optPtr->priceType >= zop_predef) + return WEIGHT(litLength, optLevel); + + /* dynamic statistics */ + { + U32 const llCode = ZSTD_LLcode(litLength); + int const contribution = (LL_bits[llCode] * BITCOST_MULTIPLIER) + + WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); +#if 1 + return contribution; +#else + return MAX(0, contribution); /* sometimes better, sometimes not ... */ +#endif + } +} + +/* ZSTD_literalsContribution() : + * creates a fake cost for the literals part of a sequence + * which can be compared to the ending cost of a match + * should a new match start at this position */ +static int ZSTD_literalsContribution( + const BYTE* const literals, U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + + ZSTD_litLengthContribution(litLength, optPtr, optLevel); + return contribution; +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ +FORCE_INLINE_TEMPLATE U32 ZSTD_getMatchPrice( + U32 const offset, U32 const matchLength, const optState_t* const optPtr, int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offset + 1); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ + return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); + + /* dynamic statistics */ + price = + (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel < 2) /*static*/ && offCode >= 20) + price += + (offCode - 19) * 2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { + U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / + 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLengtn <= iend */ +static void ZSTD_updateStats( + optState_t* const optPtr, U32 litLength, const BYTE* literals, U32 offsetCode, U32 matchLength) +{ + /* literals */ + { + U32 u; + for (u = 0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength * ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { + U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* match offset code (0-2=>repCode; 3+=>offset+2) */ + { + U32 const offCode = ZSTD_highbit32(offsetCode + 1); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { + U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) { + default: + case 4: + return MEM_read32(memPtr); + case 3: + if (MEM_isLittleEndian()) + return MEM_read32(memPtr) << 8; + else + return MEM_read32(memPtr) >> 8; + } +} + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = ms->nextToUpdate3; + U32 const target = ms->nextToUpdate3 = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while (idx < target) { + hashTable3[ZSTD_hash3Ptr(base + idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + +/*-************************************* + * Binary Tree search + ***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * ip : assumed <= iend-8 . + * @return : nb of positions added */ +static U32 ZSTD_insertBt1( + ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iend, U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 current = (U32)(ip - base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + U32* smallerPtr = bt + 2 * (current & btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = ms->window.lowLimit; + U32 matchEndIdx = current + 8 + 1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2 * ((current - 1) & btMask) + 0); + U32 predictedLarge = *(bt + 2 * ((current - 1) & btMask) + 1); + predictedSmall += (predictedSmall > 0); + predictedLarge += (predictedLarge > 0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", current); + + assert(ip <= iend - 8); /* required for h calculation */ + hashTable[h] = current; /* Update Hash Table */ + + assert(windowLow > 0); + while (nbCompares-- && (matchIndex >= windowLow)) { + U32* const nextPtr = bt + 2 * (matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < current); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2 * ((matchIndex - 1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { + smallerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + smallerPtr = nextPtr + 1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1] > 0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { + largerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0] > 0); + continue; + } +#endif + + if (!extDict || (matchIndex + matchLength >= dictLimit)) { + assert(matchIndex + matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip + matchLength, match + matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iend, dictEnd, prefixStart); + if (matchIndex + matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip + matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { + smallerPtr = &dummy32; + break; + } /* beyond tree size, stop searching */ + smallerPtr = nextPtr + 1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { + largerPtr = &dummy32; + break; + } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; + if (bestLength > 384) + return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > current + 8); + return matchEndIdx - (current + 8); +} + +FORCE_INLINE_TEMPLATE +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iend, const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); + + while (idx < target) + idx += ZSTD_insertBt1(ms, base + idx, iend, mls, dictMode == ZSTD_extDict); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) +{ + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE +U32 ZSTD_insertBtAndGetAllMatches(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, U32 rep[ZSTD_REP_NUM], + U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM - 1); + const BYTE* const base = ms->window.base; + U32 const current = (U32)(ip - base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls == 3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1U << btLog) - 1; + size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = btMask >= current ? 0 : current - btMask; + U32 const windowLow = ms->window.lowLimit; + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2 * (current & btMask); + U32* largerPtr = bt + 2 * (current & btMask) + 1; + U32 matchEndIdx = current + 8 + 1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit + ? dmsHighLimit - dmsBtMask + : dmsLowLimit; + + size_t bestLength = lengthToBeat - 1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { + U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode == ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = current - repOffset; + U32 repLen = 0; + assert(current >= dictLimit); + if (repOffset - 1 /* intentional overflow, discards 0 and -1 */ < + current - dictLimit) { /* equivalent to `current > repIndex >= dictLimit` */ + if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch)) { + repLen = (U32)ZSTD_count(ip + minMatch, ip + minMatch - repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= current */ + const BYTE* const repMatch = + dictMode == ZSTD_dictMatchState ? dmsBase + repIndex - dmsIndexDelta : dictBase + repIndex; + assert(current >= windowLow); + if (dictMode == ZSTD_extDict && + (((repOffset - 1) /*intentional overflow*/ < + current - windowLow) /* equivalent to `current > repIndex >= windowLow` */ + & (((U32)((dictLimit - 1) - repIndex) >= + 3)) /* intentional overflow : do not test positions overlapping 2 memory segments */) && + (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { + repLen = + (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState && + (((repOffset - 1) /*intentional overflow*/ < + current - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `current > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit - 1) - repIndex) >= + 3)) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch))) { + repLen = + (U32)ZSTD_count_2segments(ip + minMatch, repMatch + minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } + } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = repCode - ll0; + matches[mnum].len = (U32)repLen; + mnum++; + if ((repLen > sufficient_len) | (ip + repLen == iLimit)) { /* best possible */ + return mnum; + } + } + } + } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + if ((matchIndex3 >= matchLow) & + (current - matchIndex3 < (1 << 18)) /*heuristic : longer distance likely too expensive*/) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || + (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", (U32)mlen); + bestLength = mlen; + assert(current > matchIndex3); + assert(mnum == 0); /* no prior solution */ + matches[0].off = (current - matchIndex3) + ZSTD_REP_MOVE; + matches[0].len = (U32)mlen; + mnum = 1; + if ((mlen > sufficient_len) | (ip + mlen == iLimit)) { /* best possible length */ + ms->nextToUpdate = current + 1; /* skip insertion */ + return 1; + } + } + } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex >= matchLow)) { + U32* const nextPtr = bt + 2 * (matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + assert(current > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex + matchLength >= dictLimit)) { + assert(matchIndex + matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip + matchLength, match + matchLength, iLimit); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex + matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, + "found match of length %u at distance %u (offCode=%u)", + (U32)matchLength, + current - matchIndex, + current - matchIndex + ZSTD_REP_MOVE); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].len = (U32)matchLength; + mnum++; + if ((matchLength > ZSTD_OPT_NUM) | (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) + nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { + smallerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + smallerPtr = nextPtr + 1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { + largerPtr = &dummy32; + break; + } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } + } + + *smallerPtr = *largerPtr = 0; + + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + while (nbCompares-- && (dictMatchIndex > dmsLowLimit)) { + const U32* const nextPtr = dmsBt + 2 * (dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip + matchLength, match + matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex + matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, + "found dms match of length %u at distance %u (offCode=%u)", + (U32)matchLength, + current - matchIndex, + current - matchIndex + ZSTD_REP_MOVE); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = (current - matchIndex) + ZSTD_REP_MOVE; + matches[mnum].len = (U32)matchLength; + mnum++; + if ((matchLength > ZSTD_OPT_NUM) | (ip + matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (dictMatchIndex <= dmsBtLow) { + break; + } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + } + + assert(matchEndIdx > current + 8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + +FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iHighLimit, + const ZSTD_dictMode_e dictMode, U32 rep[ZSTD_REP_NUM], U32 const ll0, ZSTD_match_t* matches, U32 const lengthToBeat) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const matchLengthSearch = cParams->minMatch; + DEBUGLOG(8, "ZSTD_BtGetAllMatches"); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); + switch (matchLengthSearch) { + case 3: + return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3); + default: + case 4: + return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4); + case 5: + return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5); + case 7: + case 6: + return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6); + } +} + +/*-******************************* + * Optimal parser + *********************************/ +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +static repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) +{ + repcodes_t newReps; + if (offset >= ZSTD_REP_NUM) { /* full offset */ + newReps.rep[2] = rep[1]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = offset - ZSTD_REP_MOVE; + } else { /* repcode */ + U32 const repCode = offset + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode == ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = currentOffset; + } else { /* repCode == 0 */ + memcpy(&newReps, rep, sizeof(newReps)); + } + } + return newReps; +} + +static U32 ZSTD_totalLen(ZSTD_optimal_t sol) +{ + return sol.litlen + sol.mlen; +} + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + //RAWLOG(2, "%3i:%3i, ", enb, table[enb]); + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, const int optLevel, const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM - 1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastSequence; + + /* init */ + DEBUGLOG(5, + "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), + ms->window.dictLimit, + ms->nextToUpdate); + assert(optLevel <= 2); + ms->nextToUpdate3 = ms->nextToUpdate; + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip == prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { + U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch); + if (!nbMatches) { + ip++; + continue; + } + + /* initialize opt[0] */ + { + U32 i; + for (i = 0; i < ZSTD_REP_NUM; i++) + opt[0].rep[i] = rep[i]; + } + opt[0].mlen = 0; /* means is_a_literal */ + opt[0].litlen = litlen; + opt[0].price = ZSTD_literalsContribution(anchor, litlen, optStatePtr, optLevel); + + /* large match -> immediate encoding */ + { + U32 const maxML = matches[nbMatches - 1].len; + U32 const maxOffset = matches[nbMatches - 1].off; + DEBUGLOG(6, + "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new serie", + nbMatches, + maxML, + maxOffset, + (U32)(ip - prefixStart)); + + if (maxML > sufficient_len) { + lastSequence.litlen = litlen; + lastSequence.mlen = maxML; + lastSequence.off = maxOffset; + DEBUGLOG(6, "large match (%u>%u), immediate encoding", maxML, sufficient_len); + cur = 0; + last_pos = ZSTD_totalLen(lastSequence); + goto _shortestPath; + } + } + + /* set prices for first matches starting position == 0 */ + { + U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const end = matches[matchNb].len; + repcodes_t const repHistory = ZSTD_updateRep(rep, offset, ll0); + for (; pos <= end; pos++) { + U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); + U32 const sequencePrice = literalsPrice + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", pos, ZSTD_fCost(sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = sequencePrice; + ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory)); + memcpy(opt[pos].rep, &repHistory, sizeof(repHistory)); + } + } + last_pos = pos - 1; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur < ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr - istart, cur) + + /* Fix current position with one literal if cheaper */ + { + U32 const litlen = (opt[cur - 1].mlen == 0) ? opt[cur - 1].litlen + 1 : 1; + int const price = opt[cur - 1].price + ZSTD_rawLiteralsCost(ip + cur - 1, 1, optStatePtr, optLevel) + + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) - + ZSTD_litLengthPrice(litlen - 1, optStatePtr, optLevel); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + DEBUGLOG(7, + "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr - istart, + cur, + ZSTD_fCost(price), + ZSTD_fCost(opt[cur].price), + litlen, + opt[cur - 1].rep[0], + opt[cur - 1].rep[1], + opt[cur - 1].rep[2]); + opt[cur].mlen = 0; + opt[cur].off = 0; + opt[cur].litlen = litlen; + opt[cur].price = price; + memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(opt[cur].rep)); + } else { + DEBUGLOG(7, + "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", + inr - istart, + cur, + ZSTD_fCost(price), + ZSTD_fCost(opt[cur].price), + opt[cur].rep[0], + opt[cur].rep[1], + opt[cur].rep[2]); + } + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) + continue; + + if (cur == last_pos) + break; + + if ((optLevel == 0) /*static_test*/ + && (opt[cur + 1].price <= opt[cur].price + (BITCOST_MULTIPLIER / 2))) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur + 1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + { + U32 const ll0 = (opt[cur].mlen != 0); + U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; + U32 const previousPrice = opt[cur].price; + U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch); + U32 matchNb; + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { + U32 const maxML = matches[nbMatches - 1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", inr - istart, cur, nbMatches, maxML); + + if ((maxML > sufficient_len) || (cur + maxML >= ZSTD_OPT_NUM)) { + lastSequence.mlen = maxML; + lastSequence.off = matches[nbMatches - 1].off; + lastSequence.litlen = litlen; + cur -= (opt[cur].mlen == 0) ? opt[cur].litlen + : 0; /* last sequence is actually only literals, fix cur to last match - note : + may underflow, in which case, it's first sequence, and it's okay */ + last_pos = cur + ZSTD_totalLen(lastSequence); + if (cur > ZSTD_OPT_NUM) + cur = 0; /* underflow => first match */ + goto _shortestPath; + } + } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + repcodes_t const repHistory = ZSTD_updateRep(opt[cur].rep, offset, ll0); + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb > 0) ? matches[matchNb - 1].len + 1 : minMatch; + U32 mlen; + + DEBUGLOG( + 7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", matchNb, matches[matchNb].off, lastML, litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, + "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, + mlen, + ZSTD_fCost(price), + ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { + opt[last_pos + 1].price = ZSTD_MAX_PRICE; + last_pos++; + } /* fill empty positions */ + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = price; + ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory)); + memcpy(opt[pos].rep, &repHistory, sizeof(repHistory)); + } else { + DEBUGLOG(7, + "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, + mlen, + ZSTD_fCost(price), + ZSTD_fCost(opt[pos].price)); + if (optLevel == 0) + break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } + } + } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastSequence = opt[last_pos]; + cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) + : 0; /* single sequence, and it starts before `ip` */ + assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + + _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + + { + U32 const storeEnd = cur + 1; + U32 storeStart = storeEnd; + U32 seqPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", last_pos, cur); + (void)last_pos; + assert(storeEnd < ZSTD_OPT_NUM); + DEBUGLOG(6, + "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, + lastSequence.litlen, + lastSequence.mlen, + lastSequence.off); + opt[storeEnd] = lastSequence; + while (seqPos > 0) { + U32 const backDist = ZSTD_totalLen(opt[seqPos]); + storeStart--; + DEBUGLOG(6, + "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + seqPos, + storeStart, + opt[seqPos].litlen, + opt[seqPos].mlen, + opt[seqPos].off); + opt[storeStart] = opt[seqPos]; + seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore") + { + U32 storePos; + for (storePos = storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offCode = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG( + 6, "considering seq starting at %zi, llen=%u, mlen=%u", anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen == 0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + /* repcodes update : like ZSTD_updateRep(), but update in place */ + if (offCode >= ZSTD_REP_NUM) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offCode - ZSTD_REP_MOVE; + } else { /* repcode */ + U32 const repCode = offCode + (llen == 0); + if (repCode) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode == ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + if (repCode >= 2) + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, offCode, mlen - MINMATCH); + anchor += advance; + ip = anchor; + } + } + ZSTD_setBasePrices(optStatePtr, optLevel); + } + + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return iend - anchor; +} + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_noDict); +} + +/* used in 2-pass strategy */ +static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) +{ + U32 s, sum = 0; + assert(ZSTD_FREQ_DIV + bonus >= 0); + for (s = 0; s < lastEltIndex + 1; s++) { + table[s] <<= ZSTD_FREQ_DIV + bonus; + table[s]--; + sum += table[s]; + } + return sum; +} + +/* used in 2-pass strategy */ +MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr) +{ + optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); + optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); + optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); + optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); +} + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error, hence its constract must be respected. + */ +static void ZSTD_initStats_ultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= + 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt_generic( + ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + ms->nextToUpdate3 = ms->window.dictLimit; + + /* re-inforce weight of collected statistics */ + ZSTD_upscaleStats(&ms->opt); +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + U32 const current = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-pass strategy: + * this strategy makes a first pass over first block to collect statistics + * and seed next round's statistics with it. + * After 1st pass, function forgets everything, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + * the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ((ms->opt.litLengthSum == 0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD)) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /*optLevel*/, ZSTD_extDict); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); +} + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/src/lib/compress/zstd_1_3_8/zstd_opt.h b/src/lib/compress/zstd_1_3_8/zstd_opt.h new file mode 100644 index 0000000000000000000000000000000000000000..04bcc08e79e1a5ff92d7f517427788c9a67ad1f6 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstd_opt.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/src/lib/compress/zstd_1_3_8/zstdmt_compress.c b/src/lib/compress/zstd_1_3_8/zstdmt_compress.c new file mode 100644 index 0000000000000000000000000000000000000000..b19208eb13d2f280d578d5da2747e30fa9cd8e69 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstdmt_compress.c @@ -0,0 +1,2186 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +#pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + +/* ====== Constants ====== */ +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + +/* ====== Dependencies ====== */ +#include /* memcpy, memset */ +#include /* INT_MAX, UINT_MAX */ +#include "pool.h" /* threadpool */ +#include "threading.h" /* mutex */ +#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" +#include "zstdmt_compress.h" + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 2) && !defined(_MSC_VER) && !defined(__MINGW32__) + +#include +#include +#include + +#define DEBUG_PRINTHEX(l, p, n) \ + { \ + unsigned debug_u; \ + for (debug_u = 0; debug_u < (n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ + } + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) + _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { + struct tms junk; + clock_t newTicks = (clock_t)times(&junk); + return ((((unsigned long long)newTicks) * (1000000)) / _ticksPerSecond); + } +} + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) \ + { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { \ + unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime - beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG( \ + MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", elapsedTime, #mutex); \ + } \ + } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ + } + +#else + +#define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +#define DEBUG_PRINTHEX(l, p, n) \ + {} + +#endif + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} buffer_t; + +static const buffer_t g_nullBuffer = {NULL, 0}; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t bTable[1]; /* variable size */ +} ZSTDMT_bufferPool; + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + unsigned const maxNbBuffers = 2 * nbWorkers + 3; + ZSTDMT_bufferPool* const bufPool = + (ZSTDMT_bufferPool*)ZSTD_calloc(sizeof(ZSTDMT_bufferPool) + (maxNbBuffers - 1) * sizeof(buffer_t), cMem); + if (bufPool == NULL) + return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_free(bufPool, cMem); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + unsigned u; + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) + return; /* compatibility with free on NULL */ + for (u = 0; u < bufPool->totalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); + ZSTD_free(bufPool->bTable[u].start, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_free(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u = 0; u < bufPool->totalBuffers; u++) + totalBufferSize += bufPool->bTable[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + totalBufferSize; +} + +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) +{ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); +} + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, U32 nbWorkers) +{ + unsigned const maxNbBuffers = 2 * nbWorkers + 3; + if (srcBufPool == NULL) + return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { + ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + if (newBufPool == NULL) + return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; + } +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize >> 3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_free(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { + buffer_t buffer; + void* const start = ZSTD_malloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start == NULL) ? 0 : bSize; + if (start == NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } +} + +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_malloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) + return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, + "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, + (U32)(bufPool->nbBuffers - 1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_free(buf.start, bufPool->cMem); +} + +/* ===== Seq Pool Wrapper ====== */ + +static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0}; + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = {NULL, 0, 0, 0}; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + if (seqPool == NULL) + return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) +{ + return ZSTDMT_expandBufferPool(pool, nbWorkers); +} + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx* cctx[1]; /* variable size */ +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + int cid; + for (cid = 0; cid < pool->totalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + ZSTD_free(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = + (ZSTDMT_CCtxPool*)ZSTD_calloc(sizeof(ZSTDMT_CCtxPool) + (nbWorkers - 1) * sizeof(ZSTD_CCtx*), cMem); + assert(nbWorkers > 0); + if (!cctxPool) + return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_free(cctxPool, cMem); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->totalCCtx = nbWorkers; + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctx[0]) { + ZSTDMT_freeCCtxPool(cctxPool); + return NULL; + } + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, int nbWorkers) +{ + if (srcPool == NULL) + return NULL; + if (nbWorkers <= srcPool->totalCCtx) + return srcPool; /* good enough */ + /* need a larger cctx pool */ + { + ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { + unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + (nbWorkers - 1) * sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u = 0; u < nbWorkers; u++) { + totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { + ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx == NULL) + return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctx[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + +/* ==== Serial State ==== */ + +typedef struct { + void const* start; + size_t size; +} range_t; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is udpated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} serialState_t; + +static int ZSTDMT_serialState_reset( + serialState_t* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, size_t jobSize) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + serialState->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength); + } else { + memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + size_t const bucketSize = (size_t)1 << bucketLog; + unsigned const prevBucketLog = serialState->params.ldmParams.hashLog - serialState->params.ldmParams.bucketSizeLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_clear(&serialState->ldmState.window); + serialState->ldmWindow = serialState->ldmState.window; + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_free(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_malloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_free(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_malloc(bucketSize, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + memset(serialState->ldmState.hashTable, 0, hashSize); + memset(serialState->ldmState.bucketOffsets, 0, bucketSize); + } + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; +} + +static int ZSTDMT_serialState_init(serialState_t* serialState) +{ + int initError = 0; + memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; +} + +static void ZSTDMT_serialState_free(serialState_t* serialState) +{ + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_free(serialState->ldmState.hashTable, cMem); + ZSTD_free(serialState->ldmState.bucketOffsets, cMem); +} + +static void ZSTDMT_serialState_update( + serialState_t* serialState, ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, range_t src, unsigned jobID) +{ + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && seqStore.size == 0 && seqStore.capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); + (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); + } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + size_t const err = ZSTD_referenceExternalSequences(jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); + (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + ZSTD_pthread_mutex_unlock(&serialState->mutex); +} + +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ + +static const range_t kNullRange = {NULL, 0}; + +typedef struct { + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ + buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; + +#define JOB_ERROR(e) \ + { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ + } + +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + buffer_t dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; + + /* ressources */ + if (cctx == NULL) + JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start == NULL) + JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ + } + if (jobParams.ldmParams.enableLdm && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ + if (job->jobID != 0) + jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = 0; + + /* init */ + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal( + cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) + JOB_ERROR(initError); + } else { /* srcStart points at reloaded section */ + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { + size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) + JOB_ERROR(forceWindowError); + } + { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, + job->prefix.size, + ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + ZSTD_dtlm_fast, + NULL, /*cdict*/ + jobParams, + pledgedSrcSize); + if (ZSTD_isError(initError)) + JOB_ERROR(initError); + } + } + + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) + JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); + } + + /* compress */ + { + size_t const chunkSize = 4 * ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize - 1)) / chunkSize); + const BYTE* ip = (const BYTE*)job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) + assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue(cctx, op, oend - op, ip, chunkSize); + if (ZSTD_isError(cSize)) + JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; + assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG( + 5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/) { + size_t const lastBlockSize1 = job->src.size & (chunkSize - 1); + size_t const lastBlockSize = ((lastBlockSize1 == 0) & (job->src.size >= chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? ZSTD_compressEnd(cctx, op, oend - op, ip, lastBlockSize) + : ZSTD_compressContinue(cctx, op, oend - op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) + JOB_ERROR(cSize); + lastCBlockSize = cSize; + } + } + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) + assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); +} + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + range_t prefix; /* read-only non-owned prefix buffer */ + buffer_t buffer; + size_t filled; +} inBuff_t; + +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} roundBuff_t; + +static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + +#define RSYNC_LENGTH 32 + +typedef struct { + U64 hash; + U64 hitMask; + U64 primePower; +} rsyncState_t; + +struct ZSTDMT_CCtx_s { + POOL_ctx* factory; + ZSTDMT_jobDescription* jobs; + ZSTDMT_bufferPool* bufPool; + ZSTDMT_CCtxPool* cctxPool; + ZSTDMT_seqPool* seqPool; + ZSTD_CCtx_params params; + size_t targetSectionSize; + size_t targetPrefixSize; + int jobReady; /* 1 => one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + inBuff_t inBuff; + roundBuff_t roundBuff; + serialState_t serial; + rsyncState_t rsync; + unsigned singleBlockingThread; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; +}; + +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) + return; + for (jobNb = 0; jobNb < nbJobs; jobNb++) { + ZSTD_pthread_mutex_destroy(&jobTable[jobNb].job_mutex); + ZSTD_pthread_cond_destroy(&jobTable[jobNb].job_cond); + } + ZSTD_free(jobTable, cMem); +} + +/* ZSTDMT_allocJobsTable() + * allocate and init a job table. + * update *nbJobsPtr to next power of 2 value, as size of table */ +static ZSTDMT_jobDescription* ZSTDMT_createJobsTable(U32* nbJobsPtr, ZSTD_customMem cMem) +{ + U32 const nbJobsLog2 = ZSTD_highbit32(*nbJobsPtr) + 1; + U32 const nbJobs = 1 << nbJobsLog2; + U32 jobNb; + ZSTDMT_jobDescription* const jobTable = + (ZSTDMT_jobDescription*)ZSTD_calloc(nbJobs * sizeof(ZSTDMT_jobDescription), cMem); + int initError = 0; + if (jobTable == NULL) + return NULL; + *nbJobsPtr = nbJobs; + for (jobNb = 0; jobNb < nbJobs; jobNb++) { + initError |= ZSTD_pthread_mutex_init(&jobTable[jobNb].job_mutex, NULL); + initError |= ZSTD_pthread_cond_init(&jobTable[jobNb].job_cond, NULL); + } + if (initError != 0) { + ZSTDMT_freeJobsTable(jobTable, nbJobs, cMem); + return NULL; + } + return jobTable; +} + +static size_t ZSTDMT_expandJobsTable(ZSTDMT_CCtx* mtctx, U32 nbWorkers) +{ + U32 nbJobs = nbWorkers + 2; + if (nbJobs > mtctx->jobIDMask + 1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask + 1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs == NULL) + return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + } + return 0; +} + +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + if (nbWorkers > ZSTDMT_NBWORKERS_MAX) + nbWorkers = ZSTDMT_NBWORKERS_MAX; + params->nbWorkers = nbWorkers; + params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT; + params->jobSize = 0; + return nbWorkers; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + + if (nbWorkers < 1) + return NULL; + nbWorkers = MIN(nbWorkers, ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc != NULL) ^ (cMem.customFree != NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*)ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) + return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); + assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers) +{ + return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem); +} + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID = 0; jobID <= mtctx->jobIDMask; jobID++) { + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cSize = 0; + } + memset(mtctx->jobs, 0, (mtctx->jobIDMask + 1) * sizeof(ZSTDMT_jobDescription)); + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, + "waiting for jobCompleted signal from job %u", + mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) + return 0; /* compatible with free on NULL */ + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask + 1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_free(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_free(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) + return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + POOL_sizeof(mtctx->factory) + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask + 1) * sizeof(ZSTDMT_jobDescription) + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + ZSTD_sizeof_CDict(mtctx->cdictLocal) + mtctx->roundBuff.capacity; +} + +/* Internal only */ +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value) +{ + DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter"); + switch (parameter) { + case ZSTDMT_p_jobSize: + DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); + if (value != 0 /* default */ + && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + assert(value >= 0); + if (value > ZSTDMT_JOBSIZE_MAX) + value = ZSTDMT_JOBSIZE_MAX; + params->jobSize = value; + return value; + + case ZSTDMT_p_overlapLog: + DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value); + if (value < ZSTD_OVERLAPLOG_MIN) + value = ZSTD_OVERLAPLOG_MIN; + if (value > ZSTD_OVERLAPLOG_MAX) + value = ZSTD_OVERLAPLOG_MAX; + params->overlapLog = value; + return value; + + case ZSTDMT_p_rsyncable: + value = (value != 0); + params->rsyncable = value; + return value; + + default: + return ERROR(parameter_unsupported); + } +} + +size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value) +{ + DEBUGLOG(4, "ZSTDMT_setMTCtxParameter"); + return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); +} + +size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value) +{ + switch (parameter) { + case ZSTDMT_p_jobSize: + assert(mtctx->params.jobSize <= INT_MAX); + *value = (int)(mtctx->params.jobSize); + break; + case ZSTDMT_p_overlapLog: + *value = mtctx->params.overlapLog; + break; + case ZSTDMT_p_rsyncable: + *value = mtctx->params.rsyncable; + break; + default: + return ERROR(parameter_unsupported); + } + return 0; +} + +/* Sets parameters relevant to the compression job, + * initializing others to default values. */ +static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) +{ + ZSTD_CCtx_params jobParams; + memset(&jobParams, 0, sizeof(jobParams)); + + jobParams.cParams = params.cParams; + jobParams.fParams = params.fParams; + jobParams.compressionLevel = params.compressionLevel; + + return jobParams; +} + +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) + return ERROR(memory_allocation); + CHECK_F(ZSTDMT_expandJobsTable(mtctx, nbWorkers)); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers); + if (mtctx->bufPool == NULL) + return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) + return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) + return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { + ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0, 0); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } +} + +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { + unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; + assert(mtctx->jobReady <= 1); + DEBUGLOG(6, + "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, + lastJobNb, + mtctx->jobReady) + for (jobNb = mtctx->doneJobID; jobNb < lastJobNb; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { + size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + } + return fps; +} + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +{ + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) + return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { + unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { + size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + toFlush = produced - flushed; + if (toFlush == 0 && (jobPtr->consumed >= jobPtr->src.size)) { + /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */ + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + + return toFlush; +} + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) +{ + if (params.ldmParams.enableLdm) + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. */ + return MAX(21, params.cParams.chainLog + 4); + return MAX(20, params.cParams.windowLog + 2); +} + +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch (strat) { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; +} + +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) + return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params.ldmParams.enableLdm) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; + } + assert(0 <= ovLog && ovLog <= 30); + DEBUGLOG(4, "overlapLog : %i", params.overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog == 0) ? 0 : (size_t)1 << ovLog; +} + +static unsigned ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) +{ + assert(nbWorkers > 0); + { + size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); + size_t const jobMaxSize = jobSizeTarget << 2; + size_t const passSizeMax = jobMaxSize * nbWorkers; + unsigned const multiplier = (unsigned)(srcSize / passSizeMax) + 1; + unsigned const nbJobsLarge = multiplier * nbWorkers; + unsigned const nbJobsMax = (unsigned)(srcSize / jobSizeTarget) + 1; + unsigned const nbJobsSmall = MIN(nbJobsMax, nbWorkers); + return (multiplier > 1) ? nbJobsLarge : nbJobsSmall; + } +} + +/* ZSTDMT_compress_advanced_internal() : + * This is a blocking function : it will only give back control to caller after finishing its compression job. + */ +static size_t ZSTDMT_compress_advanced_internal(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const ZSTD_CDict* cdict, ZSTD_CCtx_params params) +{ + ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); + size_t const overlapSize = ZSTDMT_computeOverlapSize(params); + unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers); + size_t const proposedJobSize = (srcSize + (nbJobs - 1)) / nbJobs; + size_t const avgJobSize = (((proposedJobSize - 1) & 0x1FFFF) < 0x7FFF) + ? proposedJobSize + 0xFFFF + : proposedJobSize; /* avoid too small last block */ + const char* const srcStart = (const char*)src; + size_t remainingSrcSize = srcSize; + unsigned const compressWithinDst = + (dstCapacity >= ZSTD_compressBound(srcSize)) + ? nbJobs + : (unsigned)(dstCapacity / + ZSTD_compressBound(avgJobSize)); /* presumes avgJobSize >= 256 KB, which should be the case */ + size_t frameStartPos = 0, dstBufferPos = 0; + assert(jobParams.nbWorkers == 0); + assert(mtctx->cctxPool->totalCCtx == params.nbWorkers); + + params.jobSize = (U32)avgJobSize; + DEBUGLOG(4, + "ZSTDMT_compress_advanced_internal: nbJobs=%2u (rawSize=%u bytes; fixedSize=%u) ", + nbJobs, + (U32)proposedJobSize, + (U32)avgJobSize); + + if ((nbJobs == 1) | + (params.nbWorkers <= 1)) { /* fallback to single-thread mode : this is a blocking invocation anyway */ + ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; + DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode"); + if (cdict) + return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, jobParams); + } + + assert( + avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), + required to compress directly into Dst (no additional buffer) */ + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize)); + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize)) + return ERROR(memory_allocation); + + CHECK_F(ZSTDMT_expandJobsTable(mtctx, nbJobs)); /* only expands if necessary */ + + { + unsigned u; + for (u = 0; u < nbJobs; u++) { + size_t const jobSize = MIN(remainingSrcSize, avgJobSize); + size_t const dstBufferCapacity = ZSTD_compressBound(jobSize); + buffer_t const dstAsBuffer = {(char*)dst + dstBufferPos, dstBufferCapacity}; + buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : g_nullBuffer; + size_t dictSize = u ? overlapSize : 0; + + mtctx->jobs[u].prefix.start = srcStart + frameStartPos - dictSize; + mtctx->jobs[u].prefix.size = dictSize; + mtctx->jobs[u].src.start = srcStart + frameStartPos; + mtctx->jobs[u].src.size = jobSize; + assert(jobSize > 0); /* avoid job.src.size == 0 */ + mtctx->jobs[u].consumed = 0; + mtctx->jobs[u].cSize = 0; + mtctx->jobs[u].cdict = (u == 0) ? cdict : NULL; + mtctx->jobs[u].fullFrameSize = srcSize; + mtctx->jobs[u].params = jobParams; + /* do not calculate checksum within sections, but write it in header for first section */ + mtctx->jobs[u].dstBuff = dstBuffer; + mtctx->jobs[u].cctxPool = mtctx->cctxPool; + mtctx->jobs[u].bufPool = mtctx->bufPool; + mtctx->jobs[u].seqPool = mtctx->seqPool; + mtctx->jobs[u].serial = &mtctx->serial; + mtctx->jobs[u].jobID = u; + mtctx->jobs[u].firstJob = (u == 0); + mtctx->jobs[u].lastJob = (u == nbJobs - 1); + + DEBUGLOG(5, "ZSTDMT_compress_advanced_internal: posting job %u (%u bytes)", u, (U32)jobSize); + DEBUG_PRINTHEX(6, mtctx->jobs[u].prefix.start, 12); + POOL_add(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[u]); + + frameStartPos += jobSize; + dstBufferPos += dstBufferCapacity; + remainingSrcSize -= jobSize; + } + } + + /* collect result */ + { + size_t error = 0, dstPos = 0; + unsigned jobID; + for (jobID = 0; jobID < nbJobs; jobID++) { + DEBUGLOG(5, "waiting for job %u ", jobID); + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(5, "waiting for jobCompleted signal from job %u", jobID); + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + DEBUGLOG(5, "ready to write job %u ", jobID); + + { + size_t const cSize = mtctx->jobs[jobID].cSize; + if (ZSTD_isError(cSize)) + error = cSize; + if ((!error) && (dstPos + cSize > dstCapacity)) + error = ERROR(dstSize_tooSmall); + if (jobID) { /* note : job 0 is written directly at dst, which is correct position */ + if (!error) + memmove((char*)dst + dstPos, + mtctx->jobs[jobID].dstBuff.start, + cSize); /* may overlap when job compressed within dst */ + if (jobID >= compressWithinDst) { /* job compressed into its own buffer, which must be released */ + DEBUGLOG(5, "releasing buffer %u>=%u", jobID, compressWithinDst); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + } + } + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cSize = 0; + dstPos += cSize; + } + } /* for (jobID=0; jobIDserial.xxhState); + if (dstPos + 4 > dstCapacity) { + error = ERROR(dstSize_tooSmall); + } else { + DEBUGLOG(4, "writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)dst + dstPos, checksum); + dstPos += 4; + } + } + + if (!error) + DEBUGLOG(4, "compressed size : %u ", (U32)dstPos); + return error ? error : dstPos; + } +} + +size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_parameters params, int overlapLog) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX); + cctxParams.overlapLog = overlapLog; + return ZSTDMT_compress_advanced_internal(mtctx, dst, dstCapacity, src, srcSize, cdict, cctxParams); +} + +size_t ZSTDMT_compressCCtx( + ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy); + params.fParams.contentSizeFlag = 1; + return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); +} + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, + "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, + params.nbWorkers, + mtctx->cctxPool->totalCCtx); + + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + CHECK_F(ZSTDMT_resize(mtctx, params.nbWorkers)); + + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) + params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) + params.jobSize = ZSTDMT_JOBSIZE_MAX; + + mtctx->singleBlockingThread = + (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ + if (mtctx->singleBlockingThread) { + ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); + DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); + assert(singleThreadParams.nbWorkers == 0); + return ZSTD_initCStream_internal( + mtctx->cctxPool->cctx[0], dict, dictSize, cdict, singleThreadParams, pledgedSrcSize); + } + + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); + + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; + } + + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + if (dict) { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, + dictSize, + ZSTD_dlm_byCopy, + dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, + mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) + return ERROR(memory_allocation); + } else { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize >> 10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); + } + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); + U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20; + assert(jobSizeMB >= 1); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) + mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize >> 10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize >> 10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_free(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_malloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity >> 10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize)) + return ERROR(memory_allocation); + return 0; +} + +size_t ZSTDMT_initCStream_advanced( + ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */ + DEBUGLOG(4, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize); + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dct_auto, NULL, cctxParams, pledgedSrcSize); +} + +size_t ZSTDMT_initCStream_usingCDict( + ZSTDMT_CCtx* mtctx, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; + if (cdict == NULL) + return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ + cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict); + cctxParams.fParams = fParams; + return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dct_auto, cdict, cctxParams, pledgedSrcSize); +} + +/* ZSTDMT_resetCStream() : + * pledgedSrcSize can be zero == unknown (for the time being) + * prefer using ZSTD_CONTENTSIZE_UNKNOWN, + * as `0` might mean "empty" in the future */ +size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize) +{ + if (!pledgedSrcSize) + pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, 0, mtctx->params, pledgedSrcSize); +} + +size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); + ZSTD_CCtx_params cctxParams = mtctx->params; /* retrieve sticky params */ + DEBUGLOG(4, "ZSTDMT_initCStream (cLevel=%i)", compressionLevel); + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successfull if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +{ + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert( + job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); +} + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +{ + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } + + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, + "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, + (U32)srcSize, + (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID == 0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID == 0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID > 0); + mtctx->jobs[jobID].dstFlushed = 0; + + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } + } + + if ((srcSize == 0) && (mtctx->nextJobID > 0) /*single job must also write frame header*/) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } + + DEBUGLOG(5, + "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; + } + return 0; +} + +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced( + ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + if (blockToFlush && (mtctx->doneJobID < mtctx->nextJobID)) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, + "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, + (U32)mtctx->jobs[wJobID].consumed, + (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, + "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, + (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, + &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } + } + + /* try to flush something */ + { + size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but + no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, + "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, + ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ((srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; + } + + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, + "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, + mtctx->doneJobID, + (U32)srcConsumed, + (U32)srcSize, + (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + + if ((srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize)) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, + "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, + (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } + } + + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) + return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) + return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) + return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) + return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) + return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = + mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) + return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal + buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ +} + +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +{ + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; + + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + + if (consumed < mtctx->jobs[wJobID].src.size) { + range_t range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } + } + return kNullRange; +} + +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +{ + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeStart = (BYTE const*)range.start; + BYTE const* const rangeEnd = rangeStart + range.size; + + if (rangeStart == NULL || bufferStart == NULL) + return 0; + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; +} + +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", (size_t)extDict.start, (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", (size_t)prefix.start, (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", (size_t)buffer.start, (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const target = mtctx->targetSectionSize; + buffer_t buffer; + + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= target); + + if (spaceLeft < target) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; + + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; + } + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = target; + + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + + DEBUGLOG(5, + "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", (size_t)buffer.start, (size_t)buffer.start + buffer.capacity); + + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; +} + +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled >= RSYNC_LENGTH) { + /* We have enough bytes buffered to initialize the hash. + * Start scanning at the beginning of the input. + */ + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + /* We don't have enough bytes buffered to initialize the hash, but + * we know we have at least RSYNC_LENGTH bytes total. + * Start scanning after the first RSYNC_LENGTH bytes less the bytes + * already buffered. + */ + pos = RSYNC_LENGTH - mtctx->inBuff.filled; + prev = (BYTE const*)mtctx->inBuff.buffer.start - pos; + hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + break; + } + } + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize == 0) + hintInSize = mtctx->targetSectionSize; + return hintInSize; +} + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic( + ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp) +{ + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */ + return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + } + + if ((mtctx->frameEnded) && (endOp == ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + + /* single-pass shortcut (note : synchronous-mode) */ + if ((!mtctx->params.rsyncable) /* rsyncable mode is disabled */ + && (mtctx->nextJobID == 0) /* just started */ + && (mtctx->inBuff.filled == 0) /* nothing buffered */ + && (!mtctx->jobReady) /* no job already created */ + && (endOp == ZSTD_e_end) /* end order */ + && (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos))) { /* enough space in dst */ + size_t const cSize = ZSTDMT_compress_advanced_internal(mtctx, + (char*)output->dst + output->pos, + output->size - output->pos, + (const char*)input->src + input->pos, + input->size - input->pos, + mtctx->cdict, + mtctx->params); + if (ZSTD_isError(cSize)) + return cSize; + input->pos = input->size; + output->pos += cSize; + mtctx->allJobsCompleted = 1; + mtctx->frameEnded = 1; + return 0; + } + + /* fill input buffer */ + if ((!mtctx->jobReady) && (input->size > input->pos)) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, + "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", + mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, + "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, + (U32)mtctx->inBuff.filled, + (U32)mtctx->targetSectionSize); + memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, + (const char*)input->src + input->pos, + syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad > 0; + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) + endOp = ZSTD_e_flush; /* can't end now : not all input consumed */ + } + + if ((mtctx->jobReady) || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded))) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + CHECK_F(ZSTDMT_createCompressionJob(mtctx, jobSize, endOp)); + } + + /* check for potential compressed data ready to be flushed */ + { + size_t const remainingToFlush = ZSTDMT_flushProduced( + mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) + return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; + } +} + +size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + CHECK_F(ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue)); + + /* recommended next input size : fill current input buffer */ + return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and + no more availability to create new job */ +} + +static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_EndDirective endFrame) +{ + size_t const srcSize = mtctx->inBuff.filled; + DEBUGLOG(5, "ZSTDMT_flushStream_internal"); + + if (mtctx->jobReady /* one job ready for a worker to pick up */ + || (srcSize > 0) /* still some data within input buffer */ + || ((endFrame == ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */ + DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)", (U32)srcSize, (U32)endFrame); + CHECK_F(ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame)); + } + + /* check if there is any data available to flush */ + return ZSTDMT_flushProduced(mtctx, output, 1 /* blockToFlush */, endFrame); +} + +size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output) +{ + DEBUGLOG(5, "ZSTDMT_flushStream"); + if (mtctx->singleBlockingThread) + return ZSTD_flushStream(mtctx->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_flush); +} + +size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output) +{ + DEBUGLOG(4, "ZSTDMT_endStream"); + if (mtctx->singleBlockingThread) + return ZSTD_endStream(mtctx->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(mtctx, output, ZSTD_e_end); +} diff --git a/src/lib/compress/zstd_1_3_8/zstdmt_compress.h b/src/lib/compress/zstd_1_3_8/zstdmt_compress.h new file mode 100644 index 0000000000000000000000000000000000000000..4a1aba4ab31a46ca018b082f51927ab871bb42e0 --- /dev/null +++ b/src/lib/compress/zstd_1_3_8/zstdmt_compress.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTDMT_COMPRESS_H +#define ZSTDMT_COMPRESS_H + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Note : This is an internal API. + * Some methods are still exposed (ZSTDLIB_API), + * because it used to be the only way to invoke MT compression. + * Now, it's recommended to use ZSTD_compress_generic() instead. + * These methods will stop being exposed in a future version */ + +/* === Dependencies === */ +#include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX +#define ZSTDMT_NBWORKERS_MAX 200 +#endif +#ifndef ZSTDMT_JOBSIZE_MIN +#define ZSTDMT_JOBSIZE_MIN (1 MB) +#endif +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem); +ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + +/* === Simple one-pass compression function === */ + +ZSTDLIB_API size_t ZSTDMT_compressCCtx( + ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); + +/* === Streaming functions === */ + +ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); +ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, + unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: + for compatibility with older programs, 0 means the same as + ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ + +ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); +ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTDMT_flushStream( + ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or + an error code (ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTDMT_endStream( + ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or + an error code (ZSTD_isError()) */ + +/* === Advanced functions and parameters === */ + +ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, + size_t srcSize, const ZSTD_CDict* cdict, ZSTD_parameters params, int overlapLog); + +ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, + size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ + ZSTD_parameters params, + unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ + +ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, const ZSTD_CDict* cdict, + ZSTD_frameParameters fparams, unsigned long long pledgedSrcSize); /* note : zero means empty */ + +/* ZSTDMT_parameter : + * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ +typedef enum { + ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on + compression parameters. Can be set explicitly here. */ + ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, + 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its + value will be re-used on next compression job */ + ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ +} ZSTDMT_parameter; + +/* ZSTDMT_setMTCtxParameter() : + * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter. + * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__ + * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value); + +/* ZSTDMT_getMTCtxParameter() : + * Query the ZSTDMT_CCtx for a parameter value. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value); + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +ZSTDLIB_API size_t ZSTDMT_compressStream_generic( + ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); + +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + +/*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_CCtxParam_setMTCtxParameter() + * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */ +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value); + +/*! ZSTDMT_CCtxParam_setNbWorkers() + * Set nbWorkers, and clamp it. + * Also reset jobSize and overlapLog */ +size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers); + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); + +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/src/lib/encode/ob_base64_encode.cpp b/src/lib/encode/ob_base64_encode.cpp index f1b1270b4aee1fc0eb845c62c639e6c48818b0af..27df4693c7ea4bf172cee2f8acdb32129d825d47 100644 --- a/src/lib/encode/ob_base64_encode.cpp +++ b/src/lib/encode/ob_base64_encode.cpp @@ -142,7 +142,7 @@ int ObBase64Encoder::decode(const char *input, const int64_t input_len, uint8_t uint8_array_4[4]; int64_t i = 0; int64_t rounds = input_len / 4; - if (OB_UNLIKELY(rounds * 3) + pos > output_len) { + if ((rounds * 3) + pos > output_len) { ret = OB_BUF_NOT_ENOUGH; _OB_LOG(WARN, "buffer not enough, pos=%ld, output_len=%ld, input_len=%ld", pos, output_len, input_len); diff --git a/src/lib/hash/ob_hashmap.h b/src/lib/hash/ob_hashmap.h index 13b8c63b8dd1251a9844395489f9fa7abe41a661..3572c2f876dcc75508acca50efe8648795bf6be2 100644 --- a/src/lib/hash/ob_hashmap.h +++ b/src/lib/hash/ob_hashmap.h @@ -208,6 +208,152 @@ private: DISALLOW_COPY_AND_ASSIGN(ObHashMap); }; + +template , + class _equal = equal_to<_key_type>, + class _allocer = SimpleAllocer::AllocType>, + template class _bucket_array = NormalPointer, + class _bucket_allocer = oceanbase::common::ObMalloc> +class ObHashMapWrapper +{ + typedef ObHashMapWrapper<_key_type, _value_type, _defendmode, _hashfunc, _equal, _allocer, _bucket_array, _bucket_allocer> + OB_HASH_MAP_WRAPPER; + typedef ObHashMap<_key_type, _value_type, _defendmode, _hashfunc, _equal, _allocer, _bucket_array, _bucket_allocer> + OB_HASH_MAP; +public: + typedef typename OB_HASH_MAP::iterator iterator; + typedef typename OB_HASH_MAP::const_iterator const_iterator; + +public: + ObHashMapWrapper() : is_inited_(false), bucket_num_(-1), bucket_mod_id_(0), node_mod_id_(0), + allocer_(NULL), bucket_allocer_(NULL), hash_map_() {} + ~ObHashMapWrapper() { + hash_map_.destroy(); + } + + int reuse() + { + return hash_map_.reuse(); + } + + int init(int64_t bucket_num, int64_t bucket_mod_id, int64_t node_mod_id = ObModIds::OB_HASH_NODE) { + int ret = OB_SUCCESS; + + if (!is_inited_) { + bucket_num_ = bucket_num; + bucket_mod_id_ = bucket_mod_id; + node_mod_id_ = node_mod_id; + + if (OB_SUCC(hash_map_.create(bucket_num, bucket_mod_id, node_mod_id))) { + is_inited_ = true; + } + } + + return ret; + } + + int init(int64_t bucket_num, _allocer *allocer, int64_t bucket_mod_id, + int64_t node_mod_id = ObModIds::OB_HASH_NODE) { + int ret = OB_SUCCESS; + + if (!is_inited_) { + bucket_num_ = bucket_num; + allocer_ = allocer; + bucket_mod_id_ = bucket_mod_id; + node_mod_id_ = node_mod_id; + + if (OB_SUCC(hash_map_.create(bucket_num, allocer, bucket_mod_id, node_mod_id))) { + is_inited_ = true; + } + } + + return ret; + } + + int init(int64_t bucket_num, _allocer *allocer, _bucket_allocer *bucket_allocer) { + int ret = OB_SUCCESS; + + if (!is_inited_) { + bucket_num_ = bucket_num; + allocer_ = allocer; + bucket_allocer_ = bucket_allocer; + + if (OB_SUCC(hash_map_.create(bucket_num, allocer, bucket_allocer))) { + is_inited_ = true; + } + } + + return ret; + } + + int init(const OB_HASH_MAP_WRAPPER &hash_map_wrapper) { + int ret = OB_SUCCESS; + + if (!is_inited_) { + bucket_num_ = hash_map_wrapper.get_bucket_num(); + bucket_mod_id_ = hash_map_wrapper.get_bucket_mod_id(); + node_mod_id_ = hash_map_wrapper.get_node_mod_id(); + allocer_ = hash_map_wrapper.get_allocer(); + bucket_allocer_ = hash_map_wrapper.get_bucket_allocer(); + + if (NULL != allocer_ && NULL != bucket_allocer_) { + ret = hash_map_.create(bucket_num_, allocer_, bucket_allocer_); + } else if (NULL != allocer_) { + ret = hash_map_.create(bucket_num_, allocer_, bucket_mod_id_, node_mod_id_); + } else { + ret = hash_map_.create(bucket_num_, bucket_mod_id_, node_mod_id_); + } + + if (OB_SUCC(ret)) { + is_inited_ = true; + } + } + + return ret; + } + + int assign(const OB_HASH_MAP_WRAPPER &hash_map_wrapper) { + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = init(hash_map_wrapper); + } + + if (OB_SUCC(ret)) { + const OB_HASH_MAP &hash_map = hash_map_wrapper.get_hash_map(); + const_iterator iter = hash_map.begin(); + const_iterator end = hash_map.end(); + for (; OB_SUCC(ret) && iter !=end; iter++) { + ret = hash_map_.set_refactored(iter->first, iter->second); + } + } + + return ret; + } + + int64_t get_bucket_num() const { return bucket_num_; } + int64_t get_bucket_mod_id() const { return bucket_mod_id_; } + int64_t get_node_mod_id() const { return node_mod_id_; } + _allocer *get_allocer() const { return allocer_; } + _bucket_allocer *get_bucket_allocer() const { return bucket_allocer_; } + const OB_HASH_MAP &get_hash_map() const { return hash_map_; } + OB_HASH_MAP &get_hash_map() { return hash_map_; } + + TO_STRING_KV(K_(is_inited), K_(bucket_num), K_(bucket_mod_id), + K_(node_mod_id), KP_(allocer), KP_(bucket_allocer)); + +private: + bool is_inited_; + int64_t bucket_num_; + int64_t bucket_mod_id_; + int64_t node_mod_id_; + _allocer *allocer_; + _bucket_allocer *bucket_allocer_; + OB_HASH_MAP hash_map_; +}; + }//namespace hash }//namespace common }//namespace oceanbase diff --git a/src/lib/hash/ob_link_hashmap.h b/src/lib/hash/ob_link_hashmap.h index f30fe21e54aa61cf642b2c363204c4d69c38ae37..a31fadfde828a9fd3ca74b5cf50ec40c0c5a9f6b 100644 --- a/src/lib/hash/ob_link_hashmap.h +++ b/src/lib/hash/ob_link_hashmap.h @@ -101,6 +101,18 @@ private: DISALLOW_COPY_AND_ASSIGN(LinkHashNode); }; +template +struct LinkHashValue +{ + LinkHashValue() : hash_node_(NULL) {} + ~LinkHashValue() {} + int32_t get_uref() const { return hash_node_ == NULL ? 0 : hash_node_->uref_; } + int32_t get_href() const { return hash_node_ == NULL ? 0 : hash_node_->href_; } + LinkHashNode* hash_node_; +private: + DISALLOW_COPY_AND_ASSIGN(LinkHashValue); +}; + template class AllocHandle { diff --git a/src/lib/hash/ob_pointer_hashmap.h b/src/lib/hash/ob_pointer_hashmap.h index 761214b7558a003479619e0e42f84f4b515f0bdb..8cc1e04ce760cc2560259c7d872cfbff6e956549 100644 --- a/src/lib/hash/ob_pointer_hashmap.h +++ b/src/lib/hash/ob_pointer_hashmap.h @@ -45,7 +45,7 @@ namespace hash /** * not thread safe hash map only for pointer */ -template class GetKey> +template class GetKey> class ObPointerHashArray { public: @@ -456,7 +456,7 @@ private: V cells_[0]; }; -template < class K, class V, template class GetKey, +template < class K, class V, template class GetKey, int64_t default_size = OB_MALLOC_NORMAL_BLOCK_SIZE, class Allocator = ModulePageAllocator > class ObPointerHashMap { diff --git a/src/lib/hash_func/murmur_hash.cpp b/src/lib/hash_func/murmur_hash.cpp index 76ae561703ca4bcabe4f8db572d56cccc21b2bcb..157fd6b754968d7cdec6233390d38b01e05ffab2 100644 --- a/src/lib/hash_func/murmur_hash.cpp +++ b/src/lib/hash_func/murmur_hash.cpp @@ -52,8 +52,10 @@ uint32_t murmurhash2(const void *key, int32_t len, uint32_t seed) switch (len) { case 3: h ^= data[2] << 16; + __attribute__ ((fallthrough)); case 2: h ^= data[1] << 8; + __attribute__ ((fallthrough)); case 1: h ^= data[0]; h *= m; @@ -94,16 +96,22 @@ uint64_t murmurhash64A(const void *key, int32_t len, uint64_t seed) switch (len & 7) { case 7: h ^= uint64_t(data2[6]) << 48; + __attribute__ ((fallthrough)); case 6: h ^= uint64_t(data2[5]) << 40; + __attribute__ ((fallthrough)); case 5: h ^= uint64_t(data2[4]) << 32; + __attribute__ ((fallthrough)); case 4: h ^= uint64_t(data2[3]) << 24; + __attribute__ ((fallthrough)); case 3: h ^= uint64_t(data2[2]) << 16; + __attribute__ ((fallthrough)); case 2: h ^= uint64_t(data2[1]) << 8; + __attribute__ ((fallthrough)); case 1: h ^= uint64_t(data2[0]); h *= m; diff --git a/src/lib/number/ob_number_v2.cpp b/src/lib/number/ob_number_v2.cpp index b0c900cd56cb657b7b299b1e7578aaf79e6d783c..3be310ad14d89f67d749294961b24eca0a8e8ccb 100644 --- a/src/lib/number/ob_number_v2.cpp +++ b/src/lib/number/ob_number_v2.cpp @@ -1825,7 +1825,7 @@ int ObNumberBuilder::build(const char *str, } else if (OB_FAIL(find_point_(str, length, integer_start, integer_end, decimal_start, negative, integer_zero, decimal_zero, warning))) { _OB_LOG(WARN, "lookup fail ret=%d str=[%.*s]", ret, (int)length, str); - } else if (OB_FAIL(ib_.build(str, integer_start, integer_end, integer_zero))) { + } else if (OB_FAIL(build_integer_(str, integer_start, integer_end, decimal_zero))) { _OB_LOG(WARN, "build integer fail, ret=%d str=[%.*s]", ret, (int)length, str); } else if (OB_FAIL(build_decimal_(str, length, decimal_start, decimal_zero))) { _OB_LOG(WARN, "build decimal fail, ret=%d str=[%.*s]", ret, (int)length, str); @@ -1999,6 +1999,7 @@ int ObNumberBuilder::find_point_( } else { b_decimal_zero = false; } + __attribute__ ((fallthrough)); /* no break. */ case '0': if (-2 == i_integer_start) { diff --git a/src/lib/number/ob_number_v2.h b/src/lib/number/ob_number_v2.h index c143473d50705018e3aa8be98312b03959592a51..328f6349e3f9b78ffd7f2dc2d6a9f6b2bc5dbbe4 100644 --- a/src/lib/number/ob_number_v2.h +++ b/src/lib/number/ob_number_v2.h @@ -118,6 +118,7 @@ public: static const ObPrecision MAX_PRECISION = 65; static const ObScale MIN_SCALE = -84; static const ObScale MAX_SCALE = 127; + static const ObScale MAX_TOTAL_SCALE = MAX_SCALE + MAX_PRECISION + 1; // 5 valid digits, another: 1 for round, 1 for negative end flag static const int64_t MAX_STORE_LEN = 9; static const int64_t MAX_BYTE_LEN = sizeof(uint32_t) * MAX_STORE_LEN + sizeof(Desc::desc_); diff --git a/src/lib/ob_define.h b/src/lib/ob_define.h index 5f16e3a06a4269185a929607d845e5feb92e4908..96a831197714933c80d00d9b0e35226326d0630b 100644 --- a/src/lib/ob_define.h +++ b/src/lib/ob_define.h @@ -41,6 +41,7 @@ const int64_t OB_LATEST_VERSION = 0; const uint32_t OB_INVALID_FILE_ID = UINT32_MAX; const int16_t OB_COMPACT_INVALID_INDEX = -1; const int OB_INVALID_INDEX = -1; +const int64_t OB_INVALID_INDEX_INT64 = -1; const int OB_INVALID_SIZE = -1; const int OB_INVALID_COUNT = -1; const int OB_INVALID_PTHREAD_KEY = -1; @@ -236,6 +237,12 @@ const int64_t OB_PLAN_CACHE_PERCENTAGE = 20; const int64_t OB_PLAN_CACHE_EVICT_HIGH_PERCENTAGE = 90; const int64_t OB_PLAN_CACHE_EVICT_LOW_PERCENTAGE = 50; +// time zone info +const int64_t OB_MAX_TZ_ABBR_LEN = 32; // according to statistics +const int64_t OB_MAX_TZ_NAME_LEN = 64; // according to statistics +const int64_t OB_INVALID_TZ_ID = -1; +const int64_t OB_INVALID_TZ_TRAN_TIME = INT64_MIN; + // OceanBase Log Synchronization Type const int64_t OB_LOG_NOSYNC = 0; const int64_t OB_LOG_SYNC = 1; @@ -717,7 +724,9 @@ static const int64_t OB_CAST_TO_VARCHAR_MAX_LENGTH = 256; static const int64_t OB_CAST_BUFFER_LENGTH = 256; static const int64_t OB_PREALLOCATED_NUM = 21; // half of 42 static const int64_t OB_PREALLOCATED_COL_ID_NUM = 128; +static const int64_t OB_MAX_DATE_PRECISION = 0; static const int64_t OB_MAX_DATETIME_PRECISION = 6; +static const int64_t OB_MAX_TIMESTAMP_TZ_PRECISION = 9; const char *const SYS_DATE = "$SYS_DATE"; const char *const OB_DEFAULT_COMPRESS_FUNC_NAME = "none"; @@ -787,6 +796,7 @@ static const char *const OB_CONFIG_NOT_NEED_REBOOT = "false"; //Precision in user data type static const int16_t MAX_SCALE_FOR_TEMPORAL = 6; static const int16_t MIN_SCALE_FOR_TEMPORAL = 0; +static const int16_t MAX_SCALE_FOR_ORACLE_TEMPORAL = 9; static const int16_t DEFAULT_SCALE_FOR_INTEGER = 0; static const int16_t DEFAULT_LENGTH_FOR_NUMERIC = -1; static const int16_t DEFAULT_SCALE_FOR_DATE = 0; @@ -799,6 +809,9 @@ static const int16_t DEFAULT_PRECISION_FOR_TEMPORAL = -1; static const int16_t DEFAULT_LENGTH_FOR_TEMPORAL = -1; static const int16_t DEFAULT_PRECISION_FOR_STRING = -1; static const int16_t DEFAULT_SCALE_FOR_STRING = -1; +static const int16_t DEFAULT_SCALE_FOR_TEXT = 0; +static const int16_t DEFAULT_SCALE_FOR_ORACLE_FRACTIONAL_SECONDS = 6; //SEE : https://docs.oracle.com/cd/B19306_01/server.102/b14225/ch4datetime.htm +static const int16_t DEFAULT_SCALE_FOR_ORACLE_TIMESTAMP = 6; enum ObDmlType { diff --git a/src/lib/ob_errno.cpp b/src/lib/ob_errno.cpp index dd2a44f66c32654a155eef161fa3862e54616622..cd22aa14e0f9f7f51a1937c6fda5ac90ad744a1c 100644 --- a/src/lib/ob_errno.cpp +++ b/src/lib/ob_errno.cpp @@ -32,7 +32,8 @@ static struct ObStrErrorInit memset(SQLSTATE, 0, sizeof(SQLSTATE)); memset(MYSQL_ERRNO, 0, sizeof(MYSQL_ERRNO)); memset(STR_USER_ERROR, 0, sizeof(STR_USER_ERROR)); - MYSQL_ERRNO[-OB_ERROR] = -1; + + MYSQL_ERRNO[-OB_ERROR] = -1; SQLSTATE[-OB_ERROR] = "HY000"; STR_ERROR[-OB_ERROR] = "Common error"; STR_USER_ERROR[-OB_ERROR] = "Common error"; @@ -640,6 +641,10 @@ static struct ObStrErrorInit SQLSTATE[-OB_ITEM_NOT_MATCH] = "HY000"; STR_ERROR[-OB_ITEM_NOT_MATCH] = "Item not match"; STR_USER_ERROR[-OB_ITEM_NOT_MATCH] = "Item not match"; + MYSQL_ERRNO[-OB_INVALID_DATE_FORMAT_END] = ER_TRUNCATED_WRONG_VALUE; + SQLSTATE[-OB_INVALID_DATE_FORMAT_END] = "22007"; + STR_ERROR[-OB_INVALID_DATE_FORMAT_END] = "Incorrect value"; + STR_USER_ERROR[-OB_INVALID_DATE_FORMAT_END] = "Incorrect value"; MYSQL_ERRNO[-OB_HASH_EXIST] = -1; SQLSTATE[-OB_HASH_EXIST] = "HY000"; STR_ERROR[-OB_HASH_EXIST] = "hash map/set entry exist"; @@ -1266,8 +1271,8 @@ static struct ObStrErrorInit STR_USER_ERROR[-OB_SERVER_NOT_ALIVE] = "server is not alive"; MYSQL_ERRNO[-OB_GET_LOCATION_TIME_OUT] = 4012; SQLSTATE[-OB_GET_LOCATION_TIME_OUT] = "HY000"; - STR_ERROR[-OB_GET_LOCATION_TIME_OUT] = "Timeout"; - STR_USER_ERROR[-OB_GET_LOCATION_TIME_OUT] = "Timeout"; + STR_ERROR[-OB_GET_LOCATION_TIME_OUT] = "Get Location Cache Fail"; + STR_USER_ERROR[-OB_GET_LOCATION_TIME_OUT] = "Get Location Cache Fail"; MYSQL_ERRNO[-OB_UNIT_IS_MIGRATING] = -1; SQLSTATE[-OB_UNIT_IS_MIGRATING] = "HY000"; STR_ERROR[-OB_UNIT_IS_MIGRATING] = "Unit is migrating, can not migrate again"; @@ -2344,6 +2349,186 @@ static struct ObStrErrorInit SQLSTATE[-OB_ERR_UNEXPECTED_TZ_TRANSITION] = "HY000"; STR_ERROR[-OB_ERR_UNEXPECTED_TZ_TRANSITION] = "unexpected time zone info transition"; STR_USER_ERROR[-OB_ERR_UNEXPECTED_TZ_TRANSITION] = "unexpected time zone info transition"; + MYSQL_ERRNO[-OB_ERR_INVALID_TIMEZONE_REGION_ID] = -1; + SQLSTATE[-OB_ERR_INVALID_TIMEZONE_REGION_ID] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_TIMEZONE_REGION_ID] = "timezone region ID is invalid"; + STR_USER_ERROR[-OB_ERR_INVALID_TIMEZONE_REGION_ID] = "timezone region ID is invalid"; + MYSQL_ERRNO[-OB_ERR_INVALID_HEX_NUMBER] = -1; + SQLSTATE[-OB_ERR_INVALID_HEX_NUMBER] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_HEX_NUMBER] = "invalid hex number"; + STR_USER_ERROR[-OB_ERR_INVALID_HEX_NUMBER] = "invalid hex number"; + MYSQL_ERRNO[-OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE] = -1; + SQLSTATE[-OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "HY000"; + STR_ERROR[-OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "year conflicts with Julian date"; + STR_USER_ERROR[-OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "year conflicts with Julian date"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "day of year conflicts with Julian date"; + STR_USER_ERROR[-OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE] = "day of year conflicts with Julian date"; + MYSQL_ERRNO[-OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE] = -1; + SQLSTATE[-OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "HY000"; + STR_ERROR[-OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "month conflicts with Julian date"; + STR_USER_ERROR[-OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "month conflicts with Julian date"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "day of month conflicts with Julian date"; + STR_USER_ERROR[-OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE] = "day of month conflicts with Julian date"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE] = "day of week conflicts with Julian date"; + STR_USER_ERROR[-OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE] = "day of week conflicts with Julian date"; + MYSQL_ERRNO[-OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = -1; + SQLSTATE[-OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "HY000"; + STR_ERROR[-OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "hour conflicts with seconds in day"; + STR_USER_ERROR[-OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "hour conflicts with seconds in day"; + MYSQL_ERRNO[-OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = -1; + SQLSTATE[-OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "HY000"; + STR_ERROR[-OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "minutes of hour conflicts with seconds in day"; + STR_USER_ERROR[-OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY] = "minutes of hour conflicts with seconds in day"; + MYSQL_ERRNO[-OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY] = -1; + SQLSTATE[-OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY] = "HY000"; + STR_ERROR[-OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY] = "seconds of minute conflicts with seconds in day"; + STR_USER_ERROR[-OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY] = "seconds of minute conflicts with seconds in day"; + MYSQL_ERRNO[-OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED] = -1; + SQLSTATE[-OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED] = "HY000"; + STR_ERROR[-OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED] = "date not valid for month specified"; + STR_USER_ERROR[-OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED] = "date not valid for month specified"; + MYSQL_ERRNO[-OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH] = -1; + SQLSTATE[-OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH] = "HY000"; + STR_ERROR[-OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH] = "input value not long enough for date format"; + STR_USER_ERROR[-OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH] = "input value not long enough for date format"; + MYSQL_ERRNO[-OB_ERR_INVALID_YEAR_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_YEAR_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_YEAR_VALUE] = "(full) year must be between -4713 and +9999, and not be 0"; + STR_USER_ERROR[-OB_ERR_INVALID_YEAR_VALUE] = "(full) year must be between -4713 and +9999, and not be 0"; + MYSQL_ERRNO[-OB_ERR_INVALID_MONTH] = -1; + SQLSTATE[-OB_ERR_INVALID_MONTH] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_MONTH] = "not a valid month"; + STR_USER_ERROR[-OB_ERR_INVALID_MONTH] = "not a valid month"; + MYSQL_ERRNO[-OB_ERR_INVALID_DAY_OF_THE_WEEK] = -1; + SQLSTATE[-OB_ERR_INVALID_DAY_OF_THE_WEEK] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_DAY_OF_THE_WEEK] = "not a valid day of the week"; + STR_USER_ERROR[-OB_ERR_INVALID_DAY_OF_THE_WEEK] = "not a valid day of the week"; + MYSQL_ERRNO[-OB_ERR_INVALID_DAY_OF_YEAR_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_DAY_OF_YEAR_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_DAY_OF_YEAR_VALUE] = "day of year must be between 1 and 365 (366 for leap year)"; + STR_USER_ERROR[-OB_ERR_INVALID_DAY_OF_YEAR_VALUE] = "day of year must be between 1 and 365 (366 for leap year)"; + MYSQL_ERRNO[-OB_ERR_INVALID_HOUR12_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_HOUR12_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_HOUR12_VALUE] = "hour must be between 1 and 12"; + STR_USER_ERROR[-OB_ERR_INVALID_HOUR12_VALUE] = "hour must be between 1 and 12"; + MYSQL_ERRNO[-OB_ERR_INVALID_HOUR24_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_HOUR24_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_HOUR24_VALUE] = "hour must be between 0 and 23"; + STR_USER_ERROR[-OB_ERR_INVALID_HOUR24_VALUE] = "hour must be between 0 and 23"; + MYSQL_ERRNO[-OB_ERR_INVALID_MINUTES_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_MINUTES_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_MINUTES_VALUE] = "minutes must be between 0 and 59"; + STR_USER_ERROR[-OB_ERR_INVALID_MINUTES_VALUE] = "minutes must be between 0 and 59"; + MYSQL_ERRNO[-OB_ERR_INVALID_SECONDS_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_SECONDS_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_SECONDS_VALUE] = "seconds must be between 0 and 59"; + STR_USER_ERROR[-OB_ERR_INVALID_SECONDS_VALUE] = "seconds must be between 0 and 59"; + MYSQL_ERRNO[-OB_ERR_INVALID_SECONDS_IN_DAY_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_SECONDS_IN_DAY_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_SECONDS_IN_DAY_VALUE] = "seconds in day must be between 0 and 86399"; + STR_USER_ERROR[-OB_ERR_INVALID_SECONDS_IN_DAY_VALUE] = "seconds in day must be between 0 and 86399"; + MYSQL_ERRNO[-OB_ERR_INVALID_JULIAN_DATE_VALUE] = -1; + SQLSTATE[-OB_ERR_INVALID_JULIAN_DATE_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_JULIAN_DATE_VALUE] = "julian date must be between 1 and 5373484"; + STR_USER_ERROR[-OB_ERR_INVALID_JULIAN_DATE_VALUE] = "julian date must be between 1 and 5373484"; + MYSQL_ERRNO[-OB_ERR_AM_OR_PM_REQUIRED] = -1; + SQLSTATE[-OB_ERR_AM_OR_PM_REQUIRED] = "HY000"; + STR_ERROR[-OB_ERR_AM_OR_PM_REQUIRED] = "AM/A.M. or PM/P.M. required"; + STR_USER_ERROR[-OB_ERR_AM_OR_PM_REQUIRED] = "AM/A.M. or PM/P.M. required"; + MYSQL_ERRNO[-OB_ERR_BC_OR_AD_REQUIRED] = -1; + SQLSTATE[-OB_ERR_BC_OR_AD_REQUIRED] = "HY000"; + STR_ERROR[-OB_ERR_BC_OR_AD_REQUIRED] = "BC/B.C. or AD/A.D. required"; + STR_USER_ERROR[-OB_ERR_BC_OR_AD_REQUIRED] = "BC/B.C. or AD/A.D. required"; + MYSQL_ERRNO[-OB_ERR_FORMAT_CODE_APPEARS_TWICE] = -1; + SQLSTATE[-OB_ERR_FORMAT_CODE_APPEARS_TWICE] = "HY000"; + STR_ERROR[-OB_ERR_FORMAT_CODE_APPEARS_TWICE] = "format code appears twice"; + STR_USER_ERROR[-OB_ERR_FORMAT_CODE_APPEARS_TWICE] = "format code appears twice"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE] = "day of week may only be specified once"; + STR_USER_ERROR[-OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE] = "day of week may only be specified once"; + MYSQL_ERRNO[-OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD] = -1; + SQLSTATE[-OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD] = "HY000"; + STR_ERROR[-OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD] = "signed year precludes use of BC/AD"; + STR_USER_ERROR[-OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD] = "signed year precludes use of BC/AD"; + MYSQL_ERRNO[-OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR] = -1; + SQLSTATE[-OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR] = "HY000"; + STR_ERROR[-OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR] = "Julian date precludes use of day of year"; + STR_USER_ERROR[-OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR] = "Julian date precludes use of day of year"; + MYSQL_ERRNO[-OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE] = -1; + SQLSTATE[-OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE] = "HY000"; + STR_ERROR[-OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE] = "year may only be specified once"; + STR_USER_ERROR[-OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE] = "year may only be specified once"; + MYSQL_ERRNO[-OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE] = -1; + SQLSTATE[-OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE] = "HY000"; + STR_ERROR[-OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE] = "hour may only be specified once"; + STR_USER_ERROR[-OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE] = "hour may only be specified once"; + MYSQL_ERRNO[-OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT] = -1; + SQLSTATE[-OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT] = "HY000"; + STR_ERROR[-OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT] = "AM/PM conflicts with use of A.M./P.M."; + STR_USER_ERROR[-OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT] = "AM/PM conflicts with use of A.M./P.M."; + MYSQL_ERRNO[-OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT] = -1; + SQLSTATE[-OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT] = "HY000"; + STR_ERROR[-OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT] = "BC/AD conflicts with use of B.C./A.D."; + STR_USER_ERROR[-OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT] = "BC/AD conflicts with use of B.C./A.D."; + MYSQL_ERRNO[-OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE] = -1; + SQLSTATE[-OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE] = "HY000"; + STR_ERROR[-OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE] = "month may only be specified once"; + STR_USER_ERROR[-OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE] = "month may only be specified once"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE] = "day of week may only be specified once"; + STR_USER_ERROR[-OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE] = "day of week may only be specified once"; + MYSQL_ERRNO[-OB_ERR_FORMAT_CODE_CANNOT_APPEAR] = -1; + SQLSTATE[-OB_ERR_FORMAT_CODE_CANNOT_APPEAR] = "HY000"; + STR_ERROR[-OB_ERR_FORMAT_CODE_CANNOT_APPEAR] = "format code cannot appear in date input format"; + STR_USER_ERROR[-OB_ERR_FORMAT_CODE_CANNOT_APPEAR] = "format code cannot appear in date input format"; + MYSQL_ERRNO[-OB_ERR_NON_NUMERIC_CHARACTER_VALUE] = -1; + SQLSTATE[-OB_ERR_NON_NUMERIC_CHARACTER_VALUE] = "HY000"; + STR_ERROR[-OB_ERR_NON_NUMERIC_CHARACTER_VALUE] = "a non-numeric character was found where a numeric was expected"; + STR_USER_ERROR[-OB_ERR_NON_NUMERIC_CHARACTER_VALUE] = "a non-numeric character was found where a numeric was expected"; + MYSQL_ERRNO[-OB_INVALID_MERIDIAN_INDICATOR_USE] = -1; + SQLSTATE[-OB_INVALID_MERIDIAN_INDICATOR_USE] = "HY000"; + STR_ERROR[-OB_INVALID_MERIDIAN_INDICATOR_USE] = "'HH24' precludes use of meridian indicator"; + STR_USER_ERROR[-OB_INVALID_MERIDIAN_INDICATOR_USE] = "'HH24' precludes use of meridian indicator"; + MYSQL_ERRNO[-OB_ERR_DAY_OF_MONTH_RANGE] = -1; + SQLSTATE[-OB_ERR_DAY_OF_MONTH_RANGE] = "HY000"; + STR_ERROR[-OB_ERR_DAY_OF_MONTH_RANGE] = "day of month must be between 1 and last day of month"; + STR_USER_ERROR[-OB_ERR_DAY_OF_MONTH_RANGE] = "day of month must be between 1 and last day of month"; + MYSQL_ERRNO[-OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL] = -1; + SQLSTATE[-OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL] = "HY000"; + STR_ERROR[-OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL] = "the leading precision of the interval is too small"; + STR_USER_ERROR[-OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL] = "the leading precision of the interval is too small"; + MYSQL_ERRNO[-OB_ERR_INVALID_TIME_ZONE_HOUR] = -1; + SQLSTATE[-OB_ERR_INVALID_TIME_ZONE_HOUR] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_TIME_ZONE_HOUR] = "time zone hour must be between -12 and 14"; + STR_USER_ERROR[-OB_ERR_INVALID_TIME_ZONE_HOUR] = "time zone hour must be between -12 and 14"; + MYSQL_ERRNO[-OB_ERR_INVALID_TIME_ZONE_MINUTE] = -1; + SQLSTATE[-OB_ERR_INVALID_TIME_ZONE_MINUTE] = "HY000"; + STR_ERROR[-OB_ERR_INVALID_TIME_ZONE_MINUTE] = "time zone minute must be between -59 and 59"; + STR_USER_ERROR[-OB_ERR_INVALID_TIME_ZONE_MINUTE] = "time zone minute must be between -59 and 59"; + MYSQL_ERRNO[-OB_ERR_NOT_A_VALID_TIME_ZONE] = -1; + SQLSTATE[-OB_ERR_NOT_A_VALID_TIME_ZONE] = "HY000"; + STR_ERROR[-OB_ERR_NOT_A_VALID_TIME_ZONE] = "not a valid time zone"; + STR_USER_ERROR[-OB_ERR_NOT_A_VALID_TIME_ZONE] = "not a valid time zone"; + MYSQL_ERRNO[-OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER] = -1; + SQLSTATE[-OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER] = "HY000"; + STR_ERROR[-OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER] = "date format is too long for internal buffer"; + STR_USER_ERROR[-OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER] = "date format is too long for internal buffer"; + MYSQL_ERRNO[-OB_INVALID_ROWID] = -1; + SQLSTATE[-OB_INVALID_ROWID] = "HY000"; + STR_ERROR[-OB_INVALID_ROWID] = "invalid ROWID"; + STR_USER_ERROR[-OB_INVALID_ROWID] = "invalid ROWID"; + MYSQL_ERRNO[-OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR] = -1; + SQLSTATE[-OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR] = "HY000"; + STR_ERROR[-OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR] = "Datetime/Interval internal error"; + STR_USER_ERROR[-OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR] = "Datetime/Interval internal error"; MYSQL_ERRNO[-OB_ERR_FETCH_OUT_SEQUENCE] = 1002; SQLSTATE[-OB_ERR_FETCH_OUT_SEQUENCE] = "HY000"; STR_ERROR[-OB_ERR_FETCH_OUT_SEQUENCE] = "ORA-01002: fetch out of sequence"; @@ -2674,16 +2859,16 @@ static struct ObStrErrorInit STR_USER_ERROR[-OB_ERR_DISTRIBUTED_TRANSACTION_NOT_SUPPORTED] = "Cannot execute update SQLs cross databases in transaction"; MYSQL_ERRNO[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = -1; SQLSTATE[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = "HY000"; - STR_ERROR[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = "sharding sql unsupported"; - STR_USER_ERROR[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = "sharding sql unsupported"; + STR_ERROR[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = "Expr not supported yet"; + STR_USER_ERROR[-OB_ERROR_UNSUPPORT_EXPR_TYPE] = "Expr not supported yet"; MYSQL_ERRNO[-OB_ERR_FORMAT_FOR_TESTLOAD_TABLE_MAP] = -1; SQLSTATE[-OB_ERR_FORMAT_FOR_TESTLOAD_TABLE_MAP] = "HY000"; STR_ERROR[-OB_ERR_FORMAT_FOR_TESTLOAD_TABLE_MAP] = "DatabaseVariables\'s format of testloadTableMap error"; STR_USER_ERROR[-OB_ERR_FORMAT_FOR_TESTLOAD_TABLE_MAP] = "DatabaseVariables\'s format (\'%s\') of testloadTableMap error"; MYSQL_ERRNO[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = -1; SQLSTATE[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = "HY000"; - STR_ERROR[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = "Table count in testload request isn\'t 1 with \'table_name\' hint"; - STR_USER_ERROR[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = "Table count in testload request isn\'t 1 with \'table_name\' hint %ld"; + STR_ERROR[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = "Table count isn\'t 1 with \'table_name\' hint"; + STR_USER_ERROR[-OB_ERR_MORE_TABLES_WITH_TABLE_HINT] = "Table count isn\'t 1 with \'table_name\' hint %ld"; MYSQL_ERRNO[-OB_ERR_GET_PHYSIC_INDEX_BY_RULE] = -1; SQLSTATE[-OB_ERR_GET_PHYSIC_INDEX_BY_RULE] = "HY000"; STR_ERROR[-OB_ERR_GET_PHYSIC_INDEX_BY_RULE] = "Cannot get route info by user\'s rules and partition values"; @@ -2700,6 +2885,10 @@ static struct ObStrErrorInit SQLSTATE[-OB_ERR_BATCH_INSERT_FOUND] = "HY000"; STR_ERROR[-OB_ERR_BATCH_INSERT_FOUND] = "Batch insert is not supported for sharding table"; STR_USER_ERROR[-OB_ERR_BATCH_INSERT_FOUND] = "Batch insert is not supported for sharding table"; + MYSQL_ERRNO[-OB_ERR_UNSUPPORT_DIFF_TOPOLOGY] = 7202; + SQLSTATE[-OB_ERR_UNSUPPORT_DIFF_TOPOLOGY] = "HY000"; + STR_ERROR[-OB_ERR_UNSUPPORT_DIFF_TOPOLOGY] = "Not support different topology table join"; + STR_USER_ERROR[-OB_ERR_UNSUPPORT_DIFF_TOPOLOGY] = "Not support different topology table join"; MYSQL_ERRNO[-OB_SESSION_POOL_CMD_ERROR] = -1; SQLSTATE[-OB_SESSION_POOL_CMD_ERROR] = "HY000"; STR_ERROR[-OB_SESSION_POOL_CMD_ERROR] = "Session pool error"; @@ -2708,6 +2897,14 @@ static struct ObStrErrorInit SQLSTATE[-OB_SESSION_POOL_FULL_ERROR] = "HY000"; STR_ERROR[-OB_SESSION_POOL_FULL_ERROR] = "Session pool is full"; STR_USER_ERROR[-OB_SESSION_POOL_FULL_ERROR] = "Session is full"; + MYSQL_ERRNO[-OB_ERR_SHARD_DDL_UNEXPECTED] = ER_SYNTAX_ERROR; //8204; + SQLSTATE[-OB_ERR_SHARD_DDL_UNEXPECTED] = "HY000"; + STR_ERROR[-OB_ERR_SHARD_DDL_UNEXPECTED] = "Execute DDL job error"; + STR_USER_ERROR[-OB_ERR_SHARD_DDL_UNEXPECTED] = "Execute DDL job error"; + MYSQL_ERRNO[-OB_ERR_CAN_NOT_PASS_WHITELIST] = -1; + SQLSTATE[-OB_ERR_CAN_NOT_PASS_WHITELIST] = "HY000"; + STR_ERROR[-OB_ERR_CAN_NOT_PASS_WHITELIST] = "Can not pass whitelist"; + STR_USER_ERROR[-OB_ERR_CAN_NOT_PASS_WHITELIST] = "Can not pass whitelist"; } } local_init; diff --git a/src/lib/ob_errno.h b/src/lib/ob_errno.h index ba4772e1b46cf011f9f7303ac1271b0684d65716..39f10dbe42d87e4ca357d6d5f9a879083f4a2e59 100644 --- a/src/lib/ob_errno.h +++ b/src/lib/ob_errno.h @@ -181,6 +181,7 @@ namespace common static const int OB_COLUMN_GROUP_NOT_FOUND = -4185; static const int OB_CS_COMPRESS_LIB_ERROR = -4186; static const int OB_ITEM_NOT_MATCH = -4187; + static const int OB_INVALID_DATE_FORMAT_END = -4190; static const int OB_HASH_EXIST = -4200; static const int OB_HASH_NOT_EXIST = -4201; static const int OB_HASH_GET_TIMEOUT = -4204; @@ -608,6 +609,49 @@ namespace common static const int OB_ERR_TOO_MANY_ROWS = -5294; static const int OB_WRONG_FIELD_TERMINATORS = -5295; static const int OB_ERR_UNEXPECTED_TZ_TRANSITION = -5296; + static const int OB_ERR_INVALID_TIMEZONE_REGION_ID = -5341; + static const int OB_ERR_INVALID_HEX_NUMBER = -5342; + static const int OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE = -5629; + static const int OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE = -5630; + static const int OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE = -5631; + static const int OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE = -5632; + static const int OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE = -5633; + static const int OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY = -5634; + static const int OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY = -5635; + static const int OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY = -5636; + static const int OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED = -5637; + static const int OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH = -5638; + static const int OB_ERR_INVALID_YEAR_VALUE = -5639; + static const int OB_ERR_INVALID_MONTH = -5641; + static const int OB_ERR_INVALID_DAY_OF_THE_WEEK = -5642; + static const int OB_ERR_INVALID_DAY_OF_YEAR_VALUE = -5643; + static const int OB_ERR_INVALID_HOUR12_VALUE = -5644; + static const int OB_ERR_INVALID_HOUR24_VALUE = -5645; + static const int OB_ERR_INVALID_MINUTES_VALUE = -5646; + static const int OB_ERR_INVALID_SECONDS_VALUE = -5647; + static const int OB_ERR_INVALID_SECONDS_IN_DAY_VALUE = -5648; + static const int OB_ERR_INVALID_JULIAN_DATE_VALUE = -5649; + static const int OB_ERR_AM_OR_PM_REQUIRED = -5650; + static const int OB_ERR_BC_OR_AD_REQUIRED = -5651; + static const int OB_ERR_FORMAT_CODE_APPEARS_TWICE = -5652; + static const int OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE = -5653; + static const int OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD = -5654; + static const int OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR = -5655; + static const int OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE = -5656; + static const int OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE = -5657; + static const int OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT = -5658; + static const int OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT = -5659; + static const int OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE = -5660; + static const int OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE = -5661; + static const int OB_ERR_FORMAT_CODE_CANNOT_APPEAR = -5662; + static const int OB_ERR_NON_NUMERIC_CHARACTER_VALUE = -5663; + static const int OB_INVALID_MERIDIAN_INDICATOR_USE = -5664; + static const int OB_ERR_DAY_OF_MONTH_RANGE = -5667; + static const int OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL = -5708; + static const int OB_ERR_INVALID_TIME_ZONE_HOUR = -5709; + static const int OB_ERR_INVALID_TIME_ZONE_MINUTE = -5710; + static const int OB_ERR_NOT_A_VALID_TIME_ZONE = -5711; + static const int OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER = -5712; static const int OB_ERR_REROUTE = -5727; static const int OB_ERR_REGEXP_NOMATCH = -5809; static const int OB_ERR_REGEXP_BADPAT = -5810; @@ -627,6 +671,7 @@ namespace common static const int OB_ERR_REGEXP_BADOPT = -5824; static const int OB_ERR_REGEXP_ETOOBIG = -5825; static const int OB_INVALID_ROWID = -5802; + static const int OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR = -5898; static const int OB_ERR_FETCH_OUT_SEQUENCE = -5931; static const int OB_TRANSACTION_SET_VIOLATION = -6001; static const int OB_TRANS_ROLLBACKED = -6002; @@ -707,19 +752,21 @@ namespace common static const int OB_EXPR_CALC_ERROR = -8057; static const int OB_EXPR_COLUMN_NOT_EXIST = -8058; static const int OB_ERROR_UNSUPPORT_EXPR_TYPE = -8059; - static const int OB_ERROR_UNSUPPORT_HAS_AGG_EXPR_TYPE = -8060; static const int OB_ERR_FORMAT_FOR_TESTLOAD_TABLE_MAP = -8061; static const int OB_ERR_MORE_TABLES_WITH_TABLE_HINT = -8062; static const int OB_ERR_GET_PHYSIC_INDEX_BY_RULE = -8063; static const int OB_ERR_TESTLOAD_ALIPAY_COMPATIBLE = -8064; static const int OB_ERR_NULL_DB_VAL_TESTLOAD_TABLE_MAP = -8065; static const int OB_ERR_BATCH_INSERT_FOUND = -8066; + static const int OB_ERR_UNSUPPORT_DIFF_TOPOLOGY = -8067; static const int OB_SESSION_POOL_CMD_ERROR = -8101; static const int OB_SESSION_POOL_FULL_ERROR = -8102; static const int OB_ERR_NO_ZONE_SHARD_TPO = -8200; static const int OB_ERR_NO_DEFAULT_SHARD_TPO = -8201; static const int OB_ERR_NO_TABLE_RULE = -8202; static const int OB_ERR_DISTRIBUTED_TRANSACTION_NOT_SUPPORTED = -8203; + static const int OB_ERR_SHARD_DDL_UNEXPECTED = -8204; + static const int OB_ERR_CAN_NOT_PASS_WHITELIST = -8205; #define OB_SUCCESS__USER_ERROR_MSG "Success" #define OB_ERROR__USER_ERROR_MSG "Common error" @@ -874,6 +921,7 @@ namespace common #define OB_COLUMN_GROUP_NOT_FOUND__USER_ERROR_MSG "Column group not found" #define OB_CS_COMPRESS_LIB_ERROR__USER_ERROR_MSG "ChunkServer failed to get compress library" #define OB_ITEM_NOT_MATCH__USER_ERROR_MSG "Item not match" +#define OB_INVALID_DATE_FORMAT_END__USER_ERROR_MSG "Invalid date format" #define OB_HASH_EXIST__USER_ERROR_MSG "hash map/set entry exist" #define OB_HASH_NOT_EXIST__USER_ERROR_MSG "hash map/set entry not exist" #define OB_HASH_GET_TIMEOUT__USER_ERROR_MSG "hash map/set get timeout" @@ -1300,6 +1348,51 @@ namespace common #define OB_ERR_TOO_MANY_ROWS__USER_ERROR_MSG "Result consisted of more than one row" #define OB_WRONG_FIELD_TERMINATORS__USER_ERROR_MSG "Field separator argument is not what is expected; check the manual" #define OB_ERR_UNEXPECTED_TZ_TRANSITION__USER_ERROR_MSG "unexpected time zone info transition" +#define OB_ERR_INVALID_TIMEZONE_REGION_ID__USER_ERROR_MSG "timezone region ID is invalid" +#define OB_ERR_INVALID_HEX_NUMBER__USER_ERROR_MSG "invalid hex number" +#define OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE__USER_ERROR_MSG "year conflicts with Julian date" +#define OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE__USER_ERROR_MSG "day of year conflicts with Julian date" +#define OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE__USER_ERROR_MSG "month conflicts with Julian date" +#define OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE__USER_ERROR_MSG "day of month conflicts with Julian date" +#define OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE__USER_ERROR_MSG "day of week conflicts with Julian date" +#define OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY__USER_ERROR_MSG "hour conflicts with seconds in day" +#define OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY__USER_ERROR_MSG "minutes of hour conflicts with seconds in day" +#define OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY__USER_ERROR_MSG "seconds of minute conflicts with seconds in day" +#define OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED__USER_ERROR_MSG "date not valid for month specified" +#define OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH__USER_ERROR_MSG "input value not long enough for date format" +#define OB_ERR_INVALID_YEAR_VALUE__USER_ERROR_MSG "(full) year must be between -4713 and +9999, and not be 0" +#define OB_ERR_INVALID_MONTH__USER_ERROR_MSG "not a valid month" +#define OB_ERR_INVALID_DAY_OF_THE_WEEK__USER_ERROR_MSG "not a valid day of the week" +#define OB_ERR_INVALID_DAY_OF_YEAR_VALUE__USER_ERROR_MSG "day of year must be between 1 and 365 (366 for leap year)" +#define OB_ERR_INVALID_HOUR12_VALUE__USER_ERROR_MSG "hour must be between 1 and 12" +#define OB_ERR_INVALID_HOUR24_VALUE__USER_ERROR_MSG "hour must be between 0 and 23" +#define OB_ERR_INVALID_MINUTES_VALUE__USER_ERROR_MSG "minutes must be between 0 and 59" +#define OB_ERR_INVALID_SECONDS_VALUE__USER_ERROR_MSG "seconds must be between 0 and 59" +#define OB_ERR_INVALID_SECONDS_IN_DAY_VALUE__USER_ERROR_MSG "seconds in day must be between 0 and 86399" +#define OB_ERR_INVALID_JULIAN_DATE_VALUE__USER_ERROR_MSG "julian date must be between 1 and 5373484" +#define OB_ERR_AM_OR_PM_REQUIRED__USER_ERROR_MSG "AM/A.M. or PM/P.M. required" +#define OB_ERR_BC_OR_AD_REQUIRED__USER_ERROR_MSG "BC/B.C. or AD/A.D. required" +#define OB_ERR_FORMAT_CODE_APPEARS_TWICE__USER_ERROR_MSG "format code appears twice" +#define OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE__USER_ERROR_MSG "day of week may only be specified once" +#define OB_ERR_SIGNED_YEAR_PRECLUDES_USE_OF_BC_AD__USER_ERROR_MSG "signed year precludes use of BC/AD" +#define OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR__USER_ERROR_MSG "Julian date precludes use of day of year" +#define OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE__USER_ERROR_MSG "year may only be specified once" +#define OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE__USER_ERROR_MSG "hour may only be specified once" +#define OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT__USER_ERROR_MSG "AM/PM conflicts with use of A.M./P.M." +#define OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT__USER_ERROR_MSG "BC/AD conflicts with use of B.C./A.D." +#define OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE__USER_ERROR_MSG "month may only be specified once" +#define OB_ERR_DAY_OF_WEEK_MAY_ONLY_BE_SPECIFIED_ONCE__USER_ERROR_MSG "day of week may only be specified once" +#define OB_ERR_FORMAT_CODE_CANNOT_APPEAR__USER_ERROR_MSG "format code cannot appear in date input format" +#define OB_ERR_NON_NUMERIC_CHARACTER_VALUE__USER_ERROR_MSG "a non-numeric character was found where a numeric was expected" +#define OB_INVALID_MERIDIAN_INDICATOR_USE__USER_ERROR_MSG "'HH24' precludes use of meridian indicator" +#define OB_ERR_DAY_OF_MONTH_RANGE__ORA_USER_ERROR_MSG "ORA-01847: day of month must be between 1 and last day of month" +#define OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL__USER_ERROR_MSG "the leading precision of the interval is too small" +#define OB_ERR_INVALID_TIME_ZONE_HOUR__USER_ERROR_MSG "time zone hour must be between -12 and 14" +#define OB_ERR_INVALID_TIME_ZONE_MINUTE__USER_ERROR_MSG "time zone minute must be between -59 and 59" +#define OB_ERR_NOT_A_VALID_TIME_ZONE__USER_ERROR_MSG "not a valid time zone" +#define OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER__USER_ERROR_MSG "date format is too long for internal buffer" +#define OB_INVALID_ROWID__USER_ERROR_MSG "invalid ROWID" +#define OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR__USER_ERROR_MSG "Datetime/Interval internal error" #define OB_ERR_FETCH_OUT_SEQUENCE__USER_ERROR_MSG "ORA-01002: fetch out of sequence" #define OB_TRANSACTION_SET_VIOLATION__USER_ERROR_MSG "Transaction set changed during the execution" #define OB_TRANS_ROLLBACKED__USER_ERROR_MSG "transaction is rolled back" diff --git a/src/lib/ob_name_def.h b/src/lib/ob_name_def.h index 279796b2a89c889c76dfaedf2c9570e9749e43b3..fbe99d60fade102bdbc0e01bb42557ef9e9057ec 100644 --- a/src/lib/ob_name_def.h +++ b/src/lib/ob_name_def.h @@ -49,6 +49,7 @@ #define N_INDEX_RANGE "index_range" #define N_LIMIT "limit" #define N_OFFSET "offset" +#define N_ERROR_ON_OVERLAP_TIME "error_on_overlap_time" #define N_LIMIT_OFFSET "limit_offset" #define N_READ_METHOD "read_method" #define N_SCAN "scan" diff --git a/src/lib/objectpool/ob_concurrency_objpool.cpp b/src/lib/objectpool/ob_concurrency_objpool.cpp index c669bd10e95e678782f048f587e2b4e867da7e2d..819343194051bbae52b0139924e2ea54c7855287 100644 --- a/src/lib/objectpool/ob_concurrency_objpool.cpp +++ b/src/lib/objectpool/ob_concurrency_objpool.cpp @@ -292,7 +292,7 @@ inline bool ObObjFreeList::need_to_reclaim(ObThreadCache *thread_cache) { bool ret = false; if (reclaim_opt_.enable_reclaim_) { - if(thread_cache->nr_free_ >= thread_cache->nr_average_ + if(thread_cache->nr_free_ >= static_cast(thread_cache->nr_average_) && thread_cache->nr_free_chunks_ > chunk_count_ && thread_cache->nr_total_ > std::max(obj_count_base_, obj_count_per_chunk_)) { if (thread_cache->nr_overage_++ >= reclaim_opt_.max_overage_) { diff --git a/src/lib/oblog/ob_async_log_struct.cpp b/src/lib/oblog/ob_async_log_struct.cpp index a314de746392ad7bf5510f353743522c52bf2dc0..657b1576091f2ea89fbe1741a06b7c2d0cc0b62c 100644 --- a/src/lib/oblog/ob_async_log_struct.cpp +++ b/src/lib/oblog/ob_async_log_struct.cpp @@ -171,7 +171,8 @@ int ObLogFileStruct::reopen_wf() LOG_STDERR("invalid argument log_file = %p\n", filename_); ret = OB_INVALID_ARGUMENT; } else { - char tmp_file_name[MAX_LOG_FILE_NAME_SIZE]; + const int64_t WARN_NAME_LEN = 3; + char tmp_file_name[MAX_LOG_FILE_NAME_SIZE + WARN_NAME_LEN]; snprintf(tmp_file_name, sizeof(tmp_file_name), "%s.wf", filename_); if (OB_UNLIKELY(tmp_fd = ::open(tmp_file_name, O_WRONLY | O_CREAT | O_APPEND , LOG_FILE_MODE)) < 0) { LOG_STDERR("open file = %s errno = %d error = %m\n", tmp_file_name, errno); diff --git a/src/lib/oblog/ob_log.cpp b/src/lib/oblog/ob_log.cpp index 0dd49e9815c0647ed9556e45b3fb5ff207644463..6bb5e9fff33a2e8f5aa87c1086057da51e2c8f1c 100644 --- a/src/lib/oblog/ob_log.cpp +++ b/src/lib/oblog/ob_log.cpp @@ -647,7 +647,7 @@ void ObLogger::log_head_info(const ObLogFDType type, "%04d-%02d-%02d %02d:%02d:%02d.%06ld [%s] ", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, tv.tv_usec, errstr_[level]); - } else if (FD_CONFIG_FILE == type) { // header for config file + } else if (FD_CONFIG_FILE == type) { // header for config file /** Format for config_log is: '###${content}' * '###' is consultation flag for Inspection Module * ${content} is value of config in json format. */ @@ -954,7 +954,7 @@ void ObLogger::check_file(ObLogFileStruct &log_struct, const bool redirect_flag) } if (log_struct.open_wf_flag_) { - char wf_file_name[ObLogFileStruct::MAX_LOG_FILE_NAME_SIZE]; + char wf_file_name[ObLogFileStruct::MAX_LOG_FILE_NAME_SIZE + 3]; memset(wf_file_name, 0, sizeof(wf_file_name)); snprintf(wf_file_name, sizeof(wf_file_name), "%s.wf", log_struct.filename_); err = stat(wf_file_name, &st_file); @@ -1585,7 +1585,7 @@ void ObLogger::do_async_flush_to_file(ObLogItem **log_item, const int64_t count) if (max_file_size_ > 0) { for (int32_t i = 0; i < static_cast(MAX_FD_FILE); i++) { - if (OB_LIKELY(need_auto_rotate_log_by_size(static_cast(i)))) { + if (OB_LIKELY(need_auto_rotate_log_by_size(static_cast(i)))) { const bool redirect_flag = (static_cast(FD_DEFAULT_FILE) == i ? redirect_flag_ : false); rotate_log(writen[i], redirect_flag, log_file_[i]); } @@ -2017,6 +2017,58 @@ void ObLogger::async_log_message(const ObLogFDType type, } } + +void ObLogger::log_message_kv(const char *mod_name, + const int32_t level, + const char *file, + const int32_t line, + const char *function, + const char *info_string) +{ + const ObLogFDType type = (NULL == mod_name ? FD_XFLUSH_FILE : FD_DEFAULT_FILE); + log_message_kv(type, mod_name, level, file, line, function, info_string); +} + +void ObLogger::log_message_kv(const ObLogFDType type, + const char *mod_name, + const int32_t level, + const char *file, + const int32_t line, + const char *function, + const char *info_string) +{ + int ret = common::OB_SUCCESS; + LogBuffer *log_buffer = NULL; + if (OB_NOT_NULL(info_string)) { + if (get_trace_mode()) { + if (OB_LIKELY(is_enable_logging()) + && OB_NOT_NULL(log_buffer = get_thread_buffer()) + && OB_LIKELY(!log_buffer->is_oversize())) { + set_disable_logging(true); + log_head_info(type, mod_name, level, LogLocation(file, line, function), *log_buffer); + int64_t &pos = log_buffer->pos_; + char *data = log_buffer->buffer_; + LOG_PRINT_INFO(info_string); + log_tail(level, *log_buffer); + set_disable_logging(false); + } + } else if (is_async_log_used()) { + ret = async_log_message_kv(type, mod_name, level, LogLocation(file, line, function), info_string, + static_cast(strlen(info_string))); + } else if (OB_LIKELY(is_enable_logging())) {//sync away + set_disable_logging(true); + if (OB_NOT_NULL(log_buffer = get_thread_buffer()) + && OB_LIKELY(!log_buffer->is_oversize())) { + int64_t &pos = log_buffer->pos_; + char *data = log_buffer->buffer_; + LOG_PRINT_INFO(info_string); + log_data(type, mod_name, level, LogLocation(file, line, function), *log_buffer); + set_disable_logging(false); + } + } + } +} + } } diff --git a/src/lib/oblog/ob_log.h b/src/lib/oblog/ob_log.h index 2bc2d4a8223318cdce086aba98279aa4eb67725f..26359f58664dfd1f595fa0aa1e2e1677f57068ae 100644 --- a/src/lib/oblog/ob_log.h +++ b/src/lib/oblog/ob_log.h @@ -361,14 +361,14 @@ public: const int errcode, const char *fmt, ...) __attribute__((format(printf, 4, 5))); - inline void log_message_kv(const char *mod_name, + void log_message_kv(const char *mod_name, const int32_t level, const char *file, const int32_t line, const char *function, const char *info_string); - inline void log_message_kv(const ObLogFDType type, + void log_message_kv(const ObLogFDType type, const char *mod_name, const int32_t level, const char *file, @@ -786,57 +786,6 @@ inline void ObLogger::check_log_end(ObLogItem &log_item, int64_t pos) } } -inline void ObLogger::log_message_kv(const char *mod_name, - const int32_t level, - const char *file, - const int32_t line, - const char *function, - const char *info_string) -{ - const ObLogFDType type = (NULL == mod_name ? FD_XFLUSH_FILE : FD_DEFAULT_FILE); - log_message_kv(type, mod_name, level, file, line, function, info_string); -} - -inline void ObLogger::log_message_kv(const ObLogFDType type, - const char *mod_name, - const int32_t level, - const char *file, - const int32_t line, - const char *function, - const char *info_string) -{ - int ret = common::OB_SUCCESS; - LogBuffer *log_buffer = NULL; - if (OB_NOT_NULL(info_string)) { - if (get_trace_mode()) { - if (OB_LIKELY(is_enable_logging()) - && OB_NOT_NULL(log_buffer = get_thread_buffer()) - && OB_LIKELY(!log_buffer->is_oversize())) { - set_disable_logging(true); - log_head_info(type, mod_name, level, LogLocation(file, line, function), *log_buffer); - int64_t &pos = log_buffer->pos_; - char *data = log_buffer->buffer_; - LOG_PRINT_INFO(info_string); - log_tail(level, *log_buffer); - set_disable_logging(false); - } - } else if (is_async_log_used()) { - ret = async_log_message_kv(type, mod_name, level, LogLocation(file, line, function), info_string, - static_cast(strlen(info_string))); - } else if (OB_LIKELY(is_enable_logging())) {//sync away - set_disable_logging(true); - if (OB_NOT_NULL(log_buffer = get_thread_buffer()) - && OB_LIKELY(!log_buffer->is_oversize())) { - int64_t &pos = log_buffer->pos_; - char *data = log_buffer->buffer_; - LOG_PRINT_INFO(info_string); - log_data(type, mod_name, level, LogLocation(file, line, function), *log_buffer); - set_disable_logging(false); - } - } - } -} - inline void ObLogger::log_user_message_info( const char *mod_name, UserMsgLevel user_msg_level, diff --git a/src/lib/oblog/ob_log_module.h b/src/lib/oblog/ob_log_module.h index 014ee1d1eda5106ff1dcfcbd1c672cda0e457a5a..bf950454b1d62b613069ee1ae1eaa78ffbd29eb6 100644 --- a/src/lib/oblog/ob_log_module.h +++ b/src/lib/oblog/ob_log_module.h @@ -187,6 +187,7 @@ LOG_MOD_END(STORAGETEST) #define OB_XFLUSH_LOG_NEED_TO_PRINT(level) OB_LOGGER.need_to_print_xflush(OB_LOG_LEVEL_##level) #define OB_MONITOR_LOG_NEED_TO_PRINT(level) OB_LOGGER.need_to_print_monitor(OB_LOG_LEVEL_##level) #define OB_NEED_USE_ASYNC_LOG (OB_LOGGER.is_async_log_used() && !OB_LOGGER.get_trace_mode()) +#define IS_DEBUG_ENABLED() OB_LOGGER.need_to_print(OB_LOG_LEVEL_DEBUG) #define OB_PRINT(modName, level, infoString, args...) \ OB_LOGGER.log_message_kv(modName, OB_LOG_LEVEL(level), infoString, ##args) diff --git a/src/lib/oblog/ob_log_print_kv.h b/src/lib/oblog/ob_log_print_kv.h index 54ea26bc3463f6c8cad8077d4dbcb13d814274aa..fedee40e40504cc1b44e0b7c7912e9d96624371b 100644 --- a/src/lib/oblog/ob_log_print_kv.h +++ b/src/lib/oblog/ob_log_print_kv.h @@ -254,11 +254,7 @@ int logdata_print_key_obj(char *buf, const int64_t buf_len, int64_t &pos, const const bool with_comma, const T &obj) { int ret = OB_SUCCESS; - if (NULL == &obj) { - ret = logdata_printf(buf, buf_len, pos, WITH_COMMA("%s=NULL"), key); - } else { - ret = logdata_print_key_obj(buf, buf_len, pos, key, with_comma, obj, BoolType<__is_enum(T)>()); - } + ret = logdata_print_key_obj(buf, buf_len, pos, key, with_comma, obj, BoolType<__is_enum(T)>()); return ret; } template diff --git a/src/lib/string/ob_string.h b/src/lib/string/ob_string.h index 9218152d0b9215411ccbea462f57f83f274f981b..defd36fc0e38a337966b222ac710d46b52f6cca8 100644 --- a/src/lib/string/ob_string.h +++ b/src/lib/string/ob_string.h @@ -523,6 +523,14 @@ public: // or @c NULL if @a c is not found. const char *find(char c) const { return static_cast(memchr(ptr_, c, data_length_)); } + // Reverse Find a character from backwards. + // @return A pointer to the first occurrence of @a c in @a this + // or @c NULL if @a c is not found. + const char* reverse_find(char c) const + { + return static_cast(memrchr(ptr_, c, data_length_)); + } + // Split the buffer on the character at @a p. // // The buffer is split in to two parts and the character at @a p diff --git a/src/lib/timezone/ob_oracle_format_models.cpp b/src/lib/timezone/ob_oracle_format_models.cpp new file mode 100644 index 0000000000000000000000000000000000000000..436aa32bea22e71c85e6f4ddc9765d3a88a50437 --- /dev/null +++ b/src/lib/timezone/ob_oracle_format_models.cpp @@ -0,0 +1,687 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX LIB_TIME + +#include "lib/timezone/ob_oracle_format_models.h" +#include "lib/timezone/ob_time_convert.h" + +namespace oceanbase { +namespace common { + +const ObOracleTimeLimiter ObDFMLimit::YEAR(1, 9999, OB_ERR_INVALID_YEAR_VALUE); +const ObOracleTimeLimiter ObDFMLimit::MONTH(1, 12, OB_ERR_INVALID_MONTH); +const ObOracleTimeLimiter ObDFMLimit::MONTH_DAY(1, 31, OB_ERR_DAY_OF_MONTH_RANGE); +const ObOracleTimeLimiter ObDFMLimit::WEEK_DAY(1, 7, OB_ERR_INVALID_DAY_OF_THE_WEEK); +const ObOracleTimeLimiter ObDFMLimit::YEAR_DAY(1, 366, OB_ERR_INVALID_DAY_OF_YEAR_VALUE); +const ObOracleTimeLimiter ObDFMLimit::HOUR12(1, 12, OB_ERR_INVALID_HOUR12_VALUE); +const ObOracleTimeLimiter ObDFMLimit::HOUR24(0, 23, OB_ERR_INVALID_HOUR24_VALUE); +const ObOracleTimeLimiter ObDFMLimit::MINUTE(0, 59, OB_ERR_INVALID_MINUTES_VALUE); +const ObOracleTimeLimiter ObDFMLimit::SECOND(0, 59, OB_ERR_INVALID_SECONDS_VALUE); +const ObOracleTimeLimiter ObDFMLimit::SECS_PAST_MIDNIGHT(0, 86399, OB_ERR_INVALID_SECONDS_IN_DAY_VALUE); +const ObOracleTimeLimiter ObDFMLimit::TIMEZONE_HOUR_ABS(0, 15, OB_INVALID_DATE_VALUE); // ORA-01874: time zone hour must be between -15 and 15 +const ObOracleTimeLimiter ObDFMLimit::TIMEZONE_MIN_ABS(0, 59, OB_INVALID_DATE_VALUE); // ORA-01875: time zone minute must be between -59 and 59 +const ObOracleTimeLimiter ObDFMLimit::JULIAN_DATE(1, 5373484, OB_ERR_INVALID_JULIAN_DATE_VALUE); // -4712-01-01 ~ 9999-12-31 + + +const ObTimeConstStr ObDFMFlag::PATTERN[ObDFMFlag::MAX_FLAG_NUMBER] = +{ + ObTimeConstStr("AD"), + ObTimeConstStr("A.D."), + ObTimeConstStr("BC"), + ObTimeConstStr("B.C."), + ObTimeConstStr("CC"), + ObTimeConstStr("SCC"), + ObTimeConstStr("D"), + ObTimeConstStr("DAY"), + ObTimeConstStr("DD"), + ObTimeConstStr("DDD"), + ObTimeConstStr("DY"), + ObTimeConstStr("FF1"), + ObTimeConstStr("FF2"), + ObTimeConstStr("FF3"), + ObTimeConstStr("FF4"), + ObTimeConstStr("FF5"), + ObTimeConstStr("FF6"), + ObTimeConstStr("FF7"), + ObTimeConstStr("FF8"), + ObTimeConstStr("FF9"), + ObTimeConstStr("FF"), + ObTimeConstStr("HH"), + ObTimeConstStr("HH24"), + ObTimeConstStr("HH12"), + ObTimeConstStr("IW"), + ObTimeConstStr("I"), + ObTimeConstStr("IY"), + ObTimeConstStr("IYY"), + ObTimeConstStr("IYYY"), + ObTimeConstStr("MI"), + ObTimeConstStr("MM"), + ObTimeConstStr("MONTH"), + ObTimeConstStr("MON"), + ObTimeConstStr("AM"), + ObTimeConstStr("A.M."), + ObTimeConstStr("PM"), + ObTimeConstStr("P.M."), + ObTimeConstStr("Q"), + ObTimeConstStr("RR"), + ObTimeConstStr("RRRR"), + ObTimeConstStr("SS"), + ObTimeConstStr("SSSSS"), + ObTimeConstStr("WW"), + ObTimeConstStr("W"), + ObTimeConstStr("Y,YYY"), + ObTimeConstStr("YEAR"), + ObTimeConstStr("SYEAR"), + ObTimeConstStr("YYYY"), + ObTimeConstStr("SYYYY"), + ObTimeConstStr("YYY"), + ObTimeConstStr("YY"), + ObTimeConstStr("Y"), + ObTimeConstStr("DS"), + ObTimeConstStr("DL"), + ObTimeConstStr("TZH"), + ObTimeConstStr("TZM"), + ObTimeConstStr("TZD"), + ObTimeConstStr("TZR"), + ObTimeConstStr("X"), + ObTimeConstStr("J"), +}; + +const int ObDFMFlag::CONFLICT_GROUP_MAP[ObDFMFlag::MAX_FLAG_NUMBER] = { + /**AD*/ ObDFMFlag::ERA_GROUP, + /**AD2*/ ObDFMFlag::ERA_GROUP, + /**BC*/ ObDFMFlag::ERA_GROUP, + /**BC2*/ ObDFMFlag::ERA_GROUP, + /**CC,*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**SCC*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**D*/ ObDFMFlag::WEEK_OF_DAY_GROUP, + /**DAY*/ ObDFMFlag::WEEK_OF_DAY_GROUP, + /**DD*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**DDD*/ ObDFMFlag::DAY_OF_YEAR_GROUP, + /**DY*/ ObDFMFlag::WEEK_OF_DAY_GROUP, + /**FF1*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF2*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF3*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF4*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF5*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF6*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF7*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF8*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF9*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**FF*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**HH*/ ObDFMFlag::HOUR_GROUP, + /**HH24*/ ObDFMFlag::HOUR_GROUP, + /**HH12*/ ObDFMFlag::HOUR_GROUP, + /**IW*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**I*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**IY*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**IYY*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**IYYY*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**MI*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**MM*/ ObDFMFlag::MONTH_GROUP, + /**MONTH*/ ObDFMFlag::MONTH_GROUP, + /**MON*/ ObDFMFlag::MONTH_GROUP, + /**AM*/ ObDFMFlag::MERIDIAN_INDICATOR_GROUP, + /**AM2*/ ObDFMFlag::MERIDIAN_INDICATOR_GROUP, + /**PM*/ ObDFMFlag::MERIDIAN_INDICATOR_GROUP, + /**PM2*/ ObDFMFlag::MERIDIAN_INDICATOR_GROUP, + /**Q*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**RR*/ ObDFMFlag::YEAR_GROUP, + /**RRRR*/ ObDFMFlag::YEAR_GROUP, + /**SS*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**SSSSS*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**WW*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**W*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**YGYYY*/ ObDFMFlag::YEAR_GROUP, + /**YEAR*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**SYEAR*/ ObDFMFlag::NEVER_APPEAR_GROUP, + /**YYYY*/ ObDFMFlag::YEAR_GROUP, + /**SYYYY*/ ObDFMFlag::YEAR_GROUP, + /**YYY*/ ObDFMFlag::YEAR_GROUP, + /**YY*/ ObDFMFlag::YEAR_GROUP, + /**Y*/ ObDFMFlag::YEAR_GROUP, + /**DS*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**DL*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**TZH*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**TZM*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**TZD*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**TZR*/ ObDFMFlag::RUNTIME_CONFLICT_SOLVE_GROUP, + /**X*/ ObDFMFlag::NON_CONFLICT_GROUP, + /**J*/ ObDFMFlag::DAY_OF_YEAR_GROUP, +}; + +const int ObDFMFlag::CONFLICT_GROUP_ERR[ObDFMFlag::MAX_CONFLICT_GROUP_NUMBER] = { + /*NEVER_APPEAR_GROUP*/ OB_ERR_FORMAT_CODE_CANNOT_APPEAR, // ORA-01820: format code cannot appear in date input + // format + /*YEAR_GROUP*/ OB_ERR_YEAR_MAY_ONLY_BE_SPECIFIED_ONCE, // ORA-01812: year may only be specified once + /*MERIDIAN_INDICATOR_GROUP*/ OB_ERR_AM_PM_CONFLICTS_WITH_USE_OF_AM_DOT_PM_DOT, // ORA-01810: format code appears + // twice + /*WEEK_OF_DAY_GROUP*/ OB_ERR_DAY_OF_WEEK_SPECIFIED_MORE_THAN_ONCE, // ORA-01817: day of week may only be + // specified once + /*ERA_GROUP*/ OB_ERR_BC_AD_CONFLICT_WITH_USE_OF_BC_DOT_AD_DOT, // ORA-01810: format code appears twice + /*HOUR_GROUP*/ OB_ERR_HOUR_MAY_ONLY_BE_SPECIFIED_ONCE, // ORA-01813: hour may only be specified once + /*MONTH_GROUP*/ OB_ERR_MONTH_MAY_ONLY_BE_SPECIFIED_ONCE, // ORA-01816: month may only be specified once + /*DAY_OF_YEAR_GROUP*/ OB_ERR_JULIAN_DATE_PRECLUDES_USE_OF_DAY_OF_YEAR // ORA-1811: Julian date precludes use of + // day of year +}; + +const int ObDFMFlag::EXPECTED_MATCHING_LENGTH[ObDFMFlag::MAX_FLAG_NUMBER] = { + /**AD*/ 2, + /**AD2*/ 4, + /**BC*/ 2, + /**BC2*/ 4, + /**CC,*/ 0, // never used + /**SCC*/ 0, // never used + /**D*/ 1, + /**DAY*/ 0, // non-numeric, ignored + /**DD*/ 2, + /**DDD*/ 3, + /**DY*/ 0, // non-numeric, ignored + /**FF1*/ 1, + /**FF2*/ 2, + /**FF3*/ 3, + /**FF4*/ 4, + /**FF5*/ 5, + /**FF6*/ 6, + /**FF7*/ 7, + /**FF8*/ 8, + /**FF9*/ 9, + /**FF*/ 9, + /**HH*/ 2, + /**HH24*/ 2, + /**HH12*/ 2, + /**IW*/ 0, // never used + /**I*/ 0, // never used + /**IY*/ 0, // never used + /**IYY*/ 0, // never used + /**IYYY*/ 0, // never used + /**MI*/ 2, + /**MM*/ 2, + /**MONTH*/ 0, // non-numeric, ignored + /**MON*/ 0, // non-numeric, ignored + /**AM*/ 2, + /**AM2*/ 4, + /**PM*/ 2, + /**PM2*/ 4, + /**Q*/ 0, // never used + /**RR*/ 0, // special case + /**RRRR*/ 0, // special case + /**SS*/ 2, + /**SSSSS*/ 5, + /**WW*/ 0, // never used + /**W*/ 0, // never used + /**YGYYY*/ 5, + /**YEAR*/ 0, // never used + /**SYEAR*/ 0, // never used + /**YYYY*/ 4, + /**SYYYY*/ 4, + /**YYY*/ 3, + /**YY*/ 2, + /**Y*/ 1, + /**DS*/ 0, // todo + /**DL*/ 0, // todo + /**TZH*/ 2, // todo + /**TZM*/ 2, // todo + /**TZD*/ 0, // non-numeric, ignored + /**TZR*/ 0, // non-numeric, ignored + /**X*/ 0, // non-numeric, ignored + /**J*/ 7, +}; + +int64_t ObDFMFlag::calc_max_len_of_patterns() +{ + int64_t result = 0; + for (int64_t flag = 0; flag < MAX_FLAG_NUMBER; ++flag) { + if (result < PATTERN[flag].len_) { + result = PATTERN[flag].len_; + } + } + return result; +} + +ObString ObDFMElem::get_elem_name() const +{ + ObString result; + if (ObDFMFlag::is_flag_valid(elem_flag_)) { + result = ObDFMFlag::PATTERN[elem_flag_].to_obstring(); + } else { + result = ObString("INVALID ELEMENT"); + } + return result; +} + +int ObDFMUtil::match_int_value_with_comma(ObDFMParseCtx &ctx, + const int64_t expected_len, + int64_t &value_len, + int32_t &result) +{ + int ret = OB_SUCCESS; + int32_t temp_value = 0; + int64_t real_data_len = 0; + int64_t digits_len = 0; + int64_t continuous_comma_count = 0; + bool stop_flag = false; + + if (OB_UNLIKELY(!ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } + while (OB_SUCC(ret) + && !stop_flag + && real_data_len < ctx.remain_len_ + && digits_len < expected_len) { // look digits by # of value_len + char cur_char = *(ctx.cur_ch_ + real_data_len); + if (',' == cur_char) { + continuous_comma_count++; + if (continuous_comma_count == 2) { + --real_data_len; + stop_flag = true; + } else { + ++real_data_len; + } + } else { + continuous_comma_count = 0; + if (ObDFMUtil::is_split_char(cur_char)) { + stop_flag = true; + } else { + if (OB_UNLIKELY(!isdigit(cur_char))) { + ret = OB_ERR_NON_NUMERIC_CHARACTER_VALUE; // ORA-01858: a non-numeric character was found where a numeric was + // expected + LOG_WARN("failed to match int value", K(ret)); + } else { + temp_value *= 10; + temp_value += cur_char - '0'; + ++real_data_len; + ++digits_len; + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(check_int_value_length(ctx, expected_len, real_data_len))) { + LOG_WARN("int value length is not equal to expected len", K(ret), K(real_data_len), K(expected_len), K(ctx)); + } else { + value_len = real_data_len; + result = temp_value; + } + } + return ret; +} + +const char* ObDFMUtil::find_first_separator(ObDFMParseCtx &ctx) +{ + const char* result = NULL; + for (int64_t i = 0; NULL == result && i < ctx.remain_len_; i++) { + if (is_split_char(ctx.cur_ch_[i])) { + result = ctx.cur_ch_ + i; + } + } + return result; +} + +int ObDFMUtil::match_chars_until_space(ObDFMParseCtx &ctx, ObString &result, int64_t &value_len) +{ + int ret = OB_SUCCESS; + int32_t str_len = 0; + if (OB_UNLIKELY(ctx.is_parse_finish())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } + while (OB_SUCC(ret) && str_len < ctx.remain_len_ && !isspace(ctx.cur_ch_[str_len])) { + if (OB_UNLIKELY(str_len >= value_len)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("size over flow", K(ret)); + } else { + ++str_len; + } + } + if (OB_SUCC(ret)) { + result.assign_ptr(ctx.cur_ch_, str_len); + value_len = str_len; + } + return ret; +} + +int ObDFMUtil::check_int_value_length(const ObDFMParseCtx &ctx, const int64_t expected_len, const int64_t real_data_len) +{ + int ret = OB_SUCCESS; + /* + * format need separate chars but input omit separate chars like: + * to_date('20181225', 'YYYY-MM-DD') input omit '-' + * in this situation, the numeric value are matched in fixed length mode. + * which means real_data_len should be equal to element expected length, or will return with an error + */ + if (OB_SUCC(ret) && ctx.is_matching_by_expected_len_) { // is true only in only in str_to_ob_time_oracle_dfm + bool legal = true; + if (ObDFMFlag::RR == ctx.expected_elem_flag_ || ObDFMFlag::RRRR == ctx.expected_elem_flag_) { // one special case + legal = (2 == real_data_len || 4 == real_data_len); + } else if (expected_len > 0) { // usual case, for numeric value + legal = (real_data_len == expected_len); + } + + if (OB_UNLIKELY(!legal)) { + ret = OB_INVALID_DATE_VALUE; + } + } + return ret; +} + +int ObDFMUtil::match_int_value(ObDFMParseCtx &ctx, + const int64_t expected_len, + int64_t &value_len, + int32_t &result, + int32_t value_sign /* = 1*/) +{ + // only unsigned int + int ret = OB_SUCCESS; + int32_t temp_value = 0; + int64_t real_data_len = 0; + + if (OB_UNLIKELY(!ctx.is_valid()) + || OB_UNLIKELY(expected_len < 0) + || OB_UNLIKELY(value_sign != -1 && value_sign != 1)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(ctx), K(expected_len), K(value_sign)); + } else if (!isdigit(ctx.cur_ch_[0])) { // check the first char + ret = OB_ERR_NON_NUMERIC_CHARACTER_VALUE; // ORA-01858: a non-numeric character was found where a numeric was + // expected + } + + int64_t date_max_len = std::min(ctx.remain_len_, expected_len); + + while (OB_SUCC(ret) && real_data_len < date_max_len && isdigit(ctx.cur_ch_[real_data_len])) { + int32_t cur_digit = static_cast(ctx.cur_ch_[real_data_len] - '0'); + + if (temp_value * 10LL > INT32_MAX - cur_digit) { + ret = OB_OPERATE_OVERFLOW; + LOG_WARN("datetime part value is out of range", K(ret)); + } else { + temp_value = temp_value * 10 + cur_digit; + ++real_data_len; + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(check_int_value_length(ctx, expected_len, real_data_len))) { + LOG_WARN("int value length is not equal to expected len", K(ret), K(real_data_len), K(expected_len), K(ctx)); + } else { + value_len = real_data_len; + result = temp_value * value_sign; + } + } + + return ret; +} + +int ObDFMUtil::parse_datetime_format_string(const ObString &fmt_str, ObDFMElemArr &elements) +{ + int ret = OB_SUCCESS; + if (fmt_str.empty()) { + // do nothing + } else { + ObDFMParseCtx parse_ctx(fmt_str.ptr(), fmt_str.length()); + int64_t skipped_len = 0; + + while (OB_SUCC(ret) && parse_ctx.remain_len_ > 0) { + // skip separate chars + skipped_len = skip_separate_chars(parse_ctx); + // parse one element from head + if (OB_SUCC(ret) && parse_ctx.remain_len_ > 0) { + ObDFMElem value_elem; + + if (parse_ctx.get_parsed_len() > 0) { + value_elem.is_single_dot_before_ = (skipped_len == 1 && '.' == parse_ctx.cur_ch_[-1]); + } + + if (OB_FAIL(parse_one_elem(parse_ctx, value_elem))) { + LOG_WARN("failed to parse one element", K(ret)); + } else if (OB_FAIL(elements.push_back(value_elem))) { + LOG_WARN("failed to push back elem", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + LOG_DEBUG("dmf parse format string result", K(fmt_str), K(elements)); + } + } + return ret; +} + +/* search matched pattern */ +int ObDFMUtil::parse_one_elem(ObDFMParseCtx &ctx, ObDFMElem &elem) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ctx.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + int64_t winner_flag = ObDFMFlag::INVALID_FLAG; + int64_t max_matched_len = 0; + for (int64_t flag = 0; flag < ObDFMFlag::MAX_FLAG_NUMBER; ++flag) { + const ObTimeConstStr& pattern = ObDFMFlag::PATTERN[flag]; + if (max_matched_len < pattern.len_ && ObDFMUtil::match_pattern_ignore_case(ctx, pattern)) { + winner_flag = flag; + max_matched_len = pattern.len_; + } + } + + // uppercase adjust + if (OB_SUCC(ret)) { + if (OB_LIKELY(winner_flag != ObDFMFlag::INVALID_FLAG)) { + elem.elem_flag_ = winner_flag; + elem.offset_ = ctx.cur_ch_ - ctx.fmt_str_; + switch (winner_flag) { + case ObDFMFlag::MON: + case ObDFMFlag::MONTH: + case ObDFMFlag::DAY: + case ObDFMFlag::DY: + case ObDFMFlag::AM: + case ObDFMFlag::PM: + case ObDFMFlag::AD: + case ObDFMFlag::BC: { + if (OB_UNLIKELY(ctx.remain_len_ < 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("winner flag length must bigger than 2", K(ret), K(elem)); + } else if (isupper(ctx.cur_ch_[0]) && isupper(ctx.cur_ch_[1])) { + elem.upper_case_mode_ = ObDFMElem::ALL_CHARACTER; + } else if (isupper(ctx.cur_ch_[0])) { + elem.upper_case_mode_ = ObDFMElem::ONLY_FIRST_CHARACTER; + } else { + elem.upper_case_mode_ = ObDFMElem::NON_CHARACTER; + } + break; + } + + case ObDFMFlag::AM2: + case ObDFMFlag::PM2: + case ObDFMFlag::AD2: + case ObDFMFlag::BC2: { + if (OB_UNLIKELY(ctx.remain_len_ < 4)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("winner flag length must bigger than 4", K(ret), K(elem)); + } else if (isupper(ctx.cur_ch_[0])) { + elem.upper_case_mode_ = ObDFMElem::ALL_CHARACTER; + } else { + elem.upper_case_mode_ = ObDFMElem::NON_CHARACTER; + } + } + default: + // do nothing + break; + } + ctx.update(max_matched_len); + } else { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("date format is invalid", K(ret), K(ctx.remain_len_)); + } + } + } + + return ret; +} + +int ObDFMUtil::special_mode_sprintf(char *buf, + const int64_t buf_len, + int64_t &pos, + const ObTimeConstStr &str, + const ObDFMElem::UpperCaseMode mode, + int64_t padding) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(str.len_ <= 0) + || OB_ISNULL(str.ptr_) + || OB_ISNULL(buf) + || OB_UNLIKELY(padding > 0 && padding < str.len_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(str.len_), KP(str.ptr_), KP(buf)); + } else if (OB_UNLIKELY(pos + (padding > 0 ? padding : str.len_) >= buf_len)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("buf is overflow", K(ret), K(buf_len), K(str.len_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < str.len_; ++i) { + char cur_char = str.ptr_[i]; + if ((cur_char >= 'a' && cur_char <= 'z') || (cur_char >= 'A' && cur_char <= 'Z')) { + switch (mode) { + case ObDFMElem::ALL_CHARACTER: { + buf[pos++] = static_cast(toupper(cur_char)); + break; + } + case ObDFMElem::ONLY_FIRST_CHARACTER: { + if (i == 0) { + buf[pos++] = static_cast(toupper(cur_char)); + } else { + buf[pos++] = static_cast(tolower(cur_char)); + } + break; + } + case ObDFMElem::NON_CHARACTER: { + buf[pos++] = static_cast(tolower(cur_char)); + break; + } + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unknown dfm elem mode", K(ret), K(mode)); + break; + } + } + } else { + buf[pos++] = cur_char; + } // end if + } // end for + for (int64_t i = str.len_; i < padding; ++i) { + buf[pos++] = ' '; + } + } + + return ret; +} + +int ObDFMUtil::check_semantic(const ObDFMElemArr &elements, + ObBitSet &flag_bitmap, + uint64_t mode) +{ + int ret = OB_SUCCESS; + flag_bitmap.reset(); + int64_t conflict_group_bitset = 0; + STATIC_ASSERT(ObDFMFlag::MAX_CONFLICT_GROUP_NUMBER < 20, + "bitset will overflow, because conflict_group_bitset is type of int64_t"); + for (int64_t i = 0; OB_SUCC(ret) && i < elements.count(); ++i) { + int64_t flag = elements.at(i).elem_flag_; + if (OB_UNLIKELY(!ObDFMFlag::is_flag_valid(flag))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid array value", K(ret), K(flag)); + } + // The following datetime format elements can be used in timestamp and interval format models, + // but not in the original DATE format model: FF, TZD, TZH, TZM, and TZR + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(flag >= ObDFMFlag::FF1 && flag <= ObDFMFlag::FF && !HAS_TYPE_ORACLE(mode))) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("oracle date type can not have fractional seconds", K(ret), K(mode), K(ObDFMFlag::PATTERN[flag])); + } else if (OB_UNLIKELY(!HAS_TYPE_TIMEZONE(mode) + && (ObDFMFlag::TZD == flag + || ObDFMFlag::TZR == flag + || ObDFMFlag::TZH == flag + || ObDFMFlag::TZM == flag))) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("oracle timestamp or timestamp with local timezone can not has timezone", + K(ret), + K(mode), + K(ObDFMFlag::PATTERN[flag])); + } + } + + // check no duplicate elem first + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(flag_bitmap.has_member(flag))) { + ret = OB_ERR_FORMAT_CODE_APPEARS_TWICE; // ORA-01810: format code appears twice + LOG_WARN("datetime format model check failed", K(ret), "flag", ObString(ObDFMFlag::PATTERN[flag].ptr_)); + } else if (OB_FAIL(flag_bitmap.add_member(flag))) { + LOG_WARN("failed to add bitmap", K(ret), "flag", ObString(ObDFMFlag::PATTERN[flag].ptr_)); + } + } + // check conflict in group which the element belongs to + if (OB_SUCC(ret)) { + int64_t conf_group = ObDFMFlag::CONFLICT_GROUP_MAP[flag]; + if (ObDFMFlag::need_check_conflict(conf_group)) { + if (OB_UNLIKELY(0 != (conflict_group_bitset & (1 << conf_group)))) { + ret = ObDFMFlag::CONFLICT_GROUP_ERR[conf_group]; + LOG_WARN("invalid element in group conflict check", + K(ret), + "elem offset", + i, + "elem name", + ObDFMFlag::PATTERN[flag].ptr_, + "conflict_group_id", + conf_group); + } else { + conflict_group_bitset |= (1 << conf_group); + } + } + } // end if + } // end for + + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(flag_bitmap.has_member(ObDFMFlag::TZM) && !flag_bitmap.has_member(ObDFMFlag::TZH))) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("given TZM without TZH is forbidden", K(ret)); + } + } + + if (OB_SUCC(ret)) { + LOG_DEBUG("conflict summary", K(conflict_group_bitset)); + } + return ret; +} + +int ObDFMUtil::check_ctx_valid(ObDFMParseCtx &ctx, int err_code) +{ + return ctx.is_valid() ? OB_SUCCESS : err_code; +} + +int ObDFMUtil::match_char(ObDFMParseCtx &ctx, const char c, const int err_code) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_ctx_valid(ctx, err_code))) { + LOG_WARN("parsing finished", K(ret)); + } else if (OB_UNLIKELY(c != ctx.cur_ch_[0])) { + ret = err_code; + } else { + ctx.update(1); + } + return ret; +} + + +} // end of common +} // end of oceanbase + diff --git a/src/lib/timezone/ob_oracle_format_models.h b/src/lib/timezone/ob_oracle_format_models.h new file mode 100644 index 0000000000000000000000000000000000000000..50b4993f46c41e2f996894f7fab62ab775c1fe8e --- /dev/null +++ b/src/lib/timezone/ob_oracle_format_models.h @@ -0,0 +1,390 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ORACLE_FORMAT_MODELS_H_ +#define OCEANBASE_ORACLE_FORMAT_MODELS_H_ + +#include "lib/ob_define.h" +#include "lib/container/ob_iarray.h" +#include "lib/container/ob_bit_set.h" +#include "lib/timezone/ob_time_convert.h" + +// Note: DFM is abbr of datetime format models +// see oracle doc Format Models: https://docs.oracle.com/cd/B19306_01/server.102/b14200/sql_elements004.htm#i34924 + +namespace oceanbase { +namespace common { +struct ObTimeConstStr; + +struct ObOracleTimeLimiter { + ObOracleTimeLimiter(int32_t min_val, int32_t max_val, int err_code) + : min_val_(min_val), + max_val_(max_val), + err_code_(err_code) + {} + + int32_t min_val_; + int32_t max_val_; + int err_code_; + + inline int validate(int32_t value) const + { + int ret = OB_SUCCESS; + if (OB_UNLIKELY(value < min_val_ || value > max_val_)) { + ret = err_code_; + } + return ret; + } +}; + +struct ObDFMLimit { + static const ObOracleTimeLimiter YEAR; + static const ObOracleTimeLimiter MONTH; + static const ObOracleTimeLimiter MONTH_DAY; + static const ObOracleTimeLimiter WEEK_DAY; + static const ObOracleTimeLimiter YEAR_DAY; + static const ObOracleTimeLimiter HOUR12; + static const ObOracleTimeLimiter HOUR24; + static const ObOracleTimeLimiter MINUTE; + static const ObOracleTimeLimiter SECOND; + static const ObOracleTimeLimiter SECS_PAST_MIDNIGHT; + static const ObOracleTimeLimiter TIMEZONE_HOUR_ABS; + static const ObOracleTimeLimiter TIMEZONE_MIN_ABS; + static const ObOracleTimeLimiter JULIAN_DATE; +}; + +class ObDFMFlag { +public: + // ElementFlag are defined according to oracle doc + // see Format Models: https://docs.oracle.com/cd/B19306_01/server.102/b14200/sql_elements004.htm#i34924 + // Note: FF1-FF9 and FF should be together + enum ElementFlag { + INVALID_FLAG = -1, + AD = 0, + AD2, // A.D. + BC, + BC2, // B.C. + CC, + SCC, + D, + DAY, + DD, + DDD, + DY, + FF1, + FF2, + FF3, + FF4, + FF5, + FF6, + FF7, + FF8, + FF9, + FF, + HH, + HH24, + HH12, + IW, + I, + IY, + IYY, + IYYY, + MI, + MM, + MONTH, + MON, + AM, + AM2, // A.M. + PM, + PM2, // P.M. + Q, + RR, + RRRR, + SS, + SSSSS, + WW, + W, + YGYYY, + YEAR, + SYEAR, + YYYY, + SYYYY, + YYY, + YY, + Y, + DS, + DL, + TZH, + TZM, + TZD, + TZR, + X, + J, + ///<<< !!!add any flag before this line!!! + // Please also add in ELEMENTFLAG_MAX_LEN[ObDFMFlag::MAX_FLAG_NUMBER] + MAX_FLAG_NUMBER + }; + + // ElementGroup to handle conflict. Each group should only contain one element. + enum ElementGroup { + RUNTIME_CONFLICT_SOLVE_GROUP = -2, + NON_CONFLICT_GROUP = -1, + ///<<< conflict in group, before this line, will be ignored + NEVER_APPEAR_GROUP = 0, // the element should never appear + YEAR_GROUP, // include : SYYYY YYYY YYY YY Y YGYYY RR RRRR + MERIDIAN_INDICATOR_GROUP, // include : AM PM + WEEK_OF_DAY_GROUP, // include : D Day Dy + ERA_GROUP, // include : AD BC + HOUR_GROUP, // include : HH HH12 HH24 + MONTH_GROUP, // include : MONTH MON MM + DAY_OF_YEAR_GROUP, // include : DDD, J + ///<<< !!!add any flag before this line!!! + MAX_CONFLICT_GROUP_NUMBER + }; + + // For matching format string, patterns of each flag are defined + static const ObTimeConstStr PATTERN[MAX_FLAG_NUMBER]; + // conflict group of the elements. + static const int CONFLICT_GROUP_MAP[MAX_FLAG_NUMBER]; + // the user error code returned, if conflict happend,. + + static const int CONFLICT_GROUP_ERR[MAX_CONFLICT_GROUP_NUMBER]; + + // max length for matching element + static const int EXPECTED_MATCHING_LENGTH[MAX_FLAG_NUMBER]; + + + static inline bool is_flag_valid(int64_t flag) + { + return (flag > INVALID_FLAG && flag < MAX_FLAG_NUMBER); + } + static inline bool need_check_conflict(int64_t elem_group) + { + return elem_group >= 0; + } + static inline bool need_check_expected_length(ElementFlag flag) + { + return is_flag_valid(flag) && (EXPECTED_MATCHING_LENGTH[flag] > 0); + } + +private: + static int64_t calc_max_len_of_patterns(); +}; + +struct ObDFMParseCtx { + explicit ObDFMParseCtx(const char *fmt_str, const int64_t fmt_len) + : fmt_str_(fmt_str), + cur_ch_(fmt_str), + remain_len_(fmt_len), + expected_elem_flag_(ObDFMFlag::INVALID_FLAG), + is_matching_by_expected_len_(false) + {} + inline void update(const int64_t succ_len) + { + cur_ch_ += succ_len; + remain_len_ -= succ_len; + } + inline bool is_valid() + { + return cur_ch_ != NULL && remain_len_ > 0; + } + inline int64_t get_parsed_len() + { + return static_cast(cur_ch_ - fmt_str_); + } + inline bool is_parse_finish() + { + return 0 == remain_len_; + } + inline void revert(const int64_t rev_len) + { + cur_ch_ -= rev_len; + remain_len_ += rev_len; + } + + void set_next_expected_elem(int64_t elem_flag, bool is_matching_by_expected_len) + { + expected_elem_flag_ = elem_flag; + is_matching_by_expected_len_ = is_matching_by_expected_len; + } + + const char *const fmt_str_; + const char *cur_ch_; + int64_t remain_len_; + + // the following values are only used in function str_to_ob_time_oracle_dfm + int64_t expected_elem_flag_; + bool is_matching_by_expected_len_; // only used for match_int_value + + TO_STRING_KV("parsed len", static_cast(cur_ch_ - fmt_str_), "remain chars", ObString(remain_len_, cur_ch_), + K_(expected_elem_flag), K_(is_matching_by_expected_len)); +}; + +struct ObDFMElem { + + enum UpperCaseMode { NON_CHARACTER, ONLY_FIRST_CHARACTER, ALL_CHARACTER }; + + ObDFMElem() + : elem_flag_(ObDFMFlag::INVALID_FLAG), + offset_(OB_INVALID_INDEX_INT64), + is_single_dot_before_(false), + upper_case_mode_(NON_CHARACTER) + {} + int64_t elem_flag_; // flag from enum ObDFMFlag + int64_t offset_; // offset in origin format string + bool is_single_dot_before_; // for the dot before FF + UpperCaseMode upper_case_mode_; + ObString get_elem_name() const; + TO_STRING_KV("elem_flag", get_elem_name(), K_(offset), K_(is_single_dot_before), K_(upper_case_mode)); + + bool inline is_valid() + { + return ObDFMFlag::is_flag_valid(elem_flag_) && offset_ >= 0; + } +}; + +typedef ObIArray ObDFMElemArr; + +class ObDFMUtil { +public: + static const int64_t UNKNOWN_LENGTH_OF_ELEMENT = 20; + static const int64_t COMMON_ELEMENT_NUMBER = 10; + static int parse_datetime_format_string(const ObString &fmt_str, ObDFMElemArr &elements); + static int check_semantic(const ObDFMElemArr &elements, + ObBitSet &flag_bitmap, + uint64_t mode); + static int parse_one_elem(ObDFMParseCtx &ctx, ObDFMElem &elem); + static inline int64_t skip_separate_chars(ObDFMParseCtx &ctx, + const int64_t limit = OB_MAX_VARCHAR_LENGTH, + const int64_t stop_char = INT64_MAX); + static inline int64_t skip_blank_chars(ObDFMParseCtx &ctx); + static inline bool is_element_can_omit(const ObDFMElem &elem); + // Explain padding: day or month name is padded with blanks to display in the same wide, please see oracle doc + static int special_mode_sprintf(char *buf, + const int64_t buf_len, + int64_t &pos, + const ObTimeConstStr &str, + const ObDFMElem::UpperCaseMode mode, + int64_t padding = -1); + static int match_chars_until_space(ObDFMParseCtx &ctx, ObString &result, int64_t &value_len); + static const char* find_first_separator(ObDFMParseCtx &ctx); + static int match_int_value(ObDFMParseCtx &ctx, + const int64_t expected_len, + int64_t &value_len, + int32_t &result, + int32_t value_sign = 1); + static int match_int_value_with_comma(ObDFMParseCtx &ctx, + const int64_t expected_len, + int64_t &value_len, + int32_t &result); + static int check_int_value_length(const ObDFMParseCtx &ctx, const int64_t expected_len, const int64_t real_data_len); + static int check_ctx_valid(ObDFMParseCtx &ctx, int err_code); + static int match_char(ObDFMParseCtx &ctx, const char c, const int err_code); + + static inline bool match_pattern_ignore_case(ObDFMParseCtx &ctx, const ObTimeConstStr &pattern) + { + bool ret_bool = false; + if (ctx.remain_len_ >= pattern.len_) { + ret_bool = (0 == strncasecmp(ctx.cur_ch_, pattern.ptr_, pattern.len_)); + } else { + // false + } + return ret_bool; + } + static inline bool elem_has_meridian_indicator(ObBitSet &flag_bitmap) + { + return flag_bitmap.has_member(ObDFMFlag::AM) + || flag_bitmap.has_member(ObDFMFlag::PM) + || flag_bitmap.has_member(ObDFMFlag::AM2) + || flag_bitmap.has_member(ObDFMFlag::PM2); + } + static bool is_split_char(const char ch); + static inline bool is_sign_char(const char ch) + { + return '-' == ch || '+' == ch; + } + +private: + static inline bool is_uppercase_char(const char ch) + { + return (0 == (ch & (1 << 5))); + } +}; + +int64_t ObDFMUtil::skip_blank_chars(ObDFMParseCtx &ctx) +{ + int64_t blank_char_len = 0; + while (blank_char_len < ctx.remain_len_ && ' ' == ctx.cur_ch_[blank_char_len]) { + blank_char_len++; + } + ctx.update(blank_char_len); + return blank_char_len; +} + +int64_t ObDFMUtil::skip_separate_chars(ObDFMParseCtx &ctx, + const int64_t limit /*= OB_MAX_VARCHAR_LENGTH*/, + const int64_t stop_char /*= INT64_MAX*/) +{ + int64_t sep_len = 0; + while (sep_len < ctx.remain_len_ + && sep_len < limit + && is_split_char(ctx.cur_ch_[sep_len]) + && static_cast(ctx.cur_ch_[sep_len]) != stop_char) { + sep_len++; + } + ctx.update(sep_len); + return sep_len; +} + +inline bool ObDFMUtil::is_split_char(const char ch) +{ + int ret_bool = false; + if (ch == '\n' + || ch == '\t' + || ((ch >= 0x20 && ch <= 0x7E) + && !((ch >= '0' && ch <= '9') + || (ch >= 'a' && ch <= 'z') + || (ch >= 'A' && ch <= 'Z')))) { + ret_bool = true; + } + return ret_bool; +} +/* + * if format elements contains TZR + * hour minuts seconds and fracial second can not omit + * Because, I guess, the daylight-saving time may be uncertain + * if the time part is omitted. + * The day + */ +inline bool ObDFMUtil::is_element_can_omit(const ObDFMElem &elem) +{ + int ret_bool = true; + int64_t flag = elem.elem_flag_; + int conf_group = ObDFMFlag::CONFLICT_GROUP_MAP[flag]; + if (ObDFMFlag::YEAR_GROUP == conf_group + || ObDFMFlag::WEEK_OF_DAY_GROUP == conf_group + || ObDFMFlag::MONTH_GROUP == conf_group + || ObDFMFlag::DD == flag + || ObDFMFlag::DS == flag + || ObDFMFlag::DL == flag) { + ret_bool = false; + } else { + // return true + } + return ret_bool; +} + + +} // namespace common +} // namespace oceanbase + +#endif // OCEANBASE_ORACLE_FORMAT_MODELS_H_ diff --git a/src/lib/timezone/ob_time_convert.cpp b/src/lib/timezone/ob_time_convert.cpp index 261462f1e053ddcf3054c2d6714bcfaf7550f87b..8e92be2ade64977437170a1903c4dead730318d0 100644 --- a/src/lib/timezone/ob_time_convert.cpp +++ b/src/lib/timezone/ob_time_convert.cpp @@ -18,7 +18,9 @@ #include #include "lib/ob_define.h" #include "lib/oblog/ob_log.h" -//#include "lib/timezone/ob_timezone_util.h" +#include "lib/utility/ob_template_utils.h" +#include "lib/timezone/ob_oracle_format_models.h" +#include "lib/ob_proxy_worker.h" namespace oceanbase { @@ -38,6 +40,10 @@ const int32_t DT_PART_MIN[DATETIME_PART_CNT] = { 0, 1, 1, 0, 0, 0, 0}; const int32_t DT_PART_MAX[DATETIME_PART_CNT] = {9999, 12, 31, 23, 59, 59, 1000000}; // 1000000 for usecond, because sometimes we round .9999999 to .1000000 +const int64_t TZ_PART_BASE[DATETIME_PART_CNT] = {100, 12, -1, 24, 60, 60, 1000000000}; +const int64_t TZ_PART_MIN[DATETIME_PART_CNT] = {1, 1, 1, 0, 0, 0, 0}; +const int64_t TZ_PART_MAX[DATETIME_PART_CNT] = {9999, 12, 31, 23, 59, 59, 1000000000}; + static const int8_t DAYS_PER_MON[2][12 + 1] = { {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, {0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} @@ -165,42 +171,106 @@ static const ObIntervalIndex INTERVAL_INDEX[DATE_UNIT_MAX] = { {DT_YEAR, DT_MON, 2, false}, // DATE_UNIT_YEAR_MONTH. }; -static const ObTimeConverter::ObTimeConstStr MDAY_NAMES[31 + 1] = { - {"null", 4}, - {"1st", 3}, {"2nd", 3}, {"3rd", 3}, {"4th", 3}, {"5th", 3}, {"6th", 3}, {"7th", 3}, - {"8th", 3}, {"9th", 3}, {"10th", 4}, {"11th", 4}, {"12th", 4}, {"13th", 4}, {"14th", 4}, - {"15th", 4}, {"16th", 4}, {"17th", 4}, {"18th", 4}, {"19th", 4}, {"20th", 4}, {"21st", 4}, - {"22nd", 4}, {"23rd", 4}, {"24th", 4}, {"25th", 4}, {"26th", 4}, {"27th", 4}, {"28th", 4}, - {"29th", 4}, {"30th", 4}, {"31st", 4} +static const ObTimeConstStr MDAY_NAMES[31 + 1] = { + ObTimeConstStr("null", 4), + ObTimeConstStr("1st", 3), + ObTimeConstStr("2nd", 3), + ObTimeConstStr("3rd", 3), + ObTimeConstStr("4th", 3), + ObTimeConstStr("5th", 3), + ObTimeConstStr("6th", 3), + ObTimeConstStr("7th", 3), + ObTimeConstStr("8th", 3), + ObTimeConstStr("9th", 3), + ObTimeConstStr("10th", 4), + ObTimeConstStr("11th", 4), + ObTimeConstStr("12th", 4), + ObTimeConstStr("13th", 4), + ObTimeConstStr("14th", 4), + ObTimeConstStr("15th", 4), + ObTimeConstStr("16th", 4), + ObTimeConstStr("17th", 4), + ObTimeConstStr("18th", 4), + ObTimeConstStr("19th", 4), + ObTimeConstStr("20th", 4), + ObTimeConstStr("21st", 4), + ObTimeConstStr("22nd", 4), + ObTimeConstStr("23rd", 4), + ObTimeConstStr("24th", 4), + ObTimeConstStr("25th", 4), + ObTimeConstStr("26th", 4), + ObTimeConstStr("27th", 4), + ObTimeConstStr("28th", 4), + ObTimeConstStr("29th", 4), + ObTimeConstStr("30th", 4), + ObTimeConstStr("31st", 4) }; static const char *DAY_NAME[31 + 1] = { - "", "1st", "2nd", "3rd", "4th", "5th", "6th", "7th", "8th", "9th", "10th", + "0th", "1st", "2nd", "3rd", "4th", "5th", "6th", "7th", "8th", "9th", "10th", "11th", "12th", "13th", "14th", "15th", "16th", "17th", "18th", "19th", "20th", "21st", "22nd", "23rd", "24th", "25th", "26th", "27th", "28th", "29th", "30th", "31st" }; -static const ObTimeConverter::ObTimeConstStr WDAY_NAMES[DAYS_PER_WEEK + 1] = { - {"null", 4}, - {"Monday", 6}, {"Tuesday", 7}, {"Wednesday", 9}, {"Thursday", 8}, {"Friday", 6}, {"Saturday", 8}, {"Sunday", 6} +static const ObTimeConstStr WDAY_NAMES[DAYS_PER_WEEK + 1] = { + ObTimeConstStr("null", 4), + ObTimeConstStr("Monday", 6), + ObTimeConstStr("Tuesday", 7), + ObTimeConstStr("Wednesday", 9), + ObTimeConstStr("Thursday", 8), + ObTimeConstStr("Friday", 6), + ObTimeConstStr("Saturday", 8), + ObTimeConstStr("Sunday", 6) }; -static const ObTimeConverter::ObTimeConstStr WDAY_ABBR_NAMES[DAYS_PER_WEEK + 1] = { - {"null", 4}, - {"Mon", 3}, {"Tue", 3}, {"Wed", 3}, {"Thu", 3}, {"Fri", 3}, {"Sat", 3}, {"Sun", 3} +static const int32_t MAX_WDAY_NAME_LENGTH = ObTimeConverter::calc_max_name_length(WDAY_NAMES, DAYS_PER_WEEK); + + +static const ObTimeConstStr WDAY_ABBR_NAMES[DAYS_PER_WEEK + 1] = { + ObTimeConstStr("null", 4), + ObTimeConstStr("Mon", 3), + ObTimeConstStr("Tue", 3), + ObTimeConstStr("Wed", 3), + ObTimeConstStr("Thu", 3), + ObTimeConstStr("Fri", 3), + ObTimeConstStr("Sat", 3), + ObTimeConstStr("Sun", 3) }; -static const ObTimeConverter::ObTimeConstStr MON_NAMES[12 + 1] = { - {"null", 4}, - {"January", 7}, {"February", 8}, {"March", 5}, {"April", 5}, {"May", 3}, {"June", 4}, - {"July", 4}, {"August", 6}, {"September", 9}, {"October", 7}, {"November", 8}, {"December", 8} +static const ObTimeConstStr MON_NAMES[12 + 1] = { + ObTimeConstStr("null", 4), + ObTimeConstStr("January", 7), + ObTimeConstStr("February", 8), + ObTimeConstStr("March", 5), + ObTimeConstStr("April", 5), + ObTimeConstStr("May", 3), + ObTimeConstStr("June", 4), + ObTimeConstStr("July", 4), + ObTimeConstStr("August", 6), + ObTimeConstStr("September", 9), + ObTimeConstStr("October", 7), + ObTimeConstStr("November", 8), + ObTimeConstStr("December", 8) }; -static const ObTimeConverter::ObTimeConstStr MON_ABBR_NAMES[12 + 1] = { - {"null", 4}, - {"Jan", 3}, {"Feb", 3}, {"Mar", 3}, {"Apr", 3}, {"May", 3}, {"Jun", 3}, - {"Jul", 3}, {"Aug", 3}, {"Sep", 3}, {"Oct", 3}, {"Nov", 3}, {"Dec", 3} +static const int32_t MAX_MON_NAME_LENGTH = ObTimeConverter::calc_max_name_length(MON_NAMES, 12); +static const int32_t MAX_MON_NAME_LENGTH_ORACLE = 36; + +static const ObTimeConstStr MON_ABBR_NAMES[12 + 1] = { + ObTimeConstStr("null", 4), + ObTimeConstStr("Jan", 3), + ObTimeConstStr("Feb", 3), + ObTimeConstStr("Mar", 3), + ObTimeConstStr("Apr", 3), + ObTimeConstStr("May", 3), + ObTimeConstStr("Jun", 3), + ObTimeConstStr("Jul", 3), + ObTimeConstStr("Aug", 3), + ObTimeConstStr("Sep", 3), + ObTimeConstStr("Oct", 3), + ObTimeConstStr("Nov", 3), + ObTimeConstStr("Dec", 3) }; #define START_WITH_SUNDAY 0x04 @@ -209,7 +279,53 @@ static const ObTimeConverter::ObTimeConstStr MON_ABBR_NAMES[12 + 1] = { const int64_t SECS_PER_HOUR = (SECS_PER_MIN * MINS_PER_HOUR); const int64_t SECS_PER_DAY = (SECS_PER_HOUR * HOURS_PER_DAY); -const int64_t USECS_PER_DAY = (static_cast(USECS_PER_SEC) * SECS_PER_DAY); +const int64_t USECS_PER_DAY = (USECS_PER_SEC * SECS_PER_DAY); +const int64_t NSECS_PER_DAY = (NSECS_PER_SEC * SECS_PER_DAY); +const int64_t USECS_PER_MIN = (USECS_PER_SEC * SECS_PER_MIN); + +const ObString ObTimeConverter::DEFAULT_NLS_DATE_FORMAT("DD-MON-RR"); +const ObString ObTimeConverter::DEFAULT_NLS_TIMESTAMP_FORMAT("DD-MON-RR HH.MI.SSXFF AM"); +const ObString ObTimeConverter::DEFAULT_NLS_TIMESTAMP_TZ_FORMAT("DD-MON-RR HH.MI.SSXFF AM TZR"); +const ObString ObTimeConverter::COMPAT_OLD_NLS_DATE_FORMAT("YYYY-MM-DD HH24:MI:SS"); +const ObString ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_FORMAT("YYYY-MM-DD HH24:MI:SS.FF"); +const ObString ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_TZ_FORMAT("YYYY-MM-DD HH24:MI:SS.FF TZR TZD"); + + +int ObTime::set_tz_name(const ObString &tz_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(tz_name.empty()) || OB_UNLIKELY(tz_name.length() >= OB_MAX_TZ_NAME_LEN)) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("invalid tz_name", "length", tz_name.length(), "expect_len", OB_MAX_TZ_NAME_LEN, K(ret)); + } else { + MEMCPY(tz_name_, tz_name.ptr(), tz_name.length()); + tz_name_[tz_name.length()] = '\0'; + is_tz_name_valid_ = true; + } + + if (OB_FAIL(ret)) { + tz_name_[0] = '\0'; + is_tz_name_valid_ = false; + } + return ret; +} + +int ObTime::set_tzd_abbr(const ObString &tzd_abbr) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(tzd_abbr.empty()) || OB_UNLIKELY(tzd_abbr.length() >= OB_MAX_TZ_ABBR_LEN)) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("invalid tz_name", "length", tzd_abbr.length(), "expect_len", OB_MAX_TZ_ABBR_LEN, K(ret)); + } else { + MEMCPY(tzd_abbr_, tzd_abbr.ptr(), tzd_abbr.length()); + tzd_abbr_[tzd_abbr.length()] = '\0'; + } + + if (OB_FAIL(ret)) { + tzd_abbr_[0] = '\0'; + } + return ret; +} //////////////////////////////// // int / double / string -> datetime / date / time / year. @@ -371,9 +487,60 @@ int ObTimeConverter::str_to_interval(const ObString &str, ObDateUnitType unit_ty return ret; } -//////////////////////////////// -// int / double / uint / string <- datetime / date / time / year. +/** + * @brief cast str to oracle timestamp(3 possible types) + * @param in: str input string + * @param in: cvrt_ctx + * @param in: target_type target types, includes ObTimestampTZType ObTimestampNanoType ObTimestampLTZType + * @param out: value result ObOTimestampData + * @param out: scale scale of fractional seconds + * @return + */ +int ObTimeConverter::str_to_otimestamp(const ObString &str, + const ObTimeConvertCtx &cvrt_ctx, + const ObObjType target_type, + ObOTimestampData &value, + ObScale &scale) +{ + int ret = OB_SUCCESS; + value.reset(); + // NOTE::current format is fixed like "2012-12-02 12:00:00.123456". + // it is not enough for oracle when format variables supported. + // UPDATE: complex format has supported. + if (OB_UNLIKELY(!ob_is_otimestamp_type(target_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("it is not otimestamp type", K(target_type), K(ret)); + } else if (str.empty()) { + value.set_null_value(); + scale = OB_MAX_TIMESTAMP_TZ_PRECISION; + LOG_DEBUG("succ to convert null str to otimestamp", K(target_type), K(str), K(value), K(scale), K(lbt())); + } else { + ObTime ob_time; + if (OB_FAIL(str_to_ob_time_oracle_dfm(str, cvrt_ctx, target_type, ob_time, scale))) { + LOG_WARN("failed to convert to ob_time by dfm", "format_str", cvrt_ctx.oracle_nls_format_, K(ret)); + } else if (OB_FAIL(ob_time_to_utc(target_type, cvrt_ctx, ob_time))) { + LOG_WARN("failed to convert ob_time to utc", K(ret)); + } else if (OB_FAIL(ob_time_to_otimestamp(ob_time, value))) { + LOG_WARN("failed to convert obtime to timestamp_tz", K(ret)); + } else { + LOG_DEBUG("succ to convert str to otimestamp", + K(target_type), + K(str), + K(ob_time), + K(value), + K(scale), + "format_str", + cvrt_ctx.oracle_nls_format_, + K(lbt())); + } + } + + return ret; +} + +//////////////////////////////// +// int / double / uint / string <- datetime / date / time / year int ObTimeConverter::datetime_to_int(int64_t value, const ObTimeZoneInfo *tz_info, int64_t &int64) { int ret = OB_SUCCESS; @@ -557,6 +724,223 @@ int ObTimeConverter::timestamp_to_datetime(int64_t ts_value, const ObTimeZoneInf return ret; } +// date(local) --> timestamp tz(utc, offset) +// timestamp ltz(utc) +// timestamp (local) +int ObTimeConverter::odate_to_otimestamp(int64_t in_value_us, + const ObTimeZoneInfo *tz_info, + const ObObjType out_type, + ObOTimestampData &out_value) +{ + int ret = OB_SUCCESS; + if (ZERO_DATETIME == in_value_us) { + out_value.set_null_value(); + LOG_DEBUG("null odate_to_otimestamp", K(ret), K(in_value_us), K(out_type), K(lbt())); + } else if (OB_ISNULL(tz_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_info is null, it should not happened", K(ret)); + } else if (ObTimestampTZType == out_type) { + int32_t offset_min = 0; + int32_t tz_id = OB_INVALID_INDEX; + int32_t tran_type_id = OB_INVALID_INDEX; + if (OB_FAIL(sub_timezone_offset(*tz_info, ObString(), in_value_us, offset_min, tz_id, tran_type_id))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } else { + out_value.time_us_ = in_value_us; + out_value.time_ctx_.tail_nsec_ = 0; + if (OB_INVALID_INDEX == tz_id) { + out_value.time_ctx_.store_tz_id_ = 0; + out_value.time_ctx_.set_offset_min(offset_min); + } else { + out_value.time_ctx_.store_tz_id_ = 1; + out_value.time_ctx_.set_tz_id(tz_id); + out_value.time_ctx_.set_tran_type_id(tran_type_id); + } + } + } else if (ObTimestampLTZType == out_type) { + if (OB_FAIL(sub_timezone_offset(tz_info, true, ObString(), in_value_us, true))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } else { + out_value.time_us_ = in_value_us; + out_value.time_ctx_.tail_nsec_ = 0; + } + } else if (ObTimestampNanoType == out_type) { + out_value.time_us_ = in_value_us; + out_value.time_ctx_.tail_nsec_ = 0; + } else { + ret = OB_ERR_UNEXPECTED; + } + return ret; +} + +// timestamp tz(utc, offset) --> date(local) +// timestamp ltz(utc) --> date(local) +// timestamp (local) --> date(local) +int ObTimeConverter::otimestamp_to_odate(const ObObjType in_type, + const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, + int64_t &out_usec) +{ + int ret = OB_SUCCESS; + if (in_value.is_null_value()) { + out_usec = ZERO_DATETIME; + } else if (OB_ISNULL(tz_info)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("tz_info is null", K(ret)); + } else if (ObTimestampTZType == in_type) { + int32_t offset_min = 0; + ObTime ob_time(DT_TYPE_ORACLE_TIMESTAMP); + if (OB_FAIL(extract_offset_from_otimestamp(in_value, tz_info, offset_min, ob_time))) { + LOG_WARN("failed to extract_offset_from_otimestamp", K(ret)); + } else { + out_usec = in_value.time_us_ + MIN_TO_USEC(offset_min); + } + } else if (ObTimestampLTZType == in_type) { + out_usec = in_value.time_us_; + if (OB_FAIL(add_timezone_offset(tz_info, out_usec))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } + } else if (ObTimestampNanoType == in_type) { + out_usec = in_value.time_us_; + } else { + ret = OB_ERR_UNEXPECTED; + } + LOG_DEBUG("succ otimestamp_to_odate", K(ret), K(in_type), K(in_value), K(out_usec)); + return ret; +} + +int ObTimeConverter::otimestamp_to_otimestamp(const ObObjType in_type, + const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, + const ObObjType out_type, + ObOTimestampData &out_value) +{ + int ret = OB_SUCCESS; + out_value.reset(); + if (in_value.is_null_value()) { + out_value.set_null_value(); + } else if (out_type == in_type) { + out_value = in_value; + } else if (ObTimestampLTZType == in_type && ObTimestampTZType == out_type) { + int64_t in_value_us = in_value.time_us_; + ObString tz_abbr_str; + int32_t offset_sec = 0; + int32_t tz_id = OB_INVALID_INDEX; + int32_t tran_type_id = OB_INVALID_INDEX; + if (OB_ISNULL(tz_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_info should not be null", K(ret), K(lbt())); + } else if (OB_FAIL(tz_info->get_timezone_offset(in_value_us, offset_sec, tz_abbr_str, tran_type_id))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } else { + out_value.time_us_ = in_value_us; + out_value.time_ctx_.tail_nsec_ = in_value.time_ctx_.tail_nsec_; + if (common::OB_INVALID_TZ_ID == tz_info->get_tz_id()) { + out_value.time_ctx_.store_tz_id_ = 0; + out_value.time_ctx_.set_offset_min(static_cast(SEC_TO_MIN(offset_sec))); + } else { + out_value.time_ctx_.store_tz_id_ = 1; + out_value.time_ctx_.set_tz_id(tz_id); + out_value.time_ctx_.set_tran_type_id(tran_type_id); + } + } + } else if (ObTimestampTZType == in_type && ObTimestampLTZType == out_type) { + out_value.time_us_ = in_value.time_us_; + out_value.time_ctx_.tail_nsec_ = in_value.time_ctx_.tail_nsec_; + } + LOG_DEBUG("succ otimestamp_to_otimestamp", K(ret), K(in_type), K(in_value), K(out_type), K(out_value)); + return ret; +} + +int ObTimeConverter::otimestamp_to_str(const ObOTimestampData &ot_data, + const ObDataTypeCastParams &dtc_params, + const int16_t scale, + const ObObjType type, + char *buf, + int64_t buf_len, + int64_t &pos) +{ + int ret = OB_SUCCESS; + if (ot_data.is_null_value()) { + LOG_DEBUG("succ to null otimestamp_to_str", K(ot_data), K(type), K(scale), K(lbt())); + } else { + const ObOTimestampData &tmp_ot_data = round_otimestamp(scale, ot_data); + const bool store_utc_time = false; + ObTime ob_time(DT_TYPE_ORACLE_TIMESTAMP); + const int64_t old_pos = pos; + const ObString format_str = dtc_params.get_nls_format(type); + if (OB_FAIL(otimestamp_to_ob_time(type, tmp_ot_data, dtc_params.tz_info_, ob_time, store_utc_time))) { + LOG_WARN("failed to convert otimestamp to ob time", K(ret), K(lbt())); + } else if (OB_FAIL(dtc_params.force_use_standard_format_ + ? ob_time_to_str(ob_time, DT_TYPE_DATETIME, scale, buf, buf_len, pos, true) + : ob_time_to_str_oracle_dfm(ob_time, scale, format_str, buf, buf_len, pos))) { + LOG_WARN("failed to convert ob time to string", K(format_str), K(ob_time), K(ret)); + } else { + ObString tmp(pos - old_pos, buf + old_pos); + LOG_DEBUG("succ to otimestamp_to_str", + K(ot_data), K(type), K(scale), K(tmp_ot_data), K(tmp), K(format_str), K(lbt())); + } + } + return ret; +} + +bool ObTimeConverter::is_valid_otimestamp(const int64_t time_us, const int32_t tail_nsec) +{ + return ((lib::is_oracle_mode() ? ORACLE_DATETIME_MIN_VAL : DATETIME_MIN_VAL) <= time_us + && time_us <= DATETIME_MAX_VAL + && ObOTimestampData::MIN_TAIL_NSEC <= tail_nsec + && tail_nsec <= ObOTimestampData::MAX_TAIL_NSEC); +} + +int ObTimeConverter::extract_offset_from_otimestamp(const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, + int32_t &offset_min, + ObTime &ob_time) +{ + int ret = OB_SUCCESS; + if (in_value.time_ctx_.store_tz_id_) { + ObTZInfoMap *tz_info_map = NULL; + ObTimeZoneInfoPos *literal_tz_info = NULL; + ObString tz_name_str; + ObString tz_abbr_str; + int32_t offset_sec = 0; + if (OB_ISNULL(tz_info)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("tz_info is null", K(ret)); + } else if (OB_ISNULL(tz_info_map = const_cast(tz_info->get_tz_info_map()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_info_map is NULL", K(ret)); + } else if (OB_FAIL(tz_info_map->get_tz_info_by_id(in_value.time_ctx_.tz_id_, literal_tz_info))) { + LOG_WARN("fail to get_tz_info_by_id", "tz_id", in_value.time_ctx_.tz_id_, K(ret)); + ret = OB_ERR_INVALID_TIMEZONE_REGION_ID; + } else if (OB_FAIL(literal_tz_info->get_timezone_offset(in_value.time_ctx_.tran_type_id_, tz_abbr_str, offset_sec))) { + LOG_WARN("fail to get_timezone_offset", K(in_value), K(ret)); + ret = OB_ERR_INVALID_TIMEZONE_REGION_ID; + } else if (OB_FAIL(literal_tz_info->get_tz_name(tz_name_str))) { + LOG_WARN("fail to get_tz_name", K(tz_name_str), K(ret)); + } else if (OB_FAIL(ob_time.set_tz_name(tz_name_str))) { + LOG_WARN("fail to set_tz_name", K(tz_name_str), K(ret)); + } else if (OB_FAIL(ob_time.set_tzd_abbr(tz_abbr_str))) { + LOG_WARN("fail to set_tz_abbr", K(tz_abbr_str), K(ret)); + } else { + offset_min = static_cast(SEC_TO_MIN(offset_sec)); + ob_time.time_zone_id_ = in_value.time_ctx_.tz_id_; + ob_time.transition_type_id_ = in_value.time_ctx_.tran_type_id_; + ob_time.parts_[DT_OFFSET_MIN] = offset_min; + } + LOG_DEBUG("extract_offset_from_otimestamp", K(ob_time), K(offset_min), K(offset_sec), K(ret)); + + if (NULL != tz_info_map && NULL != literal_tz_info) { + tz_info_map->free_tz_info_pos(literal_tz_info); + } + } else { + offset_min = in_value.time_ctx_.get_offset_min(); + ob_time.parts_[DT_OFFSET_MIN] = offset_min; + } + LOG_DEBUG("extract_offset_from_otimestamp", K(ob_time), K(offset_min), K(ret)); + return ret; +} + int ObTimeConverter::datetime_to_date(int64_t dt_value, const ObTimeZoneInfo *tz_info, int32_t &d_value) { int ret = OB_SUCCESS; @@ -633,21 +1017,67 @@ int ObTimeConverter::date_to_year(int32_t d_value, uint8_t &y_value) return ret; } +int ObTimeConverter::check_leading_precision(const ObTimeDigits &digits) +{ + int ret = OB_SUCCESS; + const int64_t oracle_max_leading_precision = 9; + if (digits.value_ != 0 && digits.len_ > oracle_max_leading_precision) { + int64_t leading_zero_count = 0; + for (int64_t i = 0; i < digits.len_ && '0' == digits.ptr_[i]; i++) { + leading_zero_count++; + } + if (leading_zero_count >= oracle_max_leading_precision) { + ret = OB_ERR_THE_LEADING_PRECISION_OF_THE_INTERVAL_IS_TOO_SMALL; + } + } + return ret; +} + + //////////////////////////////// // string -> offset. -#define OFFSET_MIN static_cast(-(12 * MINS_PER_HOUR + 59) * SECS_PER_MIN) // -12:59 . -#define OFFSET_MAX static_cast(13 * SECS_PER_HOUR) // +13:00 . -int ObTimeConverter::str_to_offset(const ObString &str, int32_t &value) +#define OFFSET_MIN static_cast(-(12 * MINS_PER_HOUR + 59) * SECS_PER_MIN) // -12:59 . +#define OFFSET_MAX static_cast(13 * SECS_PER_HOUR) // +13:00 . +#define ORACLE_OFFSET_MIN static_cast(-(15 * MINS_PER_HOUR + 59) * SECS_PER_MIN) // -15:59 . +#define ORACLE_OFFSET_MAX static_cast(15 * SECS_PER_HOUR) // +15:00 . +#define ORACLE_OFFSET_MAX_HOUR 15 + +/* str_to_offset usually return OB_ERR_UNKNOWN_TIME_ZONE when failed. + * Then str is treated as a position and searched in time_zone_map, such 'Asia/Shanghai'. + * If search failed, mysql reports OB_ERR_UNKNOWN_TIME_ZONE while oracle may return different error code. + * Such as OB_ERR_INVALID_TIME_ZONE_HOUR when hour is greater than 15, + * and OB_ERR_INVALID_TIME_ZONE_MINUTE when minute is greater than 59. + * ret_more records this error code, and if search time_zone_map failed, we choose whether + * set ret = ret_more according to compatible mode. + */ +int ObTimeConverter::str_to_offset(const ObString &str, + int32_t &value, + int &ret_more, + const bool is_oracle_mode, + const bool need_check_valid /* false */) { int ret = OB_SUCCESS; - if (OB_ISNULL(str.ptr()) || OB_UNLIKELY(str.length() <= 0)) { + ret_more = OB_SUCCESS; + const ObString tmp_str = (is_oracle_mode ? const_cast(str).trim() : str); + if (OB_ISNULL(tmp_str.ptr()) || OB_UNLIKELY(tmp_str.length() <= 0)) { ret = OB_ERR_UNKNOWN_TIME_ZONE; - LOG_WARN("invalid time zone offset", K(ret), K(str)); + LOG_WARN("invalid time zone offset", K(ret), K(str), K(tmp_str)); } else { - const char *pos = str.ptr(); - const char *end = pos + str.length(); - char sign = *pos++; + const char *pos = tmp_str.ptr(); + const char *end = pos + tmp_str.length(); + char sign = *pos; + if (is_oracle_mode) { + // default is + + if ('+' != sign && '-' != sign) { + sign = '+'; + } else { + pos++; + } + } else { + pos++; + } + ObTimeDigits hour; ObTimeDigits minute; ObTimeDelims colon; @@ -656,23 +1086,57 @@ int ObTimeConverter::str_to_offset(const ObString &str, int32_t &value) LOG_WARN("failed to get offset", K(ret), K(str)); } else if (OB_FAIL(get_datetime_digits_delims(pos, end, INT32_MAX, minute, none))) { LOG_WARN("failed to get offset", K(ret), K(str)); - } - if (!('+' == sign || '-' == sign) - || 0 == hour.len_ || 0 == minute.len_ - || !is_single_colon(colon) || none.len_ > 0) { + } else if (!('+' == sign || '-' == sign) + || 0 == hour.len_ + || 0 == minute.len_ + // oracle just ignore decimal part and invalid char in the end. + || !is_single_colon(colon) + || (none.len_ > 0 && !is_oracle_mode)) { + ret = OB_ERR_UNKNOWN_TIME_ZONE; + } else if (!need_check_valid) { + /* sometimes no need check validation, such as session deserialization + * and load time_zone system variable to session. + * We only need check validation when set sys variable. + */ + value = static_cast(((hour.value_ * MINS_PER_HOUR) + minute.value_) * SECS_PER_MIN); + if ('-' == sign) { + value = -value; + } + } else if (is_oracle_mode && OB_FAIL(check_leading_precision(hour))) { + LOG_WARN("check hour leading precision failed", K(ret), K(hour)); + } else if (is_oracle_mode && OB_FAIL(check_leading_precision(minute))) { + LOG_WARN("check minute leading precision failed", K(ret), K(minute)); + } else if (OB_UNLIKELY(minute.value_ >= MINS_PER_HOUR || minute.value_ < 0)) { ret = OB_ERR_UNKNOWN_TIME_ZONE; - LOG_WARN("invalid time zone offset", K(ret), K(str)); + // Oracle reports hour out of range when both hour and minute out of range. + ret_more = is_oracle_mode ? (hour.value_ > ORACLE_OFFSET_MAX_HOUR ? OB_ERR_INVALID_TIME_ZONE_HOUR + : OB_ERR_INVALID_TIME_ZONE_MINUTE) + : ret_more; } else { - value = ((hour.value_ * MINS_PER_HOUR) + minute.value_) * SECS_PER_MIN; + value = static_cast(((hour.value_ * MINS_PER_HOUR) + minute.value_) * SECS_PER_MIN); if ('-' == sign) { value = -value; } - if (!(OFFSET_MIN <= value && value <= OFFSET_MAX)) { - ret = OB_ERR_UNKNOWN_TIME_ZONE; - LOG_WARN("invalid time zone offset", K(ret), K(str)); + + if (is_oracle_mode) { + if (OB_UNLIKELY(!(ORACLE_OFFSET_MIN <= value && value <= ORACLE_OFFSET_MAX))) { + ret_more = + hour.value_ > ORACLE_OFFSET_MAX_HOUR ? OB_ERR_INVALID_TIME_ZONE_HOUR : OB_ERR_INVALID_TIME_ZONE_MINUTE; + ret = OB_ERR_UNKNOWN_TIME_ZONE; + LOG_WARN("invalid time zone offset", K(ret), K(minute.value_), K(str)); + } + LOG_DEBUG("finish str_to_offset", K(str), K(value), K(ret), K(lbt())); + } else { + if (OB_UNLIKELY(!(OFFSET_MIN <= value && value <= OFFSET_MAX))) { + ret_more = + (minute.value_ >= DT_PART_BASE[DT_MIN] ? OB_ERR_INVALID_TIME_ZONE_MINUTE : OB_ERR_INVALID_TIME_ZONE_HOUR); + ret = OB_ERR_UNKNOWN_TIME_ZONE; + LOG_WARN("invalid time zone offset", K(ret), K(minute.value_), K(str)); + } } } } + LOG_DEBUG("finish str_to_offset", K(ret), K(ret_more), K(str), K(value)); return ret; } @@ -718,6 +1182,36 @@ void ObTimeConverter::trunc_datetime(int16_t scale, int64_t &value) //todo: test the date before 1970 } +int ObTimeConverter::get_oracle_err_when_datetime_parts_conflict(int64_t part_idx) +{ + int ret = OB_SUCCESS; + switch (part_idx) { + case DT_YEAR: + ret = OB_ERR_UNEXPECTED; // never goes here for now + break; + case DT_MON: + ret = OB_ERR_MONTH_CONFLICTS_WITH_JULIAN_DATE; // ORA-01833: month conflicts with Julian date + break; + case DT_MDAY: + ret = OB_ERR_DAY_OF_MONTH_CONFLICTS_WITH_JULIAN_DATE; // ORA-01834: day of month conflicts with Julian date + break; + case DT_HOUR: + ret = OB_ERR_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY; // ORA-01836: hour conflicts with seconds in day + break; + case DT_MIN: + ret = OB_ERR_MINUTES_OF_HOUR_CONFLICTS_WITH_SECONDS_IN_DAY; // ORA-01837: minutes of hour conflicts with seconds + // in day + break; + case DT_SEC: + ret = OB_ERR_SECONDS_OF_MINUTE_CONFLICTS_WITH_SECONDS_IN_DAY; // ORA-01838: seconds of minute conflicts with + // seconds in day + break; + default: + ret = OB_ERR_UNEXPECTED; + } + return ret; +} + //////////////////////////////// // date add / sub / diff. @@ -1455,114 +1949,1090 @@ int ObTimeConverter::str_to_ob_interval(const ObString &str, ObDateUnitType unit } return ret; } -int ObTimeConverter::datetime_to_ob_time(int64_t value, const ObTimeZoneInfo *tz_info, ObTime &ob_time) + +/** + * @brief convert string to ob_time struct according to oracle datetime format model + * @param in: str input string + * @param in: format format string + * @param in: cvrt_ctx + * @param in: target_type includes ObDateTimeType, ObOTimestampTZType, ObOTimestampLTZType, ObOTimestampType + * @param out: ob_time memory struct of datetime + * @param out: scale scale of fractional seconds + */ +int ObTimeConverter::str_to_ob_time_oracle_dfm(const ObString &str, + const ObTimeConvertCtx &cvrt_ctx, + const ObObjType target_type, + ObTime &ob_time, + ObScale &scale) { int ret = OB_SUCCESS; - int64_t usec = value; - if (OB_UNLIKELY(ZERO_DATETIME == usec)) { - MEMSET(ob_time.parts_, 0, sizeof(*ob_time.parts_) * TOTAL_PART_CNT); - ob_time.parts_[DT_DATE] = ZERO_DATE; - } else if (OB_FAIL(add_timezone_offset(tz_info, usec))) { - LOG_WARN("failed to adjust value with time zone offset", K(ret)); + const ObString &format = cvrt_ctx.oracle_nls_format_; + if (OB_UNLIKELY(str.empty() + || format.empty() + || (!ob_is_otimestamp_type(target_type) + && !ob_is_datetime_tc(target_type)))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(str), K(format), K(ob_time.mode_)); } else { - int32_t days = static_cast(usec / USECS_PER_DAY); - usec %= USECS_PER_DAY; - if (OB_UNLIKELY(usec < 0)) { - --days; - usec += USECS_PER_DAY; + ob_time.mode_ |= DT_TYPE_DATETIME; + if (ob_is_otimestamp_type(target_type)) { + ob_time.mode_ |= DT_TYPE_ORACLE; } - if (OB_FAIL(date_to_ob_time(days, ob_time))) { - LOG_WARN("failed to convert date part to obtime", K(ret), K(usec)); - } else if (OB_FAIL(time_to_ob_time(usec, ob_time))) { - LOG_WARN("failed to convert time part to obtime", K(ret), K(usec)); + if (ob_is_timestamp_tz(target_type)) { + ob_time.mode_ |= DT_TYPE_TIMEZONE; } } - return ret; -} -int ObTimeConverter::date_to_ob_time(int32_t value, ObTime &ob_time) -{ - int ret = OB_SUCCESS; - int32_t *parts = ob_time.parts_; - if (OB_UNLIKELY(ZERO_DATE == value)) { - memset(parts, 0, sizeof(*parts) * DATETIME_PART_CNT); - parts[DT_DATE] = ZERO_DATE; - } else { - int32_t days = value; - int32_t leap_year = 0; - int32_t year = EPOCH_YEAR4; - parts[DT_DATE] = value; - // year. - while (days < 0 || days >= DAYS_PER_YEAR[leap_year = IS_LEAP_YEAR(year)]) { - int32_t new_year = year + days / DAYS_PER_NYEAR; - new_year -= (days < 0); - days -= (new_year - year) * DAYS_PER_NYEAR + LEAP_YEAR_COUNT(new_year - 1) - LEAP_YEAR_COUNT(year - 1); - year = new_year; + if (OB_SUCC(ret)) { + ObSEArray dfm_elems; + ObBitSet elem_flags; + + // 1. parse and check semantic of format string + if (OB_FAIL(ObDFMUtil::parse_datetime_format_string(format, dfm_elems))) { + LOG_WARN("fail to parse oracle datetime format string", K(ret), K(format)); + } else if (OB_FAIL(ObDFMUtil::check_semantic(dfm_elems, elem_flags, ob_time.mode_))) { + LOG_WARN("check semantic of format string failed", K(ret), K(format)); } - parts[DT_YEAR] = year; - parts[DT_YDAY] = days + 1; - parts[DT_WDAY] = WDAY_OFFSET[value % DAYS_PER_WEEK][EPOCH_WDAY]; - // month. - const int32_t *cur_days_until_mon = DAYS_UNTIL_MON[leap_year]; - int32_t month = 1; - for (; month < MONS_PER_YEAR && days >= cur_days_until_mon[month]; ++month) {} - parts[DT_MON] = month; - days -= cur_days_until_mon[month - 1]; - // day. - parts[DT_MDAY] = days + 1; - } - return ret; -} -int ObTimeConverter::time_to_ob_time(int64_t value, ObTime &ob_time) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(value < 0)) { - ob_time.mode_ |= DT_MODE_NEG; - value = -value; - } - int64_t secs = USEC_TO_SEC(value); - ob_time.parts_[DT_HOUR] = static_cast(secs / SECS_PER_HOUR); - secs %= SECS_PER_HOUR; - ob_time.parts_[DT_MIN] = static_cast(secs / SECS_PER_MIN); - ob_time.parts_[DT_SEC] = static_cast(secs % SECS_PER_MIN); - ob_time.parts_[DT_USEC] = static_cast(value % USECS_PER_SEC); - return ret; -} + int32_t temp_tzh_value = -1; // positive value is legal + int32_t temp_tzm_value = -1; // positive value is legal + int32_t temp_tz_factor = 0; -//////////////////////////////// -// int / uint / string <- ObTime -> datetime / date / time. + int32_t tz_hour = 0; // will be negetive when time zone offset < 0 + int32_t tz_min = 0; // will be negetive when time zone offset < 0 + int32_t session_tz_id = 0; + int32_t session_tran_type_id = 0; -#define DTAE_DIGIT_LEN 8 -#define TIME_DIGIT_LEN 6 + int32_t session_tz_offset = 0; + // 2. set default value for ob_time + if (OB_SUCC(ret)) { + int64_t utc_curr_time = ObTimeUtility::current_time(); + int64_t utc_curr_time_copy = utc_curr_time; + int32_t cur_date = 0; + if (OB_ISNULL(cvrt_ctx.tz_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session timezone info is null", K(ret)); + } else if (sub_timezone_offset(*(cvrt_ctx.tz_info_), + ObString(), + utc_curr_time_copy, + session_tz_offset, + session_tz_id, + session_tran_type_id)) { + LOG_WARN("get session timezone offset failed", K(ret)); + } else if (OB_FAIL(datetime_to_date(utc_curr_time, cvrt_ctx.tz_info_, cur_date))) { + LOG_WARN("timestamp to date failed", K(ret)); + } else if (OB_FAIL(time_to_ob_time(0, ob_time))) { + LOG_WARN("time to ob_time failed", K(ret)); + } else if (OB_FAIL(date_to_ob_time(cur_date, ob_time))) { + LOG_WARN("date to ob_time failed", K(ret), K(cur_date)); + } else { + if (OB_INVALID_TZ_ID != session_tz_id && ob_is_timestamp_tz(target_type)) { + ob_time.is_tz_name_valid_ = true; + } + ob_time.parts_[DT_MDAY] = 1; // oracle default value is the first day of current month + ob_time.parts_[DT_DATE] = 0; // will be recalculated + ob_time.parts_[DT_YDAY] = 0; // doesn't matter + ob_time.parts_[DT_WDAY] = 0; // doesn't matter + tz_hour = static_cast(session_tz_offset / MINS_PER_HOUR); + tz_min = static_cast(session_tz_offset - MINS_PER_HOUR * tz_hour); + if (OB_FAIL(ObDFMLimit::TIMEZONE_HOUR_ABS.validate(abs(tz_hour)))) { + LOG_WARN("invalid session timezone hour", K(ret), K(tz_hour)); + } else if (OB_FAIL(ObDFMLimit::TIMEZONE_MIN_ABS.validate(abs(tz_min)))) { + LOG_WARN("invali d session timezone minutes", K(ret), K(tz_min)); + } + } + } -OB_INLINE int64_t ObTimeConverter::ob_time_to_int(const ObTime &ob_time, ObDTMode mode) -{ - int64_t value = 0; - if (DT_TYPE_DATE & mode) { - value = ob_time.parts_[DT_YEAR] * static_cast(power_of_10[4]) - + ob_time.parts_[DT_MON] * static_cast(power_of_10[2]) - + ob_time.parts_[DT_MDAY]; - } - if (DT_TYPE_TIME & mode) { - value *= static_cast(power_of_10[6]); - value += (ob_time.parts_[DT_HOUR] * static_cast(power_of_10[4]) - + ob_time.parts_[DT_MIN] * static_cast(power_of_10[2]) - + ob_time.parts_[DT_SEC]); - } - return value; -} + // 3. go through each element, set and check conflict for ob_time parts: Year, Month, Day, Hour, Minute, Second + if (OB_SUCC(ret)) { + ObDFMParseCtx ctx(str.ptr(), str.length()); + int64_t last_elem_end_pos = 0; + int64_t conflict_part_bitset = 0; + int64_t elem_idx = 0; + int64_t input_sep_len = 0; + int64_t part_blank1_len = 0; + int64_t part_sep_len = 0; + int64_t part_blank2_len = 0; + int64_t format_sep_len = 0; + STATIC_ASSERT((1 << TOTAL_PART_CNT) < INT64_MAX, "for time_part_conflict_bitset"); + int32_t yday_temp_value = ZERO_DATE; // as invalid value + int32_t wday_temp_value = ZERO_DATE; // as invalid value + int32_t date_temp_value = ZERO_DATE; // as invalid value + int32_t hh12_temp_value = ZERO_TIME; + int32_t julian_year_value = ZERO_DATE; // as invalid value + bool is_after_noon = false; + bool is_before_christ = false; + bool has_digit_tz_in_TZR = false; + int64_t first_non_space_sep_char = INT64_MAX; + int64_t ignore_fs_flag = false; + + for (elem_idx = 0; OB_SUCC(ret) && elem_idx < dfm_elems.count(); ++elem_idx) { + ObDFMElem &elem = dfm_elems.at(elem_idx); + if (OB_UNLIKELY(!elem.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("element is invalid", K(ret), K(elem)); + } else { + LOG_DEBUG("DFM DEBUG: start element", K(elem), K(ctx)); + } -static const int32_t DT_PART_MUL[DATETIME_PART_CNT] = {100, 100, 100, 100, 100, 1000000, 1}; -static const int64_t ZERO_DATE_WEEK = 613566757; // for 0000-00-00 00:00:00 . -int64_t ObTimeConverter::ob_time_to_int_extract(const ObTime &ob_time, ObDateUnitType unit_type) -{ - int64_t value = 0; - switch (unit_type) { - case DATE_UNIT_WEEK: { - value = ZERO_DATE == ob_time.parts_[DT_DATE] - ? ZERO_DATE_WEEK - : ObTimeConverter::ob_time_to_week(ob_time, DT_WEEK_SUN_BEGIN | DT_WEEK_ZERO_BEGIN); + // 1. check separate chars and skip blank chars first + if (OB_SUCC(ret)) { + // get format sep chars len + format_sep_len = elem.offset_ - last_elem_end_pos; + last_elem_end_pos = elem.offset_ + ObDFMFlag::PATTERN[elem.elem_flag_].len_; + // parse input string and skip them + part_blank1_len = ObDFMUtil::skip_blank_chars(ctx); + // The # of skipped non-blank chars is according to format_str + first_non_space_sep_char = ctx.is_parse_finish() ? INT64_MAX : ctx.cur_ch_[0]; + if (ObDFMFlag::X == elem.elem_flag_) { + part_sep_len = 0; + } else { + part_sep_len = ObDFMUtil::skip_separate_chars(ctx, format_sep_len); + } + part_blank2_len = ObDFMUtil::skip_blank_chars(ctx); + input_sep_len = part_blank1_len + part_sep_len + part_blank2_len; + LOG_DEBUG("DFM DEBUG: skip blank and serarate chars", + K(part_blank1_len), + K(part_sep_len), + K(part_blank2_len), + K(format_sep_len), + K(ctx)); + } + + if (OB_SUCC(ret) && ctx.is_parse_finish()) { + break; // if all the input chars has beeen processed, break this loop + } + + // 2. next, parse the current element + if (OB_SUCC(ret)) { + int64_t parsed_elem_len = 0; + const int64_t expected_elem_len = ObDFMFlag::EXPECTED_MATCHING_LENGTH[elem.elem_flag_]; + ctx.set_next_expected_elem(elem.elem_flag_, format_sep_len > 0 && input_sep_len == 0); + switch (elem.elem_flag_) { + case ObDFMFlag::AD: + case ObDFMFlag::BC: + case ObDFMFlag::AD2: + case ObDFMFlag::BC2: { // TODO : NLS_LANGUAGE + bool is_with_dot = (ObDFMFlag::AD2 == elem.elem_flag_ || ObDFMFlag::BC2 == elem.elem_flag_); + parsed_elem_len = + is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::AD2].len_ : ObDFMFlag::PATTERN[ObDFMFlag::AD].len_; + bool is_ad = ObDFMUtil::match_pattern_ignore_case( + ctx, is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::AD2] : ObDFMFlag::PATTERN[ObDFMFlag::AD]); + bool is_bc = ObDFMUtil::match_pattern_ignore_case( + ctx, is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::BC2] : ObDFMFlag::PATTERN[ObDFMFlag::BC]); + if (!is_ad && !is_bc) { + ret = OB_ERR_BC_OR_AD_REQUIRED; + } else { + is_before_christ = is_bc; + } + break; + } + case ObDFMFlag::D: { + int32_t wday = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, wday))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::WEEK_DAY.validate(wday))) { + LOG_WARN("not a valid day of the week", K(ret), K(wday)); + } else { + // oracle numbered sunday as 1 in territory of CHINA + // TODO : hard code for now, need look up NLS_TERRITORIES + wday_temp_value = (wday + 5) % 7 + 1; + } + break; + } + case ObDFMFlag::DY: + case ObDFMFlag::DAY: { // TODO : NLS_LANGUAGE NLS_TERRITORIES + int32_t wday = 0; + for (wday = 1; wday <= DAYS_PER_WEEK; ++wday) { + const ObTimeConstStr &day_str = + (elem.elem_flag_ == ObDFMFlag::DAY) ? WDAY_NAMES[wday] : WDAY_ABBR_NAMES[wday]; + if (ObDFMUtil::match_pattern_ignore_case(ctx, day_str)) { + parsed_elem_len = day_str.len_; + break; + } + } + if (OB_FAIL(ObDFMLimit::WEEK_DAY.validate(wday))) { + LOG_WARN("validate week day failed", K(ret), K(wday)); + } else { + wday_temp_value = wday; + } + break; + } + case ObDFMFlag::DD: { + int32_t mday = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, mday))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::MONTH_DAY.validate(mday))) { + LOG_WARN("day of month must be between 1 and last day of month", K(ret), K(mday)); + } else { + // may conflict with DDD + ret = set_ob_time_part_directly(ob_time, conflict_part_bitset, DT_MDAY, mday); + } + break; + } + case ObDFMFlag::DDD: { + int32_t yday = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, yday))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::YEAR_DAY.validate(yday))) { + LOG_WARN("day of year must be between 1 and 365 (366 for leap year)", K(ret), K(yday)); + } else { + yday_temp_value = yday; + } + break; + } + case ObDFMFlag::DS: { // TODO : impl it NEED NLS_DATE_FORMAT NLS_TERRITORY NLS_LANGUAGE + ret = OB_NOT_SUPPORTED; + LOG_WARN("DS is not supported now", K(ret)); + break; + } + case ObDFMFlag::DL: { // TODO : impl it NEED NLS_DATE_FORMAT NLS_TERRITORY NLS_LANGUAGE + ret = OB_NOT_SUPPORTED; + LOG_WARN("DL is not supported now", K(ret)); + break; + } + case ObDFMFlag::FF: + case ObDFMFlag::FF1: + case ObDFMFlag::FF2: + case ObDFMFlag::FF3: + case ObDFMFlag::FF4: + case ObDFMFlag::FF5: + case ObDFMFlag::FF6: + case ObDFMFlag::FF7: + case ObDFMFlag::FF8: + case ObDFMFlag::FF9: { + int32_t usec = 0; + // format string has '.' or 'X', but input string does not contain '.' + // do nothing, skip element FF and revert ctx by the length of parsed chars + if (ignore_fs_flag) { + ctx.revert(part_blank1_len + part_sep_len + part_blank2_len); + } else if (elem.is_single_dot_before_ && '.' != first_non_space_sep_char) { + ctx.revert(part_sep_len + part_blank2_len); + } else if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, usec))) { + LOG_WARN("failed to match usecs", K(ret), K(ctx)); + } else { + scale = static_cast(parsed_elem_len); + usec = static_cast(usec * power_of_10[MAX_SCALE_FOR_ORACLE_TEMPORAL - parsed_elem_len]); + ob_time.parts_[DT_USEC] = usec; + } + break; + } + + case ObDFMFlag::TZH: + case ObDFMFlag::TZM: { + if (OB_UNLIKELY(elem_flags.has_member(ObDFMFlag::TZR) || elem_flags.has_member(ObDFMFlag::TZD))) { + ret = OB_ERR_NOT_A_VALID_TIME_ZONE; + LOG_WARN("tzh tzm and tzr tzd can not appear at the same time", K(ret)); + } else { + // SQL> alter session set NLS_TIMESTAMP_TZ_FORMAT='DD-MON-RR HH.MI.SS AM TZH:TZM'; + // SQL> alter session set time_zone='Asia/Shanghai'; + // SQL> select cast('01-SEP-20 11.11.11' as timestamp with time zone) from dual; + // 01-SEP-20 11.11.11 AM +08:00 + // If format contains TZH and time_zone is a position, need to convert first. + ob_time.is_tz_name_valid_ = false; + int32_t value = 0; + int32_t local_tz_factor = 1; + if (ObDFMFlag::TZH == elem.elem_flag_) { + if (ObDFMUtil::is_sign_char(ctx.cur_ch_[0])) { + local_tz_factor = ('-' == ctx.cur_ch_[0] ? -1 : 1); + ctx.update(1); + } else { + if (ctx.get_parsed_len() > 0 && input_sep_len > format_sep_len) { + // if the input valid separate chars > format separate chars + // the superfluous '-' will be regarded as minus sign + local_tz_factor = (static_cast('-') == ctx.cur_ch_[-1] ? -1 : 1); + } + } + temp_tz_factor = local_tz_factor; // 1 or -1, but never be 0 + } + + if (ctx.is_parse_finish()) { + // do nothing + } else if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, value))) { + LOG_WARN("failed to match usecs", K(ret), K(ctx)); + } else if (OB_FAIL((elem.elem_flag_ == ObDFMFlag::TZH) + ? ObDFMLimit::TIMEZONE_HOUR_ABS.validate(value) + : ObDFMLimit::TIMEZONE_MIN_ABS.validate(value))) { + LOG_WARN("failed to validate timezone value", K(value), K(ret)); + } else { + if (elem.elem_flag_ == ObDFMFlag::TZH) { + temp_tzh_value = value; + } else { + temp_tzm_value = value; + } + } + } + break; + } + + case ObDFMFlag::TZR: { + if (OB_UNLIKELY(elem_flags.has_member(ObDFMFlag::TZH) || elem_flags.has_member(ObDFMFlag::TZM))) { + ret = OB_ERR_NOT_A_VALID_TIME_ZONE; + LOG_WARN("tzh tzm and tzr tzd can not appear at the same time", K(ret)); + } else { + int32_t local_tz_factor = 1; + if (isdigit(ctx.cur_ch_[0]) || ObDFMUtil::is_sign_char(ctx.cur_ch_[0])) { // case1: digits + int32_t tmp_tz_hour = 0; + int32_t tmp_tz_min = 0; + if (ObDFMUtil::is_sign_char(ctx.cur_ch_[0])) { + local_tz_factor = ('-' == ctx.cur_ch_[0] ? -1 : 1); + ctx.update(1); + } else if (part_blank1_len + part_sep_len > format_sep_len && + ObDFMUtil::is_sign_char(ctx.cur_ch_[-1])) { + local_tz_factor = ('-' == ctx.cur_ch_[-1] ? -1 : 1); + } + + ObString digits_timezone; + parsed_elem_len = ObDFMUtil::UNKNOWN_LENGTH_OF_ELEMENT; + if (OB_LIKELY(!ctx.is_parse_finish())) { + // If tz str is not just character '+' or '-', result of convert should be + // offset instead of tz_id/name + ob_time.is_tz_name_valid_ = false; + } + if (OB_UNLIKELY(ctx.is_parse_finish())) { + // do nothing + } else if (OB_FAIL(ObDFMUtil::match_chars_until_space(ctx, digits_timezone, parsed_elem_len))) { + // do nothing + } else { + ObDFMParseCtx local_ctx(digits_timezone.ptr(), digits_timezone.length()); + const char *local_sep = ObDFMUtil::find_first_separator(local_ctx); + + if (OB_ISNULL(local_sep)) { + // timezone offset string is allowd to contain only hour part. + } else { + int64_t hour_expected_len = local_sep - digits_timezone.ptr(); + int64_t local_parsed_len = 0; + if (OB_FAIL(ObDFMUtil::match_int_value( + local_ctx, hour_expected_len, local_parsed_len, tmp_tz_hour, local_tz_factor))) { + LOG_WARN("matching timezone hour failed, for 'TZR'. the error is ignored.", + K(ret), + K(hour_expected_len)); + } else if (OB_FAIL(ObDFMLimit::TIMEZONE_HOUR_ABS.validate(abs(tmp_tz_hour)))) { + LOG_WARN("not valid timezone hour", K(ret), K(tmp_tz_hour)); + } else if (OB_UNLIKELY(hour_expected_len != local_parsed_len)) { + ret = OB_INVALID_DATE_VALUE; // invalid time zone + } else if (FALSE_IT(local_ctx.update(hour_expected_len + 1))) { + } else if (OB_UNLIKELY(local_ctx.is_parse_finish())) { + // When format contains TZR, tz str is allowed to be hour, which equals to hour:00. + has_digit_tz_in_TZR = true; + tz_hour = tmp_tz_hour; + tz_min = tmp_tz_min; + } else if (OB_FAIL(ObDFMUtil::match_int_value(local_ctx, + parsed_elem_len - hour_expected_len - 1, + local_parsed_len, + tmp_tz_min, + local_tz_factor))) { + LOG_WARN("matching timezone hour failed, for 'TZR'. the error is ignored.", + K(ret), + K(hour_expected_len)); + } else if (OB_FAIL(ObDFMLimit::TIMEZONE_MIN_ABS.validate(abs(tmp_tz_min)))) { + LOG_WARN("not valid timezone hour", K(ret), K(tmp_tz_min)); + } else if (OB_UNLIKELY(parsed_elem_len != hour_expected_len + local_parsed_len + 1)) { + ret = OB_INVALID_DATE_VALUE; // invalid time zone + } else { + has_digit_tz_in_TZR = true; + tz_hour = tmp_tz_hour; + tz_min = tmp_tz_min; + } + } + } + } else { // case2: strings + ObString tzr_str; + parsed_elem_len = OB_MAX_TZ_NAME_LEN - 1; + if (OB_FAIL(ObDFMUtil::match_chars_until_space(ctx, tzr_str, parsed_elem_len))) { + LOG_WARN("failed to match tzr", K(ret)); + } else { + MEMCPY(ob_time.tz_name_, tzr_str.ptr(), tzr_str.length()); + ob_time.tz_name_[tzr_str.length()] = '\0'; + ob_time.is_tz_name_valid_ = true; + } + } + } + break; + } + + case ObDFMFlag::TZD: { + if (OB_UNLIKELY(elem_flags.has_member(ObDFMFlag::TZH) || elem_flags.has_member(ObDFMFlag::TZM))) { + ret = OB_ERR_NOT_A_VALID_TIME_ZONE; + LOG_WARN("tzh tzm and tzr tzd can not appear at the same time", K(ret)); + } else if (OB_UNLIKELY(has_digit_tz_in_TZR)) { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("digit TZR with TZD is invalid", K(ret)); + } else { + ObString tzd_str; + parsed_elem_len = OB_MAX_TZ_ABBR_LEN - 1; + if (OB_FAIL(ObDFMUtil::match_chars_until_space(ctx, tzd_str, parsed_elem_len))) { + LOG_WARN("failed to match tzd", K(ret)); + } else { + MEMCPY(ob_time.tzd_abbr_, tzd_str.ptr(), tzd_str.length()); + ob_time.tzd_abbr_[tzd_str.length()] = '\0'; + // ob_time.is_tz_name_valid_ = true; + } + } + break; + } + + case ObDFMFlag::J: { + const int32_t base_julian_day = 2378497; // julian day of 1800-01-01 + const int32_t base_date = -62091; // ob_time.parts_[DT_DATE] of 1800-01-01 + int32_t julian_day = 0; + int32_t ob_time_date = 0; + ObTime tmp_ob_time; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, julian_day))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::JULIAN_DATE.validate(julian_day))) { + LOG_WARN("julian_day must between 1 and 5373484", K(ret), K(julian_day)); + } else if (julian_day < base_julian_day) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("julian day of date before 1800-01-01 not supported", K(ret)); + } else if (FALSE_IT(ob_time_date = julian_day - base_julian_day + base_date)) { + } else if (OB_FAIL(date_to_ob_time(ob_time_date, tmp_ob_time))) { + LOG_WARN("date to ob time failed", K(ret)); + } else if (OB_FAIL(set_ob_time_year_may_conflict(ob_time, + julian_year_value, + tmp_ob_time.parts_[DT_YEAR], + tmp_ob_time.parts_[DT_YEAR], + true /* overwrite */))) { + LOG_WARN("set ob_time_year conflict", K(ret)); + } else { + yday_temp_value = tmp_ob_time.parts_[DT_YDAY]; + } + break; + } + + case ObDFMFlag::HH: + case ObDFMFlag::HH12: { + int32_t hour = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, hour))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::HOUR12.validate(hour))) { + LOG_WARN("hour must be between 1 and 12", K(ret), K(hour)); + } else if (!ObDFMUtil::elem_has_meridian_indicator(elem_flags)) { + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_HOUR, hour); + } else { + hh12_temp_value = hour; + } + break; + } + case ObDFMFlag::HH24: { + int32_t hour = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, hour))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::HOUR24.validate(hour))) { + LOG_WARN("hour must be between 0 and 23", K(ret), K(hour)); + } else if (OB_UNLIKELY(ObDFMUtil::elem_has_meridian_indicator(elem_flags))) { + ret = OB_INVALID_MERIDIAN_INDICATOR_USE; + LOG_WARN("HH24 appears with meridian indicator", K(ret), K(format)); + } else { + // may conflict with SSSSS + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_HOUR, hour); + } + break; + } + case ObDFMFlag::MI: { + int32_t min = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, min))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::MINUTE.validate(min))) { + LOG_WARN("minutes must be between 0 and 59", K(ret), K(min)); + } else { + // may conflict with SSSSS + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_MIN, min); + } + break; + } + case ObDFMFlag::MM: { + int32_t mon = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, mon))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::MONTH_DAY.validate(mon))) { + LOG_WARN("not a valid month", K(ret), K(mon)); + } else { + // may conflict with DDD + ret = set_ob_time_part_directly(ob_time, conflict_part_bitset, DT_MON, mon); + } + break; + } + case ObDFMFlag::MON: + case ObDFMFlag::MONTH: { + int32_t mon = 0; + for (mon = ObDFMLimit::MONTH.min_val_; mon <= ObDFMLimit::MONTH.max_val_; ++mon) { + const ObTimeConstStr& mon_str = + (elem.elem_flag_ == ObDFMFlag::MONTH) ? MON_NAMES[mon] : MON_ABBR_NAMES[mon]; + if (ObDFMUtil::match_pattern_ignore_case(ctx, mon_str)) { + parsed_elem_len = mon_str.len_; + break; + } + } + if (OB_FAIL(ObDFMLimit::MONTH.validate(mon))) { + LOG_WARN("not a valid month", K(ret), K(mon)); + } else { + // may conflict with DDD + ret = set_ob_time_part_directly(ob_time, conflict_part_bitset, DT_MON, mon); + } + break; + } + case ObDFMFlag::AM: + case ObDFMFlag::PM: + case ObDFMFlag::AM2: + case ObDFMFlag::PM2: { + bool is_with_dot = (ObDFMFlag::AM2 == elem.elem_flag_ || ObDFMFlag::PM2 == elem.elem_flag_); + parsed_elem_len = + is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::AM2].len_ : ObDFMFlag::PATTERN[ObDFMFlag::AM].len_; + bool is_am = ObDFMUtil::match_pattern_ignore_case( + ctx, is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::AM2] : ObDFMFlag::PATTERN[ObDFMFlag::AM]); + bool is_pm = ObDFMUtil::match_pattern_ignore_case( + ctx, is_with_dot ? ObDFMFlag::PATTERN[ObDFMFlag::PM2] : ObDFMFlag::PATTERN[ObDFMFlag::PM]); + if (OB_UNLIKELY(!is_am && !is_pm)) { + ret = OB_ERR_AM_OR_PM_REQUIRED; + LOG_WARN("AM/A.M. or PM/P.M. required", K(ret)); + } else { + is_after_noon = is_pm; + } + break; + } + case ObDFMFlag::RR: + case ObDFMFlag::RRRR: { + int32_t round_year = 0; + int32_t conflict_check_year = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, 4, parsed_elem_len, round_year))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (parsed_elem_len > 2) { + conflict_check_year = round_year; + // do nothing + } else { + conflict_check_year = round_year; + int32_t first_two_digits_of_current_year = (ob_time.parts_[DT_YEAR] / 100) % 100; + int32_t last_two_digits_of_current_year = ob_time.parts_[DT_YEAR] % 100; + if (round_year < 50) { // 0~49 + if (last_two_digits_of_current_year < 50) { + round_year += first_two_digits_of_current_year * 100; + } else { + round_year += (first_two_digits_of_current_year + 1) * 100; + } + } else if (round_year < 100) { // 50~99 + if (last_two_digits_of_current_year < 50) { + round_year += (first_two_digits_of_current_year - 1) * 100; + } else { + round_year += first_two_digits_of_current_year * 100; + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObDFMLimit::YEAR.validate(round_year))) { // TODO : negetive year number + LOG_WARN("(full) year must be between -4713 and +9999, and not be 0", K(ret), K(round_year)); + } else if (OB_FAIL(set_ob_time_year_may_conflict(ob_time, + julian_year_value, + conflict_check_year, + round_year, + false /* overwrite */))) { + LOG_WARN("set ob_time_year conflict", K(ret)); + } + } + break; + } + case ObDFMFlag::SS: { + int32_t sec = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, sec))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::SECOND.validate(sec))) { + LOG_WARN("seconds must be between 0 and 59", K(ret), K(sec)); + } else { + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_SEC, sec); + } + break; + } + case ObDFMFlag::SSSSS: { + int32_t sec_past_midnight = 0; + if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, sec_past_midnight))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::SECS_PAST_MIDNIGHT.validate(sec_past_midnight))) { + LOG_WARN("seconds in day must be between 0 and 86399", K(ret), K(sec_past_midnight)); + } else { + int32_t secs = sec_past_midnight % static_cast(SECS_PER_MIN); + int32_t mins = + (sec_past_midnight / static_cast(SECS_PER_MIN)) % static_cast(MINS_PER_HOUR); + int32_t hours = + sec_past_midnight / static_cast(SECS_PER_MIN) / static_cast(MINS_PER_HOUR); + if (OB_FAIL(set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_SEC, secs))) { + LOG_WARN("set ob time conflict", K(ret), K(secs), K(ob_time)); + } else if (OB_FAIL(set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_MIN, mins))) { + LOG_WARN("set ob time conflict", K(ret), K(mins), K(ob_time)); + } else if (OB_FAIL(set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_HOUR, hours))) { + LOG_WARN("set ob time conflict", K(ret), K(hours), K(ob_time)); + } + } + break; + } + case ObDFMFlag::YGYYY: { + int32_t years = 0; + if (OB_FAIL(ObDFMUtil::match_int_value_with_comma(ctx, expected_elem_len, parsed_elem_len, years))) { + LOG_WARN("failed to match int value", K(ret)); + } else if (OB_FAIL(ObDFMLimit::YEAR.validate(years))) { + LOG_WARN("(full) year must be between -4713 and +9999, and not be 0", K(ret), K(years)); + } else if (OB_FAIL(set_ob_time_year_may_conflict(ob_time, + julian_year_value, + years, + years, + false /* overwrite */))) { + LOG_WARN("set ob_time_year conflict", K(ret)); + } + break; + } + case ObDFMFlag::SYYYY: + case ObDFMFlag::YYYY: + case ObDFMFlag::YYY: + case ObDFMFlag::YY: + case ObDFMFlag::Y: { + int32_t years = 0; + int32_t conflict_check_year = 0; + int32_t sign = 1; + if (ObDFMFlag::SYYYY == elem.elem_flag_) { + if (ObDFMUtil::is_sign_char(ctx.cur_ch_[0])) { + sign = ('-' == ctx.cur_ch_[0] ? -1 : 1); + ctx.update(1); + } else if (part_blank1_len + part_sep_len > format_sep_len && + ObDFMUtil::is_sign_char(ctx.cur_ch_[-1])) { + sign = ('-' == ctx.cur_ch_[-1] ? -1 : 1); + } + } + if (OB_UNLIKELY(!ctx.is_valid())) { + } else if (OB_FAIL(ObDFMUtil::match_int_value(ctx, expected_elem_len, parsed_elem_len, years, sign))) { + LOG_WARN("failed to match int value", K(ret)); + } + if (OB_SUCC(ret)) { + conflict_check_year = years; + if (expected_elem_len < 4) { + years += (ob_time.parts_[DT_YEAR] / static_cast(power_of_10[parsed_elem_len])) * + static_cast(power_of_10[parsed_elem_len]); + } + if (OB_FAIL(ObDFMLimit::YEAR.validate(years))) { + LOG_WARN("(full) year must be between -4713 and +9999, and not be 0", K(ret), K(years)); + } else if (OB_FAIL(set_ob_time_year_may_conflict(ob_time, + julian_year_value, + conflict_check_year, + years, + false /* overwrite */))) { + LOG_WARN("set ob_time_year conflict", K(ret)); + } + } + break; + } + case ObDFMFlag::X: { + if (OB_UNLIKELY('.' != ctx.cur_ch_[0])) { + ignore_fs_flag = true; + // revert equal to 'X' not exist + ctx.revert(part_blank1_len + part_sep_len + part_blank2_len); + } else { + parsed_elem_len = 1; + } + break; + } + + case ObDFMFlag::CC: + case ObDFMFlag::SCC: + case ObDFMFlag::IW: + case ObDFMFlag::W: + case ObDFMFlag::WW: + case ObDFMFlag::YEAR: + case ObDFMFlag::SYEAR: + case ObDFMFlag::Q: + case ObDFMFlag::I: + case ObDFMFlag::IY: + case ObDFMFlag::IYY: + case ObDFMFlag::IYYY: { + ret = OB_ERR_FORMAT_CODE_CANNOT_APPEAR; + LOG_WARN("element can not appear", K(ret), K(elem)); + break; + } + default: { + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("unsupport element", K(ret), K(elem)); + break; + } + } // end switch + + if (OB_SUCC(ret)) { + ctx.update(parsed_elem_len); + } + } // end if + + if (OB_FAIL(ret)) { + LOG_WARN("failed to convert string to ob time by oracle dfm", K(ret), K(elem), K(ctx)); + } else { + LOG_DEBUG("DFM DEBUG: finish element", K(elem), K(ctx)); + } + } // end for + + // check if the unprocessed elems has permission to be omitted + if (OB_SUCC(ret)) { + for (; elem_idx < dfm_elems.count(); ++elem_idx) { + if (OB_UNLIKELY(!ObDFMUtil::is_element_can_omit(dfm_elems[elem_idx]))) { + ret = OB_ERR_INPUT_VALUE_NOT_LONG_ENOUGH; + LOG_WARN("input value not long enough for date format", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + // all elems has finished, is there anything else in str? the rest must be separators, do check. + int64_t str_remain_sep_len = 0; + while (str_remain_sep_len < ctx.remain_len_ && ObDFMUtil::is_split_char(ctx.cur_ch_[str_remain_sep_len])) { + str_remain_sep_len++; + } + ctx.update(str_remain_sep_len); + if (ctx.remain_len_ > 0) { + ret = OB_INVALID_DATE_FORMAT_END; + LOG_WARN("input value has not finished yet", K(ctx.remain_len_), K(format), K(str), K(ret)); + } + } + + // after noon conflict: AM PM vs HH12 HH + if (OB_SUCC(ret)) { + if (hh12_temp_value != ZERO_TIME) { + // when hour value, varied by meridian indicators + // if HH12 = 12, when meridian indicator 'AM' exists, the real time is hour = 0 + // if HH12 = 12, when meridian indicator 'PM' exists, the real time is hour = 12 + if (is_after_noon) { + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_HOUR, hh12_temp_value % 12 + 12); + } else { + ret = set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_HOUR, hh12_temp_value % 12); + } + } + } + + // before christ conflict: BC AD vs YEAR //TODO : change this when ob_time support negetive years + if (OB_SUCC(ret)) { + if (is_before_christ) { + ret = OB_ERR_INVALID_YEAR_VALUE; + LOG_WARN("before christ is not supported now!", K(ret)); + } + } + + // year cannot changed after this line + // feed/validate yday + YEAR to MON and DAY + if (OB_SUCC(ret)) { + if (yday_temp_value != ZERO_DATE) { + int32_t month = 0; + int32_t day = 0; + if (OB_FAIL(get_day_and_month_from_year_day(yday_temp_value, ob_time.parts_[DT_YEAR], month, day))) { + LOG_WARN("failed to get day and month from year day", K(ret), K(yday_temp_value), K(ob_time)); + } else if (OB_FAIL(set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_MON, month))) { + LOG_WARN("failed to set ob time part with conflict", K(ret), K(month), K(ob_time)); + } else if (OB_FAIL(set_ob_time_part_may_conflict(ob_time, conflict_part_bitset, DT_MDAY, day))) { + LOG_WARN("failed to set ob time part with conflict", K(ret), K(day), K(ob_time)); + } + } + } + + // calc and validate: YDAY WDAY vs YEAR MON DAY + if (OB_SUCC(ret)) { + if (OB_FAIL(validate_oracle_date(ob_time))) { + LOG_WARN("date is invalid or out of range", K(ret), K(str)); + } else { + // ob_time_to_date func is to calc YDAY and WDAY and return DATE + date_temp_value = ob_time_to_date(ob_time); // TODO: shanting + if (yday_temp_value != ZERO_DATE && OB_UNLIKELY(ob_time.parts_[DT_YDAY] != yday_temp_value)) { + ret = OB_ERR_DAY_OF_YEAR_CONFLICTS_WITH_JULIAN_DATE; + } else if (wday_temp_value != ZERO_DATE && OB_UNLIKELY(ob_time.parts_[DT_WDAY] != wday_temp_value)) { + ret = OB_ERR_DAY_OF_WEEK_CONFLICTS_WITH_JULIAN_DATE; + } else { + ob_time.parts_[DT_DATE] = date_temp_value; + } + } + } + + // for time zone info + if (OB_SUCC(ret) && ob_is_timestamp_tz(target_type)) { + if (ob_time.is_tz_name_valid_) { // A. timezone defined by names + if (ob_time.get_tz_name_str().empty()) { + // string not contains TZR, use tz_name of sessiontimezone as default value. + int64_t pos = 0; + if (OB_FAIL(cvrt_ctx.tz_info_->timezone_to_str(ob_time.tz_name_, OB_MAX_TZ_NAME_LEN, pos))) { + LOG_WARN("print tz name failed", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(calc_tz_offset_by_tz_name(cvrt_ctx, ob_time))) { + LOG_WARN("calc timezone offset failed", K(ret)); + } + } else { // B. timezone defined by time zone hour and minute + int32_t tz_offset_value = 0; + + if (elem_flags.has_member(ObDFMFlag::TZH)) { + bool has_tzh_value = (temp_tzh_value >= 0); + bool has_tzm_value = (temp_tzm_value >= 0); + if (OB_UNLIKELY(!has_tzh_value && has_tzm_value)) { + ret = OB_INVALID_DATE_VALUE; + LOG_WARN("only TZM match, TZH is not found", K(ret)); + } else if (!has_tzh_value && !has_tzm_value) { + // do nothing + } else if (has_tzh_value && !has_tzm_value) { + tz_hour = temp_tz_factor * temp_tzh_value; + tz_min = temp_tz_factor * abs(tz_min); + } else { + tz_hour = temp_tz_factor * temp_tzh_value; + tz_min = temp_tz_factor * temp_tzm_value; + } + } else if (elem_flags.has_member(ObDFMFlag::TZR)) { + // do nothing + } else { + // no time zone info in elem_flags + } + + // calc offset + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(tz_hour * tz_min < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_hour and tz_min cantains counter arithmetic symbols", K(ret)); + } else { + tz_offset_value = static_cast(tz_hour * MINS_PER_HOUR + tz_min); + } + } + + // final validate + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(!ObOTimestampData::is_valid_offset_min(tz_offset_value))) { + ret = OB_INVALID_DATE_VALUE; + LOG_WARN("validate timezone offset failed", K(ret), K(tz_offset_value)); + } else { + ob_time.parts_[DT_OFFSET_MIN] = tz_offset_value; + } + } + } + } + + if (OB_FAIL(ret)) { + LOG_WARN("failed to convert string to ob_time", + K(format), + K(ob_time), + K(conflict_part_bitset), + K(yday_temp_value), + K(wday_temp_value), + K(date_temp_value), + K(hh12_temp_value), + K(tz_hour), + K(tz_min), + K(is_after_noon), + K(is_before_christ)); + } else { + LOG_DEBUG("convert from string to ob_time", + K(format), + K(ob_time), + K(conflict_part_bitset), + K(yday_temp_value), + K(wday_temp_value), + K(date_temp_value), + K(hh12_temp_value), + K(tz_hour), + K(tz_min), + K(is_after_noon), + K(is_before_christ)); + } + } // end if + } + + return ret; +} + + +int ObTimeConverter::usec_to_ob_time(int64_t usec, ObTime &ob_time) +{ + int ret = OB_SUCCESS; + int32_t days = static_cast(usec / USECS_PER_DAY); + usec %= USECS_PER_DAY; + if (OB_UNLIKELY(usec < 0)) { + --days; + usec += USECS_PER_DAY; + } + if (OB_FAIL(date_to_ob_time(days, ob_time))) { + LOG_WARN("failed to convert date part to obtime", K(ret), K(usec)); + } else if (OB_FAIL(time_to_ob_time(usec, ob_time))) { + LOG_WARN("failed to convert time part to obtime", K(ret), K(usec)); + } + return ret; +} + +int ObTimeConverter::datetime_to_ob_time(int64_t value, const ObTimeZoneInfo *tz_info, ObTime &ob_time) +{ + int ret = OB_SUCCESS; + int64_t usec = value; + if (OB_UNLIKELY(ZERO_DATETIME == usec)) { + MEMSET(ob_time.parts_, 0, sizeof(*ob_time.parts_) * TOTAL_PART_CNT); + ob_time.parts_[DT_DATE] = ZERO_DATE; + } else if (OB_FAIL(add_timezone_offset(tz_info, usec))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } else { + int32_t days = static_cast(usec / USECS_PER_DAY); + usec %= USECS_PER_DAY; + if (OB_UNLIKELY(usec < 0)) { + --days; + usec += USECS_PER_DAY; + } + if (OB_FAIL(date_to_ob_time(days, ob_time))) { + LOG_WARN("failed to convert date part to obtime", K(ret), K(usec)); + } else if (OB_FAIL(time_to_ob_time(usec, ob_time))) { + LOG_WARN("failed to convert time part to obtime", K(ret), K(usec)); + } + } + return ret; +} + +int ObTimeConverter::otimestamp_to_ob_time(const ObObjType type, + const ObOTimestampData &ot_data, + const ObTimeZoneInfo *tz_info, + ObTime &ob_time, + const bool store_utc_time /*true*/) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ob_is_otimestamp_type(type))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("it is not otimestamp type", K(type), K(ret)); + } else if (ot_data.is_null_value()) { + // NOTE: Any arithmetic expression containing a null always evaluates to null. + ret = OB_ERR_UNEXPECTED; + LOG_WARN("it is null otimestamp, should not arrive here", K(type), K(ot_data), K(ret)); + } else if (ObTimestampTZType == type) { + int32_t offset_min = 0; + int64_t usec = ot_data.time_us_; + if (OB_FAIL(extract_offset_from_otimestamp(ot_data, tz_info, offset_min, ob_time))) { + LOG_WARN("failed to extract_offset_from_otimestamp", K(ret)); + } else { + usec += (store_utc_time ? 0 : offset_min * SECS_PER_MIN * USECS_PER_SEC); + int64_t nsec = 0; + if (OB_FAIL(usec_to_ob_time(usec, ob_time))) { + LOG_WARN("failed to convert usec part to obtime", K(ret), K(usec)); + } else if (OB_UNLIKELY( + (nsec = ob_time.parts_[DT_USEC] * NSECS_PER_USEC + ot_data.time_ctx_.tail_nsec_) > INT32_MAX)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("nsec is overflow", K(nsec), K(ret)); + } else { + ob_time.parts_[DT_USEC] = static_cast(nsec); + ob_time.mode_ |= DT_TYPE_ORACLE; + ob_time.mode_ |= DT_TYPE_TIMEZONE; + if (store_utc_time) { + ob_time.mode_ |= DT_TYPE_STORE_UTC; + } else { + ob_time.mode_ &= ~(DT_TYPE_STORE_UTC); + } + } + } + } else { + int64_t usec = ot_data.time_us_; + int64_t nsec = 0; + if (ObTimestampLTZType == type && !store_utc_time) { + if (OB_FAIL(add_timezone_offset(tz_info, usec))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(usec_to_ob_time(usec, ob_time))) { + LOG_WARN("failed to convert usec part to obtime", K(ret), K(usec)); + } else if (OB_UNLIKELY( + (nsec = ob_time.parts_[DT_USEC] * NSECS_PER_USEC + ot_data.time_ctx_.tail_nsec_) > INT32_MAX)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("nsec is overflow", K(nsec), K(ret)); + } else { + ob_time.parts_[DT_USEC] = static_cast(nsec); + ob_time.mode_ |= DT_TYPE_ORACLE; + if (ObTimestampNanoType == type || !store_utc_time) { + ob_time.mode_ &= ~(DT_TYPE_STORE_UTC); + } else { + ob_time.mode_ |= DT_TYPE_STORE_UTC; + } + } + } + + if (OB_SUCC(ret)) { + LOG_DEBUG("succ to otimestamp_to_ob_time", K(ret), K(ot_data), K(type), K(ob_time), K(lbt())); + } + + return ret; +} + +int ObTimeConverter::date_to_ob_time(int32_t value, ObTime &ob_time) +{ + int ret = OB_SUCCESS; + int32_t *parts = ob_time.parts_; + if (OB_UNLIKELY(ZERO_DATE == value)) { + memset(parts, 0, sizeof(*parts) * DATETIME_PART_CNT); + parts[DT_DATE] = ZERO_DATE; + } else { + int32_t days = value; + int32_t leap_year = 0; + int32_t year = EPOCH_YEAR4; + parts[DT_DATE] = value; + // year. + while (days < 0 || days >= DAYS_PER_YEAR[leap_year = IS_LEAP_YEAR(year)]) { + int32_t new_year = year + days / DAYS_PER_NYEAR; + new_year -= (days < 0); + days -= (new_year - year) * DAYS_PER_NYEAR + LEAP_YEAR_COUNT(new_year - 1) - LEAP_YEAR_COUNT(year - 1); + year = new_year; + } + parts[DT_YEAR] = year; + parts[DT_YDAY] = days + 1; + parts[DT_WDAY] = WDAY_OFFSET[value % DAYS_PER_WEEK][EPOCH_WDAY]; + // month. + const int32_t *cur_days_until_mon = DAYS_UNTIL_MON[leap_year]; + int32_t month = 1; + for (; month < MONS_PER_YEAR && days >= cur_days_until_mon[month]; ++month) {} + parts[DT_MON] = month; + days -= cur_days_until_mon[month - 1]; + // day. + parts[DT_MDAY] = days + 1; + } + return ret; +} + +int ObTimeConverter::time_to_ob_time(int64_t value, ObTime &ob_time) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(value < 0)) { + ob_time.mode_ |= DT_MODE_NEG; + value = -value; + } + int64_t secs = USEC_TO_SEC(value); + ob_time.parts_[DT_HOUR] = static_cast(secs / SECS_PER_HOUR); + secs %= SECS_PER_HOUR; + ob_time.parts_[DT_MIN] = static_cast(secs / SECS_PER_MIN); + ob_time.parts_[DT_SEC] = static_cast(secs % SECS_PER_MIN); + ob_time.parts_[DT_USEC] = static_cast(value % USECS_PER_SEC); + return ret; +} + +//////////////////////////////// +// int / uint / string <- ObTime -> datetime / date / time. + +#define DTAE_DIGIT_LEN 8 +#define TIME_DIGIT_LEN 6 + +OB_INLINE int64_t ObTimeConverter::ob_time_to_int(const ObTime &ob_time, ObDTMode mode) +{ + int64_t value = 0; + if (DT_TYPE_DATE & mode) { + value = ob_time.parts_[DT_YEAR] * static_cast(power_of_10[4]) + + ob_time.parts_[DT_MON] * static_cast(power_of_10[2]) + + ob_time.parts_[DT_MDAY]; + } + if (DT_TYPE_TIME & mode) { + value *= static_cast(power_of_10[6]); + value += (ob_time.parts_[DT_HOUR] * static_cast(power_of_10[4]) + + ob_time.parts_[DT_MIN] * static_cast(power_of_10[2]) + + ob_time.parts_[DT_SEC]); + } + return value; +} + +static const int32_t DT_PART_MUL[DATETIME_PART_CNT] = {100, 100, 100, 100, 100, 1000000, 1}; +static const int64_t ZERO_DATE_WEEK = 613566757; // for 0000-00-00 00:00:00 . +int64_t ObTimeConverter::ob_time_to_int_extract(const ObTime &ob_time, ObDateUnitType unit_type) +{ + int64_t value = 0; + switch (unit_type) { + case DATE_UNIT_WEEK: { + value = ZERO_DATE == ob_time.parts_[DT_DATE] + ? ZERO_DATE_WEEK + : ObTimeConverter::ob_time_to_week(ob_time, DT_WEEK_SUN_BEGIN | DT_WEEK_ZERO_BEGIN); break; } case DATE_UNIT_QUARTER: { @@ -1580,322 +3050,921 @@ int64_t ObTimeConverter::ob_time_to_int_extract(const ObTime &ob_time, ObDateUni if (DT_MODE_NEG & ob_time.mode_) { value = - value; } - break; + break; + } + } + return value; +} + +#define DATE_FMT_WITH_DELIM "%04d-%02d-%02d" +#define DATE_FMT_NO_DELIM "%04d%02d%02d" +#define TIME_FMT_WITH_DELIM "%02d:%02d:%02d" +#define TIME_FMT_NO_DELIM "%02d%02d%02d" +static const char *USEC_FMT[7] = { + ".%d", + ".%01d", + ".%02d", + ".%03d", + ".%04d", + ".%05d", + ".%06d" +}; + +int ObTimeConverter::ob_time_to_str(const ObTime &ob_time, ObDTMode mode, int16_t scale, + char *buf, int64_t buf_len, int64_t &pos, bool with_delim) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!(0 < mode && mode <= DT_TYPE_CNT && scale <= 6 + && NULL != buf && buf_len > 0 && pos >= 0))) { + ret = OB_INVALID_ARGUMENT; + } else { + const int32_t *parts = ob_time.parts_; + const char *FMT = NULL; + if (HAS_TYPE_DATE(mode)) { + FMT = with_delim ? DATE_FMT_WITH_DELIM : DATE_FMT_NO_DELIM; + ret = databuff_printf(buf, buf_len, pos, FMT, parts[DT_YEAR], parts[DT_MON], parts[DT_MDAY]); + } + if (OB_SUCC(ret)) { + if (IS_TYPE_DATETIME(mode) && with_delim) { + ret = databuff_printf(buf, buf_len, pos, " "); + } else if (IS_TYPE_TIME(mode) && IS_NEG_TIME(ob_time.mode_)) { + ret = databuff_printf(buf, buf_len, pos, "-"); + } + } + if (OB_SUCC(ret) && HAS_TYPE_TIME(mode)) { + FMT = with_delim ? TIME_FMT_WITH_DELIM : TIME_FMT_NO_DELIM; + ret = databuff_printf(buf, buf_len, pos, FMT, parts[DT_HOUR], parts[DT_MIN], parts[DT_SEC]); + if (scale < 0 ) { + scale = parts[DT_USEC] > 0 ? 6 : 0; + } + if (OB_SUCC(ret) && scale > 0) { + ret = databuff_printf(buf, buf_len, pos, USEC_FMT[scale], parts[DT_USEC] / power_of_10[6 - scale]); + } + } + } + if (OB_FAIL(ret)) { + LOG_WARN("failed to snprintf datetime string", K(ret)); + } + return ret; +} + +int ObTimeConverter::data_fmt_nd(char *buffer, int64_t buf_len, int64_t &pos, const int64_t n, int64_t target) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(n <= 0 || target < 0 || target > 999999)) { + ret = OB_ERR_UNEXPECTED; + LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(n), K(target)); + } else if (OB_UNLIKELY(n > buf_len - pos)) { + ret = OB_SIZE_OVERFLOW; + LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(n), K(buf_len), K(pos)); + } else { + int64_t idx = pos + n - 1; + int64_t i = 0; + //BTW, when loop size is small, maybe, we can use loop unrolling to get better performance. + while (i < n) { + buffer[idx--] = static_cast(target % 10 + '0'); + target /= 10; + ++i; + } + pos += i; + } + return ret; +} + +int ObTimeConverter::data_fmt_d(char *buffer, int64_t buf_len, int64_t &pos, int64_t target) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(target < 0 || target >= 100)) { + ret = OB_ERR_UNEXPECTED; + LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(target)); + } else { + //buffer_size_need will be 1 or 2 + int64_t buffer_size_need = 1 + (target >= 10); + if (OB_UNLIKELY(buffer_size_need > buf_len - pos)) { + ret = OB_SIZE_OVERFLOW; + LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(buffer_size_need), K(buf_len), K(pos)); + } else { + int64_t idx = pos + buffer_size_need - 1; + int64_t i = 0; + //BTW, when loop size is small, maybe, we can use loop unrolling to get better performance. + while (i < buffer_size_need) { + buffer[idx--] = static_cast(target % 10 + '0'); + target /= 10; + ++i; + } + pos += i; + } + } + return ret; +} + +int ObTimeConverter::data_fmt_s(char *buffer, int64_t buf_len, int64_t &pos, const char *ptr) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ptr)) { + ret = OB_ERR_UNEXPECTED; + LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(ptr)); + } else { + int64_t buffer_size_need = strlen(ptr); + if (OB_UNLIKELY(buffer_size_need > buf_len - pos)) { + ret = OB_SIZE_OVERFLOW; + LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(buffer_size_need), K(buf_len), K(pos)); + } else { + //will not copy the '\0' in ptr + MEMCPY(buffer + pos, ptr, buffer_size_need); + pos += buffer_size_need; + } + } + return ret; +} + +int ObTimeConverter::get_day_and_month_from_year_day(const int32_t yday, + const int32_t year, + int32_t &month, + int32_t &day) +{ + int ret = OB_SUCCESS; + int32_t leap_year = IS_LEAP_YEAR(year); + if (OB_UNLIKELY(yday > DAYS_UNTIL_MON[leap_year][12])) { + ret = OB_ERR_INVALID_DAY_OF_YEAR_VALUE; + } else { + bool stop_flag = false; + for (int32_t i = ObDFMLimit::MONTH.min_val_; !stop_flag && i <= ObDFMLimit::MONTH.max_val_; ++i) { + if (yday <= DAYS_UNTIL_MON[leap_year][i]) { + month = i; + day = yday - DAYS_UNTIL_MON[leap_year][i - 1]; + stop_flag = true; + } + } + } + return ret; +} + +int ObTimeConverter::set_ob_time_year_may_conflict(ObTime &ob_time, + int32_t &julian_year_value, + int32_t check_year, + int32_t set_year, + bool overwrite) +{ + int ret = OB_SUCCESS; + if (ZERO_DATE != julian_year_value) { + if (julian_year_value != check_year) { + ret = OB_ERR_YEAR_CONFLICTS_WITH_JULIAN_DATE; + LOG_WARN("year conflicts with Julian date", K(ret), K(julian_year_value), K(check_year)); + } else if (overwrite) { + ob_time.parts_[DT_YEAR] = set_year; + } + } else { + ob_time.parts_[DT_YEAR] = set_year; + julian_year_value = check_year; + } + return ret; +} + +int ObTimeConverter::ob_time_to_str_format(const ObTime &ob_time, + const ObString &format, + char *buf, + int64_t buf_len, + int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(format.ptr()) + || OB_ISNULL(buf) + || OB_UNLIKELY(format.length() <= 0 + || buf_len <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("format or output string is invalid", K(ret), K(format), K(buf), K(buf_len)); + } else { + const char *format_ptr = format.ptr(); + const char *end_ptr = format.ptr() + format.length(); + const int32_t *parts = ob_time.parts_; + int32_t week_sunday = -1; + int32_t week_monday = -1; + int32_t delta_sunday = -2; + int32_t delta_monday = -2; + //used for am/pm conversation in order to avoid if-else tests. + const int hour_converter[24] = {12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; + while (format_ptr < end_ptr && OB_SUCCESS == ret) { + if ('%' == *format_ptr) { + format_ptr++; + if (format_ptr >= end_ptr) { + ret = OB_INVALID_ARGUMENT; + break; + } + switch (*format_ptr) { + /*The cases are not ordered alphabetically since Y y m d D are used frequently + *in order to get better performance, we locate them in front of others + */ + case 'Y': { //Year, numeric, four digits + ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR]); + break; + } + case 'y': { //Year, numeric (two digits) + int year = (ob_time.parts_[DT_YEAR]) % 100; + ret = data_fmt_nd(buf, buf_len, pos, 2, year); + break; + } + case 'M': { //Month name (January..December) + ret = data_fmt_s(buf, buf_len, pos, MON_NAMES[parts[DT_MON]].ptr_); + break; + } + case 'm': { //Month, numeric (00..12) + ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MON]); + break; + } + case 'D': { //Day of the month with English suffix (0th, 1st, 2nd, 3rd...) + ret = data_fmt_s(buf, buf_len, pos, DAY_NAME[parts[DT_MDAY]]); + break; + } + case 'd': { //Day of the month, numeric (00..31) + ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MDAY]); + break; + } + case 'a': { //Abbreviated weekday name (Sun..Sat) + ret = data_fmt_s(buf, buf_len, pos, WDAY_ABBR_NAMES[parts[DT_WDAY]].ptr_); + break; + } + case 'b': { //Abbreviated month name (Jan..Dec) + ret = data_fmt_s(buf, buf_len, pos, MON_ABBR_NAMES[parts[DT_MON]].ptr_); + break; + } + case 'c': { //Month, numeric (0..12) + ret = data_fmt_d(buf, buf_len, pos, parts[DT_MON]); + break; + } + case 'e': { //Day of the month, numeric (0..31) + ret = data_fmt_d(buf, buf_len, pos, parts[DT_MDAY]); + break; + } + case 'f': { //Microseconds (000000..999999) + ret = data_fmt_nd(buf, buf_len, pos, 6, parts[DT_USEC]); + break; + } + case 'H': { //Hour (00..23) + ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_HOUR]); + break; + } + case 'h': //Hour (01..12) + case 'I': { //Hour (01..12) + int hour = hour_converter[parts[DT_HOUR]]; + ret = data_fmt_nd(buf, buf_len, pos, 2, hour); + break; + } + case 'i': { //Minutes, numeric (00..59) + ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MIN]); + break; + } + case 'j': { //Day of year (001..366) + ret = data_fmt_nd(buf, buf_len, pos, 3, parts[DT_YDAY]); + break; + } + case 'k': { //Hour (0..23) + ret = data_fmt_d(buf, buf_len, pos, parts[DT_HOUR]); + break; + } + case 'l': { //Hour (1..12) + int hour = hour_converter[parts[DT_HOUR]]; + ret = data_fmt_d(buf, buf_len, pos, hour); + break; + } + case 'p': { //AM or PM + const char *ptr = parts[DT_HOUR] < 12 ? "AM" : "PM"; + ret = data_fmt_s(buf, buf_len, pos, ptr); + break; + } + case 'r': { //Time, 12-hour (hh:mm:ss followed by AM or PM) + int hour = hour_converter[parts[DT_HOUR]]; + const char *ptr = parts[DT_HOUR] < 12 ? "AM" : "PM"; + ret = databuff_printf(buf, buf_len, pos, "%02d:%02d:%02d %s", hour, parts[DT_MIN], parts[DT_SEC], ptr); + break; + } + case 'S': //Seconds (00..59) + case 's': { //Seconds (00..59) + ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_SEC]); + break; + } + case 'T': { //Time, 24-hour (hh:mm:ss) + ret = databuff_printf(buf, buf_len, pos, "%02d:%02d:%02d", parts[DT_HOUR], parts[DT_MIN], parts[DT_SEC]); + break; + } + case 'U': { //Week (00..53), where Sunday is the first day of the week + ret = data_fmt_nd(buf, buf_len, pos, 2, ob_time_to_week(ob_time, WEEK_MODE[0])); + break; + } + case 'u': { //Week (00..53), where Monday is the first day of the week + ret = data_fmt_nd(buf, buf_len, pos, 2, ob_time_to_week(ob_time, WEEK_MODE[1])); + break; + } + case 'V': { //Week (01..53), where Sunday is the first day of the week; used with %X + //due to the face that V is often used with X. + // In order to optimize the implementation, we set the right delta value which will possibly be used for %X case latter. + // week_sunday != -1 means that its value has been computed in %X case ever. + ret = data_fmt_nd(buf, buf_len, pos, 2, (-1 == week_sunday) ? ob_time_to_week(ob_time, WEEK_MODE[2], delta_sunday) : week_sunday); + break; + } + case 'v': { //Week (01..53), where Monday is the first day of the week; used with %x + ret = data_fmt_nd(buf, buf_len, pos, 2, (-1 == week_monday) ? ob_time_to_week(ob_time, WEEK_MODE[3], delta_monday) : week_monday); + break; + } + case 'W': { //Weekday name (Sunday..Saturday) + ret = data_fmt_s(buf, buf_len, pos, WDAY_NAMES[parts[DT_WDAY]].ptr_); + break; + } + case 'w': { //Day of the week (0=Sunday..6=Saturday) + ret = data_fmt_d(buf, buf_len, pos, parts[DT_WDAY] % DAYS_PER_WEEK); + break; + } + case 'X': { //Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + //due to the face that %X is often used with %V. + // In order to optimize the implementation, we set the right week_sunday value which will possibly be used for %V case latter. + if (-2 == delta_sunday) { + week_sunday = ob_time_to_week(ob_time, WEEK_MODE[2], delta_sunday); + } + ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR] + delta_sunday); + break; + } + case 'x': { //Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + if (-2 == delta_monday) { + week_monday = ob_time_to_week(ob_time, WEEK_MODE[3], delta_monday); + } + ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR] + delta_monday); + break; + } + case '%': { //A literal "%" character + if (pos >= buf_len) { + ret = OB_SIZE_OVERFLOW; + break; + } + buf[pos++] = '%'; + break; + } + default: { + ret = OB_NOT_SUPPORTED; + break; + } + } + if (OB_SUCC(ret)) { + format_ptr++; + } + } else if (pos >= buf_len) { + ret = OB_SIZE_OVERFLOW; + break; + } else { + buf[pos++] = *(format_ptr++); + } } } - return value; + return ret; } -#define DATE_FMT_WITH_DELIM "%04d-%02d-%02d" -#define DATE_FMT_NO_DELIM "%04d%02d%02d" -#define TIME_FMT_WITH_DELIM "%02d:%02d:%02d" -#define TIME_FMT_NO_DELIM "%02d%02d%02d" -static const char *USEC_FMT[7] = { - ".%d", - ".%01d", - ".%02d", - ".%03d", - ".%04d", - ".%05d", - ".%06d" -}; - -int ObTimeConverter::ob_time_to_str(const ObTime &ob_time, ObDTMode mode, int16_t scale, - char *buf, int64_t buf_len, int64_t &pos, bool with_delim) +int check_and_get_tz_info(ObTime &ob_time, + const ObTimeConvertCtx &cvrt_ctx, + const ObTimeZoneInfo *&tz_info, + ObTimeZoneInfoPos *&literal_tz_info, + ObTZInfoIDPosMap *&tz_id_pos_map) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!(0 < mode && mode <= DT_TYPE_CNT && scale <= 6 - && NULL != buf && buf_len > 0 && pos >= 0))) { - ret = OB_INVALID_ARGUMENT; - } else { - const int32_t *parts = ob_time.parts_; - const char *FMT = NULL; - if (HAS_TYPE_DATE(mode)) { - FMT = with_delim ? DATE_FMT_WITH_DELIM : DATE_FMT_NO_DELIM; - ret = databuff_printf(buf, buf_len, pos, FMT, parts[DT_YEAR], parts[DT_MON], parts[DT_MDAY]); - } - if (OB_SUCC(ret)) { - if (IS_TYPE_DATETIME(mode) && with_delim) { - ret = databuff_printf(buf, buf_len, pos, " "); - } else if (IS_TYPE_TIME(mode) && IS_NEG_TIME(ob_time.mode_)) { - ret = databuff_printf(buf, buf_len, pos, "-"); - } - } - if (OB_SUCC(ret) && HAS_TYPE_TIME(mode)) { - FMT = with_delim ? TIME_FMT_WITH_DELIM : TIME_FMT_NO_DELIM; - ret = databuff_printf(buf, buf_len, pos, FMT, parts[DT_HOUR], parts[DT_MIN], parts[DT_SEC]); - if (scale < 0 ) { - scale = parts[DT_USEC] > 0 ? 6 : 0; - } - if (OB_SUCC(ret) && scale > 0) { - ret = databuff_printf(buf, buf_len, pos, USEC_FMT[scale], parts[DT_USEC] / power_of_10[6 - scale]); + ObTZInfoMap *tz_info_map = NULL; + if (OB_UNLIKELY(ob_time.is_tz_name_valid_)) { // use string literal tz_inifo + // tz_info_ is expected to be not null, but sometimes we don't set it. + if (NULL == cvrt_ctx.tz_info_) { + if (HAS_TYPE_ORACLE(ob_time.mode_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tz_info_ is NULL", K(ret)); } + } else if (OB_ISNULL(tz_info_map = const_cast(cvrt_ctx.tz_info_->get_tz_info_map()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_info_map is NULL", K(ret)); + } else if (OB_FAIL(tz_info_map->get_tz_info_by_name(ob_time.get_tz_name_str(), literal_tz_info))) { + LOG_WARN("fail to get_tz_info_by_name", K(ob_time), K(ret)); + tz_info_map->id_map_.revert(literal_tz_info); + literal_tz_info = NULL; + } else { + literal_tz_info->set_error_on_overlap_time(cvrt_ctx.tz_info_->is_error_on_overlap_time()); + tz_info = literal_tz_info; + tz_id_pos_map = &(tz_info_map->id_map_); } - } - if (OB_FAIL(ret)) { - LOG_WARN("failed to snprintf datetime string", K(ret)); + } else { // use session tz_info + tz_info = cvrt_ctx.tz_info_; } return ret; } -int ObTimeConverter::data_fmt_nd(char *buffer, int64_t buf_len, int64_t &pos, const int64_t n, int64_t target) +/** + * @brief calcs tz offset value by time zone name from the input ob_time, fills the result back to ob_time + * @param in: cvrt_ctx + * @param in, out: ob_time + * @return + */ +int ObTimeConverter::calc_tz_offset_by_tz_name(const ObTimeConvertCtx &cvrt_ctx, ObTime &ob_time) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(n <= 0 || target < 0 || target > 999999)) { + int64_t usec = ob_time.parts_[DT_DATE] * USECS_PER_DAY + ob_time_to_time(ob_time); + const ObTimeZoneInfo* tz_info = NULL; + ObTimeZoneInfoPos* literal_tz_info = NULL; + ObTZInfoIDPosMap* tz_id_pos_map = NULL; + int32_t tz_id = OB_INVALID_INDEX; + int32_t tran_type_id = OB_INVALID_INDEX; + int32_t offset_min = 0; + if (OB_FAIL(check_and_get_tz_info(ob_time, cvrt_ctx, tz_info, literal_tz_info, tz_id_pos_map))) { + LOG_WARN("fail to check time zone info", K(ob_time)); + } else if (OB_ISNULL(tz_info)) { ret = OB_ERR_UNEXPECTED; - LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(n), K(target)); - } else if (OB_UNLIKELY(n > buf_len - pos)) { - ret = OB_SIZE_OVERFLOW; - LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(n), K(buf_len), K(pos)); + LOG_WARN("tz_info shoule not be null", K(ret)); + } else if (OB_FAIL(sub_timezone_offset(*tz_info, + ob_time.get_tzd_abbr_str(), + usec, + offset_min, + tz_id, + tran_type_id))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); } else { - int64_t idx = pos + n - 1; - int64_t i = 0; - //BTW, when loop size is small, maybe, we can use loop unrolling to get better performance. - while (i < n) { - buffer[idx--] = static_cast(target % 10 + '0'); - target /= 10; - ++i; - } - pos += i; + ob_time.parts_[DT_OFFSET_MIN] = offset_min; + ob_time.time_zone_id_ = tz_id; + ob_time.transition_type_id_ = tran_type_id; + } + + if (NULL != literal_tz_info && NULL != tz_id_pos_map) { + tz_id_pos_map->revert(literal_tz_info); + tz_id_pos_map = NULL; + literal_tz_info = NULL; } return ret; -} +} -int ObTimeConverter::data_fmt_d(char *buffer, int64_t buf_len, int64_t &pos, int64_t target) +int ObTimeConverter::ob_time_to_utc(const ObObjType obj_type, const ObTimeConvertCtx &cvrt_ctx, ObTime &ob_time) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(target < 0 || target >= 100)) { - ret = OB_ERR_UNEXPECTED; - LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(target)); + if (ObTimestampNanoType == obj_type) { + // ignore offset + } else if (HAS_TYPE_STORE_UTC(ob_time.mode_)) { + // has store utc time } else { - //buffer_size_need will be 1 or 2 - int64_t buffer_size_need = 1 + (target >= 10); - if (OB_UNLIKELY(buffer_size_need > buf_len - pos)) { - ret = OB_SIZE_OVERFLOW; - LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(buffer_size_need), K(buf_len), K(pos)); + int64_t usec = ob_time.parts_[DT_DATE] * USECS_PER_DAY + ob_time_to_time(ob_time); + const int64_t old_usec = usec; + if (HAS_TYPE_TIMEZONE(ob_time.mode_)) { + usec -= (ob_time.parts_[DT_OFFSET_MIN] * USECS_PER_MIN); } else { - int64_t idx = pos + buffer_size_need - 1; - int64_t i = 0; - //BTW, when loop size is small, maybe, we can use loop unrolling to get better performance. - while (i < buffer_size_need) { - buffer[idx--] = static_cast(target % 10 + '0'); - target /= 10; - ++i; + int32_t offset_min = 0; + int32_t tz_id = OB_INVALID_INDEX; + int32_t tran_type_id = OB_INVALID_INDEX; + if (OB_ISNULL(cvrt_ctx.tz_info_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("tz_info is null", K(ret)); + } else if (OB_FAIL(sub_timezone_offset(*cvrt_ctx.tz_info_, ObString(), usec, offset_min, tz_id, tran_type_id))) { + LOG_WARN("failed to adjust value with time zone offset", K(ret)); + } else { + ob_time.parts_[DT_OFFSET_MIN] = offset_min; + ob_time.time_zone_id_ = tz_id; + ob_time.transition_type_id_ = tran_type_id; + } + } + + if (OB_SUCC(ret)) { + if (old_usec == usec) { + // no need convert + } else { + const int32_t nanosecond = ob_time.parts_[DT_USEC]; + if (OB_FAIL(usec_to_ob_time(usec, ob_time))) { + LOG_WARN("failed to usec_to_ob_time", K(ret)); + } else { + ob_time.parts_[DT_USEC] = nanosecond; + ob_time.parts_[DT_DATE] = ob_time_to_date(ob_time); + } } - pos += i; + } + + if (OB_SUCC(ret)) { + ob_time.mode_ |= DT_TYPE_STORE_UTC; } } return ret; } -int ObTimeConverter::data_fmt_s(char *buffer, int64_t buf_len, int64_t &pos, const char *ptr) +// check parts of obtime before print timestamp in oracle mode +bool ObTimeConverter::valid_oracle_year(const ObTime &ob_time) { - int ret = OB_SUCCESS; - if (OB_ISNULL(ptr)) { - ret = OB_ERR_UNEXPECTED; - LIB_TIME_LOG(ERROR, "invalid argument", K(ret), K(ptr)); - } else { - int64_t buffer_size_need = strlen(ptr); - if (OB_UNLIKELY(buffer_size_need > buf_len - pos)) { - ret = OB_SIZE_OVERFLOW; - LIB_TIME_LOG(WARN, "no enough space for buffer", K(ret), K(buffer_size_need), K(buf_len), K(pos)); - } else { - //will not copy the '\0' in ptr - MEMCPY(buffer + pos, ptr, buffer_size_need); - pos += buffer_size_need; - } + int ret = true; + if (ob_time.parts_[DT_YEAR] < 0 || ob_time.parts_[DT_YEAR] > 9999) { + ret = false; } return ret; } -int ObTimeConverter::ob_time_to_str_format(const ObTime &ob_time, const ObString &format, - char *buf, int64_t buf_len, int64_t &pos) +void ObTimeConverter::calc_iso_week(bool &is_iso_week_calced, + int32_t &iso_week, + const ObTime &ob_time, + ObDTMode mode, + int32_t &delta) +{ + if (!is_iso_week_calced) { + iso_week = ob_time_to_week(ob_time, mode, delta); + is_iso_week_calced = true; + } + return; +} + +int ObTimeConverter::ob_time_to_str_oracle_dfm(const ObTime &ob_time, + ObScale scale, + const ObString &format, + char *buf, + int64_t buf_len, + int64_t &pos) { int ret = OB_SUCCESS; - if (OB_ISNULL(format.ptr()) || OB_ISNULL(buf) || OB_UNLIKELY(format.length() <= 0 || buf_len <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("format or output string is invalid", K(ret), K(format), K(buf), K(buf_len)); + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0) || OB_UNLIKELY(format.empty()) || + OB_UNLIKELY(scale > MAX_SCALE_FOR_ORACLE_TEMPORAL)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(format), K(buf), K(buf_len), K(ob_time), K(scale)); + } else if (!valid_oracle_year(ob_time)) { + ret = OB_ERR_DATETIME_INTERVAL_INTERNAL_ERROR; + LOG_WARN("invalid oracle timestamp", K(ret), K(ob_time)); } else { - const char *format_ptr = format.ptr(); - const char *end_ptr = format.ptr() + format.length(); - const int32_t *parts = ob_time.parts_; - int32_t week_sunday = -1; - int32_t week_monday = -1; - int32_t delta_sunday = -2; - int32_t delta_monday = -2; - //used for am/pm conversation in order to avoid if-else tests. - const int hour_converter[24] = {12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; - while (format_ptr < end_ptr && OB_SUCCESS == ret) { - if ('%' == *format_ptr) { - format_ptr++; - if (format_ptr >= end_ptr) { - ret = OB_INVALID_ARGUMENT; - break; + if (scale < 0) { + scale = DEFAULT_SCALE_FOR_ORACLE_FRACTIONAL_SECONDS; + } + + int32_t iso_week = 0; + int32_t iso_year_delta = 0; + bool is_iso_week_calced = false; + // avoid repeat calc iso_week. use inner func instead of c++11 lambda + + const char *const format_begin_ptr = format.ptr(); + int64_t last_elem_end_pos = 0; + ObSEArray dfm_elems; + + // 1. parse element from format string + if (OB_FAIL(ObDFMUtil::parse_datetime_format_string(format, dfm_elems))) { + LOG_WARN("fail to parse oracle datetime format string", K(ret), K(format)); + } + + // 2. print each element + for (int64_t i = 0; OB_SUCC(ret) && i < dfm_elems.count(); ++i) { + ObDFMElem& elem = dfm_elems.at(i); + + // element is valid + if (OB_UNLIKELY(!elem.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("element is invalid", K(ret), K(elem)); + } + + // print separate chars between elements + if (OB_SUCC(ret)) { + int64_t separate_chars_len = elem.offset_ - last_elem_end_pos; + if (separate_chars_len > 0) { + ret = databuff_printf(buf, + buf_len, + pos, + "%.*s", + static_cast(separate_chars_len), + format_begin_ptr + last_elem_end_pos); } - switch (*format_ptr) { - /*The cases are not ordered alphabetically since Y y m d D are used frequently - *in order to get better performance, we locate them in front of others - */ - case 'Y': { //Year, numeric, four digits - ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR]); + last_elem_end_pos = elem.offset_ + ObDFMFlag::PATTERN[elem.elem_flag_].len_; + } + + // print current elem + if (OB_SUCC(ret)) { + switch (elem.elem_flag_) { + case ObDFMFlag::AD: + case ObDFMFlag::BC: { // TODO : NLS_LANGUAGE + const ObTimeConstStr &target_str = + ob_time.parts_[DT_YEAR] > 0 ? ObDFMFlag::PATTERN[ObDFMFlag::AD] : ObDFMFlag::PATTERN[ObDFMFlag::BC]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, target_str, elem.upper_case_mode_); + break; + } + case ObDFMFlag::AD2: + case ObDFMFlag::BC2: { // TODO : NLS_LANGUAGE + const ObTimeConstStr &str = + ob_time.parts_[DT_YEAR] > 0 ? ObDFMFlag::PATTERN[ObDFMFlag::AD2] : ObDFMFlag::PATTERN[ObDFMFlag::BC2]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, str, elem.upper_case_mode_); + break; + } + case ObDFMFlag::CC: { + ret = databuff_printf(buf, buf_len, pos, "%02d", (abs(ob_time.parts_[DT_YEAR]) + 99) / 100); + break; + } + case ObDFMFlag::SCC: { + char symbol = ob_time.parts_[DT_YEAR] > 0 ? ' ' : '-'; + ret = databuff_printf(buf, buf_len, pos, "%c%02d", symbol, (abs(ob_time.parts_[DT_YEAR]) + 99) / 100); + break; + } + case ObDFMFlag::D: { + ret = databuff_printf(buf, buf_len, pos, "%d", ob_time.parts_[DT_WDAY] % 7 + 1); + break; + } + case ObDFMFlag::DAY: { // TODO : NLS_LANGUAGE + const ObTimeConstStr &day_str = WDAY_NAMES[ob_time.parts_[DT_WDAY]]; + ret = ObDFMUtil::special_mode_sprintf( + buf, buf_len, pos, day_str, elem.upper_case_mode_, MAX_WDAY_NAME_LENGTH); + break; + } + case ObDFMFlag::DD: { + ret = databuff_printf(buf, buf_len, pos, "%02d", ob_time.parts_[DT_MDAY]); + break; + } + case ObDFMFlag::DDD: { + ret = databuff_printf(buf, buf_len, pos, "%03d", ob_time.parts_[DT_YDAY]); + break; + } + case ObDFMFlag::DY: { // TODO: 1. NLS_LANGUAGE + const ObTimeConstStr &day_str = WDAY_ABBR_NAMES[ob_time.parts_[DT_WDAY]]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, day_str, elem.upper_case_mode_); break; } - case 'y': { //Year, numeric (two digits) - int year = (ob_time.parts_[DT_YEAR]) % 100; - ret = data_fmt_nd(buf, buf_len, pos, 2, year); + case ObDFMFlag::DS: { // TODO: 1. NLS_TERRITORY 2. NLS_LANGUAGE + ret = databuff_printf(buf, + buf_len, + pos, + "%02d/%02d/%d", + ob_time.parts_[DT_MON], + ob_time.parts_[DT_MDAY], + ob_time.parts_[DT_YEAR]); break; } - case 'M': { //Month name (January..December) - ret = data_fmt_s(buf, buf_len, pos, MON_NAMES[parts[DT_MON]].ptr_); + case ObDFMFlag::DL: { // TODO: 1. NLS_DATE_FORMAT 2. NLS_TERRITORY 3. NLS_LANGUAGE + const ObTimeConstStr &wday_str = WDAY_NAMES[ob_time.parts_[DT_WDAY]]; + const ObTimeConstStr &mon_str = MON_NAMES[ob_time.parts_[DT_MON]]; + ret = databuff_printf(buf, + buf_len, + pos, + "%.*s, %.*s %02d, %d", + wday_str.len_, + wday_str.ptr_, + mon_str.len_, + mon_str.ptr_, + ob_time.parts_[DT_MDAY], + ob_time.parts_[DT_YEAR]); break; } - case 'm': { //Month, numeric (00..12) - ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MON]); + case ObDFMFlag::FF: { + if (OB_UNLIKELY(!HAS_TYPE_ORACLE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else if (0 == scale) { + // print nothing + } else { + ret = data_fmt_nd(buf, + buf_len, + pos, + scale, + ob_time.parts_[DT_USEC] / power_of_10[MAX_SCALE_FOR_ORACLE_TEMPORAL - scale]); + } break; } - case 'D': { //Day of the month with English suffix (0th, 1st, 2nd, 3rd...) - ret = data_fmt_s(buf, buf_len, pos, DAY_NAME[parts[DT_MDAY]]); + case ObDFMFlag::FF1: + case ObDFMFlag::FF2: + case ObDFMFlag::FF3: + case ObDFMFlag::FF4: + case ObDFMFlag::FF5: + case ObDFMFlag::FF6: + case ObDFMFlag::FF7: + case ObDFMFlag::FF8: + case ObDFMFlag::FF9: { + int64_t scale = elem.elem_flag_ - ObDFMFlag::FF1 + 1; + if (OB_UNLIKELY(!HAS_TYPE_ORACLE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else { + ret = data_fmt_nd(buf, + buf_len, + pos, + scale, + ob_time.parts_[DT_USEC] / power_of_10[MAX_SCALE_FOR_ORACLE_TEMPORAL - scale]); + } break; } - case 'd': { //Day of the month, numeric (00..31) - ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MDAY]); + case ObDFMFlag::HH: + case ObDFMFlag::HH12: { + int32_t h = ob_time.parts_[DT_HOUR] % 12; + if (0 == h) { + h = 12; + } + ret = databuff_printf(buf, buf_len, pos, "%02d", h); break; } - case 'a': { //Abbreviated weekday name (Sun..Sat) - ret = data_fmt_s(buf, buf_len, pos, WDAY_ABBR_NAMES[parts[DT_WDAY]].ptr_); + case ObDFMFlag::HH24: { + int32_t h = ob_time.parts_[DT_HOUR]; + ret = databuff_printf(buf, buf_len, pos, "%02d", h); break; } - case 'b': { //Abbreviated month name (Jan..Dec) - ret = data_fmt_s(buf, buf_len, pos, MON_ABBR_NAMES[parts[DT_MON]].ptr_); + case ObDFMFlag::IW: { + calc_iso_week(is_iso_week_calced, iso_week, ob_time, WEEK_MODE[3], iso_year_delta); + ret = databuff_printf(buf, buf_len, pos, "%02d", iso_week); break; } - case 'c': { //Month, numeric (0..12) - ret = data_fmt_d(buf, buf_len, pos, parts[DT_MON]); + case ObDFMFlag::J: { + const int32_t base_julian_day = 2378497; // julian day of 1800-01-01 + const int32_t base_date = -62091; // ob_time.parts_[DT_DATE] of 1800-01-01 + if (ob_time.parts_[DT_DATE] < base_date) { + ret = OB_NOT_SUPPORTED; + } else { + int32_t julian_day = base_julian_day + ob_time.parts_[DT_DATE] - base_date; + ret = databuff_printf(buf, buf_len, pos, "%07d", julian_day); + } break; } - case 'e': { //Day of the month, numeric (0..31) - ret = data_fmt_d(buf, buf_len, pos, parts[DT_MDAY]); + case ObDFMFlag::MI: { + ret = databuff_printf(buf, buf_len, pos, "%02d", ob_time.parts_[DT_MIN]); break; } - case 'f': { //Microseconds (000000..999999) - ret = data_fmt_nd(buf, buf_len, pos, 6, parts[DT_USEC]); + case ObDFMFlag::MM: { + ret = databuff_printf(buf, buf_len, pos, "%02d", ob_time.parts_[DT_MON]); break; } - case 'H': { //Hour (00..23) - ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_HOUR]); + case ObDFMFlag::MONTH: { + const ObTimeConstStr &mon_str = MON_NAMES[ob_time.parts_[DT_MON]]; + ret = + ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, mon_str, elem.upper_case_mode_, MAX_MON_NAME_LENGTH); break; } - case 'h': //Hour (01..12) - case 'I': { //Hour (01..12) - int hour = hour_converter[parts[DT_HOUR]]; - ret = data_fmt_nd(buf, buf_len, pos, 2, hour); + case ObDFMFlag::MON: { + const ObTimeConstStr &mon_str = MON_ABBR_NAMES[ob_time.parts_[DT_MON]]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, mon_str, elem.upper_case_mode_); break; } - case 'i': { //Minutes, numeric (00..59) - ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_MIN]); + case ObDFMFlag::AM: + case ObDFMFlag::PM: { + const ObTimeConstStr &str = + ob_time.parts_[DT_HOUR] >= 12 ? ObDFMFlag::PATTERN[ObDFMFlag::PM] : ObDFMFlag::PATTERN[ObDFMFlag::AM]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, str, elem.upper_case_mode_); break; } - case 'j': { //Day of year (001..366) - ret = data_fmt_nd(buf, buf_len, pos, 3, parts[DT_YDAY]); + case ObDFMFlag::AM2: + case ObDFMFlag::PM2: { + const ObTimeConstStr &str = + ob_time.parts_[DT_HOUR] >= 12 ? ObDFMFlag::PATTERN[ObDFMFlag::PM2] : ObDFMFlag::PATTERN[ObDFMFlag::AM2]; + ret = ObDFMUtil::special_mode_sprintf(buf, buf_len, pos, str, elem.upper_case_mode_); break; } - case 'k': { //Hour (0..23) - ret = data_fmt_d(buf, buf_len, pos, parts[DT_HOUR]); + case ObDFMFlag::Q: { + ret = databuff_printf(buf, buf_len, pos, "%d", (ob_time.parts_[DT_MON] + 2) / 3); break; } - case 'l': { //Hour (1..12) - int hour = hour_converter[parts[DT_HOUR]]; - ret = data_fmt_d(buf, buf_len, pos, hour); + case ObDFMFlag::RR: { + ret = databuff_printf(buf, buf_len, pos, "%02d", (ob_time.parts_[DT_YEAR]) % 100); break; } - case 'p': { //AM or PM - const char *ptr = parts[DT_HOUR] < 12 ? "AM" : "PM"; - ret = data_fmt_s(buf, buf_len, pos, ptr); + case ObDFMFlag::RRRR: { + ret = databuff_printf(buf, buf_len, pos, "%04d", ob_time.parts_[DT_YEAR]); break; } - case 'r': { //Time, 12-hour (hh:mm:ss followed by AM or PM) - int hour = hour_converter[parts[DT_HOUR]]; - const char *ptr = parts[DT_HOUR] < 12 ? "AM" : "PM"; - ret = databuff_printf(buf, buf_len, pos, "%02d:%02d:%02d %s", hour, parts[DT_MIN], parts[DT_SEC], ptr); + case ObDFMFlag::SS: { + ret = databuff_printf(buf, buf_len, pos, "%02d", ob_time.parts_[DT_SEC]); break; } - case 'S': //Seconds (00..59) - case 's': { //Seconds (00..59) - ret = data_fmt_nd(buf, buf_len, pos, 2, parts[DT_SEC]); + case ObDFMFlag::SSSSS: { + ret = databuff_printf(buf, + buf_len, + pos, + "%05d", + ob_time.parts_[DT_HOUR] * 3600 + ob_time.parts_[DT_MIN] * 60 + ob_time.parts_[DT_SEC]); break; } - case 'T': { //Time, 24-hour (hh:mm:ss) - ret = databuff_printf(buf, buf_len, pos, "%02d:%02d:%02d", parts[DT_HOUR], parts[DT_MIN], parts[DT_SEC]); + case ObDFMFlag::WW: { // the first complete week of a year + ret = databuff_printf(buf, buf_len, pos, "%02d", (ob_time.parts_[DT_YDAY] - 1) / 7 + 1); break; } - case 'U': { //Week (00..53), where Sunday is the first day of the week - ret = data_fmt_nd(buf, buf_len, pos, 2, ob_time_to_week(ob_time, WEEK_MODE[0])); + case ObDFMFlag::W: { // the first complete week of a month + ret = databuff_printf(buf, buf_len, pos, "%d", (ob_time.parts_[DT_MDAY] - 1) / 7 + 1); break; } - case 'u': { //Week (00..53), where Monday is the first day of the week - ret = data_fmt_nd(buf, buf_len, pos, 2, ob_time_to_week(ob_time, WEEK_MODE[1])); + case ObDFMFlag::YGYYY: { + ret = databuff_printf( + buf, buf_len, pos, "%d,%03d", abs(ob_time.parts_[DT_YEAR]) / 1000, abs(ob_time.parts_[DT_YEAR]) % 1000); break; } - case 'V': { //Week (01..53), where Sunday is the first day of the week; used with %X - //due to the face that V is often used with X. - // In order to optimize the implementation, we set the right delta value which will possibly be used for %X case latter. - // week_sunday != -1 means that its value has been computed in %X case ever. - ret = data_fmt_nd(buf, buf_len, pos, 2, (-1 == week_sunday) ? ob_time_to_week(ob_time, WEEK_MODE[2], delta_sunday) : week_sunday); + case ObDFMFlag::YEAR: { + ret = OB_NOT_SUPPORTED; break; } - case 'v': { //Week (01..53), where Monday is the first day of the week; used with %x - ret = data_fmt_nd(buf, buf_len, pos, 2, (-1 == week_monday) ? ob_time_to_week(ob_time, WEEK_MODE[3], delta_monday) : week_monday); + case ObDFMFlag::SYEAR: { + ret = OB_NOT_SUPPORTED; break; } - case 'W': { //Weekday name (Sunday..Saturday) - ret = data_fmt_s(buf, buf_len, pos, WDAY_NAMES[parts[DT_WDAY]].ptr_); + case ObDFMFlag::SYYYY: { + const char *fmt_str = ob_time.parts_[DT_YEAR] < 0 ? "-%04d" : " %04d"; + ret = databuff_printf(buf, buf_len, pos, fmt_str, abs(ob_time.parts_[DT_YEAR])); break; } - case 'w': { //Day of the week (0=Sunday..6=Saturday) - ret = data_fmt_d(buf, buf_len, pos, parts[DT_WDAY] % DAYS_PER_WEEK); + case ObDFMFlag::YYYY: { + ret = databuff_printf(buf, buf_len, pos, "%04d", abs(ob_time.parts_[DT_YEAR])); break; } - case 'X': { //Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - //due to the face that %X is often used with %V. - // In order to optimize the implementation, we set the right week_sunday value which will possibly be used for %V case latter. - if (-2 == delta_sunday) { - week_sunday = ob_time_to_week(ob_time, WEEK_MODE[2], delta_sunday); + case ObDFMFlag::YYY: { + ret = databuff_printf(buf, buf_len, pos, "%03d", abs(ob_time.parts_[DT_YEAR] % 1000)); + break; + } + case ObDFMFlag::YY: { + ret = databuff_printf(buf, buf_len, pos, "%02d", abs(ob_time.parts_[DT_YEAR] % 100)); + break; + } + case ObDFMFlag::Y: { + ret = databuff_printf(buf, buf_len, pos, "%01d", abs(ob_time.parts_[DT_YEAR] % 10)); + break; + } + case ObDFMFlag::IYYY: { + calc_iso_week(is_iso_week_calced, iso_week, ob_time, WEEK_MODE[3], iso_year_delta); + ret = databuff_printf(buf, buf_len, pos, "%04d", abs(ob_time.parts_[DT_YEAR] + iso_year_delta)); + break; + } + case ObDFMFlag::IYY: { + calc_iso_week(is_iso_week_calced, iso_week, ob_time, WEEK_MODE[3], iso_year_delta); + ret = databuff_printf(buf, buf_len, pos, "%03d", abs((ob_time.parts_[DT_YEAR] + iso_year_delta) % 1000)); + break; + } + case ObDFMFlag::IY: { + calc_iso_week(is_iso_week_calced, iso_week, ob_time, WEEK_MODE[3], iso_year_delta); + ret = databuff_printf(buf, buf_len, pos, "%02d", abs((ob_time.parts_[DT_YEAR] + iso_year_delta) % 100)); + break; + } + case ObDFMFlag::I: { + calc_iso_week(is_iso_week_calced, iso_week, ob_time, WEEK_MODE[3], iso_year_delta); + ret = databuff_printf(buf, buf_len, pos, "%01d", abs((ob_time.parts_[DT_YEAR] + iso_year_delta) % 10)); + break; + } + case ObDFMFlag::TZD: { + if (OB_UNLIKELY(!HAS_TYPE_ORACLE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else if (OB_LIKELY(ob_time.time_zone_id_ != OB_INVALID_INDEX)) { + ret = databuff_printf( + buf, buf_len, pos, "%.*s", ob_time.get_tzd_abbr_str().length(), ob_time.get_tzd_abbr_str().ptr()); + } else { + // do nothing } - ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR] + delta_sunday); break; } - case 'x': { //Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v - if (-2 == delta_monday) { - week_monday = ob_time_to_week(ob_time, WEEK_MODE[3], delta_monday); + case ObDFMFlag::TZR: { + if (OB_UNLIKELY(!HAS_TYPE_ORACLE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else if (OB_LIKELY(ob_time.time_zone_id_ != OB_INVALID_INDEX)) { + ret = databuff_printf( + buf, buf_len, pos, "%.*s", ob_time.get_tz_name_str().length(), ob_time.get_tz_name_str().ptr()); + } else { + const char* fmt_str = ob_time.parts_[DT_OFFSET_MIN] < 0 ? "-%02d:%02d" : "+%02d:%02d"; + ret = databuff_printf(buf, + buf_len, + pos, + fmt_str, + abs(ob_time.parts_[DT_OFFSET_MIN]) / 60, + abs(ob_time.parts_[DT_OFFSET_MIN]) % 60); } - ret = data_fmt_nd(buf, buf_len, pos, 4, ob_time.parts_[DT_YEAR] + delta_monday); break; } - case '%': { //A literal "%" character - if (pos >= buf_len) { - ret = OB_SIZE_OVERFLOW; - break; + case ObDFMFlag::TZH: { + if (OB_UNLIKELY(!HAS_TYPE_TIMEZONE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else { + const char *fmt_str = ob_time.parts_[DT_OFFSET_MIN] < 0 ? "-%02d" : "+%02d"; + ret = databuff_printf(buf, buf_len, pos, fmt_str, abs(ob_time.parts_[DT_OFFSET_MIN]) / 60); } - buf[pos++] = '%'; break; } + case ObDFMFlag::TZM: { + if (OB_UNLIKELY(!HAS_TYPE_TIMEZONE(ob_time.mode_))) { + ret = OB_INVALID_DATE_FORMAT; + } else { + ret = databuff_printf(buf, buf_len, pos, "%02d", abs(ob_time.parts_[DT_OFFSET_MIN]) % 60); + } + break; + } + case ObDFMFlag::X: { + ret = databuff_printf(buf, buf_len, pos, "."); + break; + } + default: { - ret = OB_NOT_SUPPORTED; + ret = OB_INVALID_DATE_FORMAT; + LOG_WARN("unknown elem", K(ret), K(elem)); break; } + } // end switch + if (OB_FAIL(ret)) { + LOG_WARN("failed to print buf", K(elem), K(ret)); } - if (OB_SUCC(ret)) { - format_ptr++; + } // end if + } // end for + + if (OB_SUCC(ret)) { + // print the rest separate chars + int64_t separate_chars_len = format.length() - last_elem_end_pos; + if (separate_chars_len > 0) { + if (OB_FAIL(databuff_printf(buf, + buf_len, + pos, + "%.*s", + static_cast(separate_chars_len), + format_begin_ptr + last_elem_end_pos))) { + LOG_WARN("failed to print otimestamp", "buf", ObString(pos, buf), K(ret)); } - } else if (pos >= buf_len) { - ret = OB_SIZE_OVERFLOW; - break; - } else { - buf[pos++] = *(format_ptr++); } + LOG_DEBUG("succ to print otimestamp", "buf", ObString(pos, buf), K(ret)); } - } + + if (OB_UNLIKELY(OB_SIZE_OVERFLOW == ret)) { + int ori_ret = ret; + ret = OB_ERR_DATE_FORMAT_IS_TOO_LONG_FOR_INTERNAL_BUFFER; + LOG_WARN("data format is to long for internal buffer", K(ret), K(ori_ret)); + } + } // end if return ret; } @@ -1929,7 +3998,7 @@ int ObTimeConverter::ob_time_to_datetime(ObTime &ob_time, const ObTimeZoneInfo * int32_t ObTimeConverter::ob_time_to_date(ObTime &ob_time) { int32_t value = ZERO_DATE; - if (ZERO_DATE == ob_time.parts_[DT_DATE]) { + if (ZERO_DATE == ob_time.parts_[DT_DATE] && !HAS_TYPE_ORACLE(ob_time.mode_)) { value = ZERO_DATE; } else { int32_t *parts = ob_time.parts_; @@ -1944,8 +4013,35 @@ int32_t ObTimeConverter::ob_time_to_date(ObTime &ob_time) int64_t ObTimeConverter::ob_time_to_time(const ObTime &ob_time) { - return ((ob_time.parts_[DT_HOUR] * static_cast(MINS_PER_HOUR) + ob_time.parts_[DT_MIN]) - * SECS_PER_MIN + ob_time.parts_[DT_SEC]) * USECS_PER_SEC + ob_time.parts_[DT_USEC]; + return ((static_cast(ob_time.parts_[DT_HOUR]) * MINS_PER_HOUR + ob_time.parts_[DT_MIN]) * SECS_PER_MIN + + ob_time.parts_[DT_SEC]) * USECS_PER_SEC + + (HAS_TYPE_ORACLE(ob_time.mode_) ? ob_time.parts_[DT_USEC] / NSECS_PER_USEC : ob_time.parts_[DT_USEC]); +} + +int ObTimeConverter::ob_time_to_otimestamp(ObTime &ob_time, ObOTimestampData &value) +{ + int ret = OB_SUCCESS; + if (!HAS_TYPE_ORACLE(ob_time.mode_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("it is not oracle type", K(ob_time), K(ret)); + } else if (OB_FAIL(validate_oracle_timestamp(ob_time))) { + LOG_WARN("fail to validate_oracle_timestamp", K(ob_time), K(ret)); + } else { + int64_t usec = ob_time.parts_[DT_DATE] * USECS_PER_DAY + ob_time_to_time(ob_time); + value.time_us_ = usec; + value.time_ctx_.set_tail_nsec(ob_time.parts_[DT_USEC] % NSECS_PER_USEC); + if (!HAS_TYPE_TIMEZONE(ob_time.mode_)) { + // do nothing + } else if (OB_INVALID_INDEX == ob_time.time_zone_id_) { + value.time_ctx_.store_tz_id_ = 0; + value.time_ctx_.set_offset_min(ob_time.parts_[DT_OFFSET_MIN]); + } else { + value.time_ctx_.store_tz_id_ = 1; + value.time_ctx_.set_tz_id(ob_time.time_zone_id_); + value.time_ctx_.set_tran_type_id(ob_time.transition_type_id_); + } + } + return ret; } int ObTimeConverter::ob_interval_to_interval(const ObInterval &ob_interval, int64_t &value) @@ -2016,8 +4112,11 @@ int ObTimeConverter::validate_datetime(ObTime &ob_time) int32_t *parts = ob_time.parts_; int ret = OB_SUCCESS; if (OB_UNLIKELY(0 == parts[DT_MON] && 0 == parts[DT_MDAY])) { - if (!(0 == parts[DT_YEAR] && 0 == parts[DT_HOUR] && 0 == parts[DT_MIN] - && 0 == parts[DT_SEC] && 0 == parts[DT_USEC])) { + if (!(0 == parts[DT_YEAR] + && 0 == parts[DT_HOUR] + && 0 == parts[DT_MIN] + && 0 == parts[DT_SEC] + && 0 == parts[DT_USEC])) { ret = OB_INVALID_DATE_VALUE; } else { ob_time.parts_[DT_DATE] = ZERO_DATE; @@ -2074,6 +4173,127 @@ OB_INLINE int ObTimeConverter::validate_year(int64_t year) return ret; } +int ObTimeConverter::validate_oracle_timestamp(const ObTime &ob_time) +{ + const int32_t* parts = ob_time.parts_; + int ret = OB_SUCCESS; + for (int i = 0; OB_SUCC(ret) && i < DATETIME_PART_CNT; ++i) { + if (parts[i] < TZ_PART_MIN[i] || parts[i] > TZ_PART_MAX[i]) { + ret = OB_INVALID_DATE_VALUE; + } + } + if (OB_SUCC(ret)) { + int is_leap = IS_LEAP_YEAR(parts[DT_YEAR]); + if (parts[DT_MDAY] > DAYS_PER_MON[is_leap][parts[DT_MON]]) { + ret = OB_ERR_DATE_NOT_VALID_FOR_MONTH_SPECIFIED; + } + } + + if (OB_SUCC(ret)) { + if (OB_INVALID_INDEX != ob_time.time_zone_id_) { + if (OB_UNLIKELY(!ObOTimestampData::is_valid_tz_id(ob_time.time_zone_id_)) + || OB_UNLIKELY(!ObOTimestampData::is_valid_tran_type_id(ob_time.transition_type_id_))) { + ret = OB_INVALID_DATE_VALUE; + } + } else if (OB_UNLIKELY(!ObOTimestampData::is_valid_offset_min(parts[DT_OFFSET_MIN]))) { + ret = OB_INVALID_DATE_VALUE; + } + } + return ret; +} + +int ObTimeConverter::set_ob_time_part_directly(ObTime &ob_time, + int64_t &conflict_bitset, + const int64_t part_offset, + const int32_t part_value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(part_offset >= TOTAL_PART_CNT)) { + ret = OB_INVALID_ARGUMENT; + } else { + ob_time.parts_[part_offset] = part_value; + conflict_bitset |= (1 << part_offset); + } + return ret; +} + +/* + * element group may cause conflict on parts in ob_time + * 1. SSSSS vs HH, HH24, HH12, MI, SS + * 2. DDD vs DD MM/Mon/Month + * + * while call this function, the part_value must be the final value + */ +int ObTimeConverter::set_ob_time_part_may_conflict(ObTime &ob_time, + int64_t &conflict_bitset, + const int64_t part_offset, + const int32_t part_value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(part_offset >= TOTAL_PART_CNT)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(part_offset)); + } else { + if (0 != (conflict_bitset & (1 << part_offset))) { + // already has data in ob_time.part_[part_name], validate it + if (OB_UNLIKELY(part_value != ob_time.parts_[part_offset])) { + ret = get_oracle_err_when_datetime_parts_conflict(part_offset); + LOG_WARN("set time conflict", K(ret), K(part_offset), K(part_value), K(ob_time)); + } + } else { + conflict_bitset |= (1 << part_offset); + ob_time.parts_[part_offset] = part_value; + } + } + + return ret; +} + +int32_t ObTimeConverter::calc_max_name_length(const ObTimeConstStr names[], const int64_t size) +{ + int32_t res = 0; + for (int64_t i = 1; i <= size; ++i) { + if (res < names[i].len_) { + res = names[i].len_; + } + } + return res; +} + +ObOTimestampData ObTimeConverter::round_otimestamp(const int16_t scale, const ObOTimestampData &in_ot_data) +{ + ObOTimestampData ret_ot_data = in_ot_data; + const int16_t MIN_SCALE = 0; + const int16_t MAX_US_SCALE = 6; + const int16_t MAX_NS_SCALE = 9; + if (MIN_SCALE <= scale && scale <= MAX_NS_SCALE) { + if (scale < MAX_US_SCALE) { + round_datetime(scale, *(int64_t*)&ret_ot_data.time_us_); + ret_ot_data.time_ctx_.tail_nsec_ = 0; + } else if (MAX_US_SCALE == scale) { + if (ret_ot_data.time_ctx_.tail_nsec_ >= power_of_10[MAX_NS_SCALE - MAX_US_SCALE] / 2) { + ++ret_ot_data.time_us_; + } + ret_ot_data.time_ctx_.tail_nsec_ = 0; + } else if (scale < MAX_NS_SCALE) { + int32_t power_of_precision = + static_cast(power_of_10[MAX_NS_SCALE - MAX_US_SCALE - (scale - MAX_US_SCALE)]); + int32_t residue = ret_ot_data.time_ctx_.tail_nsec_ % power_of_precision; + if (residue >= power_of_precision / 2) { + ret_ot_data.time_ctx_.set_tail_nsec(ret_ot_data.time_ctx_.tail_nsec_ + power_of_precision - residue); + } else { + ret_ot_data.time_ctx_.set_tail_nsec(ret_ot_data.time_ctx_.tail_nsec_ - residue); + } + if (ret_ot_data.time_ctx_.tail_nsec_ >= power_of_10[MAX_NS_SCALE - MAX_US_SCALE]) { + ++ret_ot_data.time_us_; + ret_ot_data.time_ctx_.set_tail_nsec( + ret_ot_data.time_ctx_.tail_nsec_ - static_cast(power_of_10[MAX_NS_SCALE - MAX_US_SCALE])); + } + } + } // others, just return the original value. + return ret_ot_data; +} + int ObTimeConverter::time_overflow_trunc(int64_t &value) { int ret = OB_SUCCESS; @@ -2425,6 +4645,61 @@ OB_INLINE int ObTimeConverter::sub_timezone_offset(const ObTimeZoneInfo *tz_info return ret; } +OB_INLINE int ObTimeConverter::sub_timezone_offset(const ObTimeZoneInfo *tz_info, + bool is_timestamp, + const ObString &tz_abbr_str, + int64_t &value, + const bool is_oracle_mode) +{ + int ret = OB_SUCCESS; + if (tz_info != NULL + && (ZERO_DATETIME != value || is_oracle_mode)) { + if (is_timestamp) { + int32_t offset_sec = 0; + int32_t tz_id = OB_INVALID_INDEX; + int32_t tran_type_id = OB_INVALID_INDEX; + if (OB_FAIL(tz_info->get_timezone_sub_offset(USEC_TO_SEC(value), tz_abbr_str, offset_sec, tz_id, tran_type_id))) { + LOG_WARN("failed to get offset between utc and local", K(ret)); + } else { + value -= SEC_TO_USEC(offset_sec); + } + } + } + return ret; +} + +OB_INLINE int ObTimeConverter::sub_timezone_offset(const ObTimeZoneInfo &tz_info, + const ObString &tz_abbr_str, + int64_t &value_us, + int32_t &offset_min, + int32_t &tz_id, + int32_t &tran_type_id) +{ + int ret = OB_SUCCESS; + int32_t offset_sec = 0; + tran_type_id = OB_INVALID_INDEX; + if (OB_FAIL(tz_info.get_timezone_sub_offset(USEC_TO_SEC(value_us), tz_abbr_str, offset_sec, tz_id, tran_type_id))) { + LOG_WARN("failed to get offset between utc and local", K(ret)); + } else if (OB_INVALID_INDEX == tz_id) { + if (OB_UNLIKELY(!ObOTimestampData::is_valid_offset_min(static_cast(SEC_TO_MIN(offset_sec))))) { + ret = OB_INVALID_DATE_VALUE; + LOG_WARN("invalid offset_sec", K(offset_sec), K(ret)); + } + } else { + if (OB_UNLIKELY(!ObOTimestampData::is_valid_tz_id(tz_id)) + || OB_UNLIKELY(!ObOTimestampData::is_valid_tran_type_id(tran_type_id))) { + ret = OB_INVALID_DATE_VALUE; + LOG_WARN("invalid tz_id", K(tz_id), K(ret)); + } + } + + if (OB_SUCC(ret)) { + value_us -= SEC_TO_USEC(offset_sec); + offset_min = static_cast(SEC_TO_MIN(offset_sec)); + } + return ret; +} + int ObTimeConverter::get_str_array_idx(const ObString &str, const ObTimeConstStr *str_arr, int32_t count, int32_t &idx) { int ret = OB_SUCCESS; @@ -2456,5 +4731,94 @@ void ObTimeConverter::get_first_day_of_isoyear(ObTime &ob_time) ob_time.parts_[DT_DATE] -= offset; } +int ObTimeConverter::validate_oracle_date(const ObTime &ob_time) +{ + const int32_t *parts = ob_time.parts_; + int ret = OB_SUCCESS; + for (int i = 0; OB_SUCC(ret) && i < ORACLE_DATE_PART_CNT; ++i) { + if (parts[i] < TZ_PART_MIN[i] || parts[i] > TZ_PART_MAX[i]) { + ret = OB_INVALID_DATE_VALUE; + } + } + if (OB_SUCC(ret)) { + int is_leap = IS_LEAP_YEAR(parts[DT_YEAR]); + if (parts[DT_MDAY] > DAYS_PER_MON[is_leap][parts[DT_MON]]) { + ret = OB_INVALID_DATE_VALUE; + } + } + return ret; +} + + +ObString ObDataTypeCastParams::get_nls_format(const ObObjType input_type) const +{ + ObString format_str; + switch (input_type) { + case ObDateTimeType: + format_str = (force_use_standard_format_ + ? ObTimeConverter::COMPAT_OLD_NLS_DATE_FORMAT + : (session_nls_formats_[NLS_DATE].empty() + ? ObTimeConverter::DEFAULT_NLS_DATE_FORMAT + : session_nls_formats_[NLS_DATE])); + break; + case ObTimestampNanoType: + case ObTimestampLTZType: + format_str = (force_use_standard_format_ + ? ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_FORMAT + : (session_nls_formats_[NLS_TIMESTAMP].empty() + ? ObTimeConverter::DEFAULT_NLS_TIMESTAMP_FORMAT + : session_nls_formats_[NLS_TIMESTAMP])); + break; + case ObTimestampTZType: + format_str = (force_use_standard_format_ + ? ObTimeConverter::COMPAT_OLD_NLS_TIMESTAMP_TZ_FORMAT + : (session_nls_formats_[NLS_TIMESTAMP_TZ].empty() + ? ObTimeConverter::DEFAULT_NLS_TIMESTAMP_TZ_FORMAT + : session_nls_formats_[NLS_TIMESTAMP_TZ])); + break; + default: + break; + } + return format_str; +} + +void ObDataTypeCastParams::set_nls_date_format(ObString str) +{ + session_nls_formats_[NLS_DATE] = str; +} + +void ObDataTypeCastParams::set_nls_timestamp_format(ObString str) +{ + session_nls_formats_[NLS_TIMESTAMP] = str; +} + +void ObDataTypeCastParams::set_nls_timestamp_tz_format(ObString str) +{ + session_nls_formats_[NLS_TIMESTAMP_TZ] = str; +} + +int ObDataTypeCastParams::set_nls_format_by_type(ObObjType type, ObString str) +{ + int ret = OB_SUCCESS; + + switch (type) { + case ObDateTimeType: + set_nls_date_format(str); + break; + case ObTimestampNanoType: + case ObTimestampLTZType: + set_nls_timestamp_format(str); + break; + case ObTimestampTZType: + set_nls_timestamp_tz_format(str); + break; + default: + ret = OB_INVALID_ARGUMENT; + break; + } + + return ret; +} + } // namesapce common } // namespace oceanbase diff --git a/src/lib/timezone/ob_time_convert.h b/src/lib/timezone/ob_time_convert.h index 066ea7a1f0a86e407a2c17ce800b2a4649cd9e07..e5e3a5b78f13ff01201fc7d10b1b34d940397ff5 100644 --- a/src/lib/timezone/ob_time_convert.h +++ b/src/lib/timezone/ob_time_convert.h @@ -23,6 +23,12 @@ namespace oceanbase { namespace common { +class ObTimeZoneInfoPos; +class ObTimeZoneInfo; +class ObOTimestampData; +struct ObTimeConstStr; +struct ObDataTypeCastParams; +class ObObj; #define DT_TYPE_DATE (1UL << 0) #define DT_TYPE_TIME (1UL << 1) @@ -35,9 +41,16 @@ namespace common #define DT_WEEK_ZERO_BEGIN (1UL << 6) // week num will begin with 0, otherwise 1. #define DT_WEEK_GE_4_BEGIN (1UL << 7) // week which has 4 or more days is week 1, otherwise has // the first sunday of monday. + +#define DT_TYPE_ORACLE (1UL << 8) // oracle timestamp to nanosecond (nano, tz, ltz) +#define DT_TYPE_STORE_UTC (1UL << 9) // store utc (tz, ltz) +#define DT_TYPE_TIMEZONE (1UL << 10) // oracle timestamp with time zone (tz) + typedef uint64_t ObDTMode; #define DT_TYPE_DATETIME (DT_TYPE_DATE | DT_TYPE_TIME) +#define DT_TYPE_ORACLE_TIMESTAMP (DT_TYPE_DATETIME | DT_TYPE_ORACLE) +#define DT_TYPE_ORACLE_TTZ (DT_TYPE_DATETIME | DT_TYPE_ORACLE | DT_TYPE_TIMEZONE) #define DT_TYPE_CNT (3) #define HAS_TYPE_DATE(mode) (DT_TYPE_DATE & (mode)) @@ -49,12 +62,17 @@ typedef uint64_t ObDTMode; #define IS_SUN_BEGIN(mode) ((DT_WEEK_SUN_BEGIN & (mode)) ? 1 : 0) #define IS_ZERO_BEGIN(mode) ((DT_WEEK_ZERO_BEGIN & (mode)) ? 1 : 0) #define IS_GE_4_BEGIN(mode) ((DT_WEEK_GE_4_BEGIN & (mode)) ? 1 : 0) +#define HAS_TYPE_ORACLE(mode) ((DT_TYPE_ORACLE & (mode)) ? 1 : 0) +#define HAS_TYPE_TIMEZONE(mode) ((DT_TYPE_TIMEZONE & (mode)) ? 1 : 0) +#define HAS_TYPE_STORE_UTC(mode) ((DT_TYPE_STORE_UTC & (mode)) ? 1 : 0) #define DATE_PART_CNT 3 #define TIME_PART_CNT 4 -#define OTHER_PART_CNT 3 +#define OTHER_PART_CNT 4 #define DATETIME_PART_CNT (DATE_PART_CNT + TIME_PART_CNT) +#define ORACLE_DATE_PART_CNT (DATE_PART_CNT + TIME_PART_CNT - 1) #define TOTAL_PART_CNT (DATETIME_PART_CNT + OTHER_PART_CNT) + #define DT_YEAR 0 #define DT_MON 1 #define DT_MDAY 2 @@ -65,6 +83,11 @@ typedef uint64_t ObDTMode; #define DT_DATE 7 #define DT_YDAY 8 #define DT_WDAY 9 +#define DT_OFFSET_MIN 10 + +#define DT_MON_NAME \ + 11 // monthname doesn't contains real data by using month directly + // put it after DT_OFFSET_MIN will be fine extern const int32_t DT_PART_BASE[DATETIME_PART_CNT]; extern const int32_t DT_PART_MIN[DATETIME_PART_CNT]; @@ -101,25 +124,54 @@ extern const int64_t USECS_PER_DAY; #define TIMESTAMP_VALUE_LENGTH 12 #define SEC_TO_USEC(secs) ((secs) * static_cast(USECS_PER_SEC)) #define USEC_TO_SEC(usec) ((usec) / USECS_PER_SEC) +#define SEC_TO_MIN(secs) ((secs) / SECS_PER_MIN) +#define MIN_TO_USEC(min) ((min)*SECS_PER_MIN * USECS_PER_SEC) #define TIMESTAMP_MAX_VAL 253402272000 +#define DATETIME_MIN_VAL -62167132800000000 +#define DATETIME_MAX_VAL 253402300799999999 +#define ORACLE_DATETIME_MIN_VAL -62135596800000000 //start from '0001-1-1 00:00:00' + class ObTime { public: ObTime() - : mode_(0) + : mode_(0), + time_zone_id_(common::OB_INVALID_INDEX), + transition_type_id_(common::OB_INVALID_INDEX), + is_tz_name_valid_(false) { - for (int i = 0; i < TOTAL_PART_CNT; ++i) { - parts_[i] = 0; - } + MEMSET(parts_, 0, sizeof(parts_)); + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + MEMSET(tzd_abbr_, 0, common::OB_MAX_TZ_ABBR_LEN); } explicit ObTime(ObDTMode mode) - : mode_(mode) + : mode_(mode), + time_zone_id_(common::OB_INVALID_INDEX), + transition_type_id_(common::OB_INVALID_INDEX), + is_tz_name_valid_(false) { - for (int i = 0; i < TOTAL_PART_CNT; ++i) { - parts_[i] = 0; - } + MEMSET(parts_, 0, sizeof(parts_)); + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + MEMSET(tzd_abbr_, 0, common::OB_MAX_TZ_ABBR_LEN); + } + ~ObTime() + {} + ObString get_tz_name_str() const + { + return ObString(strlen(tz_name_), tz_name_); + } + ObString get_tzd_abbr_str() const + { + return ObString(strlen(tzd_abbr_), tzd_abbr_); } + int set_tz_name(const ObString &tz_name); + int set_tzd_abbr(const ObString &tz_abbr); + + TO_STRING_KV(K(mode_), "parts", ObArrayWrap(parts_, TOTAL_PART_CNT), "tz_name", + ObString(OB_MAX_TZ_NAME_LEN, tz_name_), "tzd_abbr", ObString(OB_MAX_TZ_ABBR_LEN, tzd_abbr_), K_(time_zone_id), + K_(transition_type_id), K_(is_tz_name_valid)); + ObDTMode mode_; int32_t parts_[TOTAL_PART_CNT]; // year: [1000, 9999]. @@ -132,18 +184,50 @@ public: // date: date value, day count since 1970-1-1. // year day: [1, 366]. // week day: [1, 7], 1 means monday, 7 means sunday. + // offset minute: [-12*60, 14*60]. + + char tz_name_[common::OB_MAX_TZ_NAME_LEN]; + char tzd_abbr_[common::OB_MAX_TZ_ABBR_LEN]; // the abbr of time zone region with Daylight Saving Time + int32_t time_zone_id_; + int32_t transition_type_id_; + bool is_tz_name_valid_; }; typedef ObTime ObInterval; +struct ObTimeConstStr { + ObTimeConstStr() : ptr_(NULL), len_(0) + {} + ObTimeConstStr(const char *str) : ptr_(str), len_(static_cast(strlen(str))) + {} + ObTimeConstStr(const char *str, int32_t len) : ptr_(str), len_(len) + {} + inline ObString to_obstring() const + { + return ObString(len_, ptr_); + } + const char *ptr_; + int32_t len_; + TO_STRING_KV("value", ObString(len_, ptr_), K_(len)); +}; + struct ObTimeConvertCtx { - ObTimeConvertCtx(const ObTimeZoneInfo *tz_info, const bool is_timestamp) - :tz_info_(tz_info), - is_timestamp_(is_timestamp) {} + ObTimeConvertCtx(const ObTimeZoneInfo *tz_info, const bool is_timestamp) + : tz_info_(tz_info), + oracle_nls_format_(), + is_timestamp_(is_timestamp) + {} + ObTimeConvertCtx(const ObTimeZoneInfo *tz_info, const ObString &oracle_nls_format, const bool is_timestamp) + : tz_info_(tz_info), + oracle_nls_format_(oracle_nls_format), + is_timestamp_(is_timestamp) + {} const ObTimeZoneInfo *tz_info_; + ObString oracle_nls_format_; bool is_timestamp_; //means mysql timestamp? }; + class ObTimeConverter { public: @@ -153,6 +237,13 @@ public: static const int32_t ZERO_DATE = static_cast(-106751991); static const int64_t ZERO_TIME = 0; static const uint8_t ZERO_YEAR = 0; + static const ObString DEFAULT_NLS_DATE_FORMAT; + static const ObString DEFAULT_NLS_TIMESTAMP_FORMAT; + static const ObString DEFAULT_NLS_TIMESTAMP_TZ_FORMAT; + static const ObString COMPAT_OLD_NLS_DATE_FORMAT; + static const ObString COMPAT_OLD_NLS_TIMESTAMP_FORMAT; + static const ObString COMPAT_OLD_NLS_TIMESTAMP_TZ_FORMAT; + public: // int / double / string -> datetime(timestamp) / interval / date / time / year. static int int_to_datetime(int64_t int_part, int64_t dec_part, const ObTimeZoneInfo *tz_info, int64_t &value); @@ -167,6 +258,8 @@ public: static int str_to_time(const ObString &str, int64_t &value, int16_t *scale = NULL); static int str_to_year(const ObString &str, uint8_t &value); static int str_to_interval(const ObString &str, ObDateUnitType unit_type, int64_t &value); + static int str_to_otimestamp(const ObString &str, const ObTimeConvertCtx &cvrt_ctx, const ObObjType target_type, + ObOTimestampData &value, ObScale &scale); // int / double / string <- datetime(timestamp) / date / time / year. static int datetime_to_int(int64_t value, const ObTimeZoneInfo *tz_info, int64_t &int64); static int datetime_to_double(int64_t value, const ObTimeZoneInfo *tz_info, double &dbl); @@ -185,13 +278,35 @@ public: // inner cast between datetime, timestamp, date, time, year. static int datetime_to_timestamp(int64_t dt_value, const ObTimeZoneInfo *tz_info, int64_t &ts_value); static int timestamp_to_datetime(int64_t ts_value, const ObTimeZoneInfo *tz_info, int64_t &dt_value); + static inline void datetime_to_odate(int64_t dt_value, int64_t &odate_value) + { + odate_value = dt_value; + } + static int odate_to_otimestamp(int64_t in_value_us, const ObTimeZoneInfo *tz_info, const ObObjType out_type, + ObOTimestampData &out_value); + static int otimestamp_to_odate(const ObObjType in_type, const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, int64_t &out_usec); + static int otimestamp_to_otimestamp(const ObObjType in_type, const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, const ObObjType out_type, + ObOTimestampData &out_value); + static int otimestamp_to_str(const ObOTimestampData &value, const ObDataTypeCastParams &dtc_params, + const int16_t scale, const ObObjType type, char *buf, int64_t buf_len, int64_t &pos); + static int extract_offset_from_otimestamp(const ObOTimestampData &in_value, + const ObTimeZoneInfo *tz_info, + int32_t &offset_min, + ObTime &ob_time); static int datetime_to_date(int64_t dt_value, const ObTimeZoneInfo *tz_info, int32_t &d_value); static int datetime_to_time(int64_t dt_value, const ObTimeZoneInfo *tz_info, int64_t &t_value); static int datetime_to_year(int64_t dt_value, const ObTimeZoneInfo *tz_info, uint8_t &y_value); + static int date_to_datetime(int32_t d_value, const ObTimeZoneInfo *tz_info, int64_t &dt_value); static int date_to_year(int32_t d_value, uint8_t &y_value); - // string -> offset. - static int str_to_offset(const ObString &str, int32_t &value); // seconds, not useconds. + // string -> offset. value seconds, not useconds. + static int str_to_offset(const ObString &str, + int32_t &value, + int &ret_more, + const bool is_oracle_mode, + const bool need_check_valid = false); // year / month / day / quarter / week / hour / minite / second / microsecond. static int int_to_week(int64_t uint64, int64_t mode, int32_t &value); // date add / sub / diff. @@ -208,7 +323,12 @@ public: static int str_to_ob_time_without_date(const ObString &str, ObTime &ob_time, int16_t *scale = NULL); static int str_to_ob_time_format(const ObString &str, const ObString &fmt, ObTime &ob_time, int16_t *scale = NULL); static int str_to_ob_interval(const ObString &str, ObDateUnitType unit_type, ObInterval &ob_interval); + static int str_to_ob_time_oracle_dfm(const ObString &str, const ObTimeConvertCtx &cvrt_ctx, + const ObObjType target_type, ObTime &ob_time, ObScale &scale); + static int usec_to_ob_time(int64_t usecs, ObTime &ob_time); static int datetime_to_ob_time(int64_t value, const ObTimeZoneInfo *tz_info, ObTime &ob_time); + static int otimestamp_to_ob_time(const ObObjType type, const ObOTimestampData &ot_data, const ObTimeZoneInfo *tz_info, + ObTime &ob_time, const bool store_utc_time = true); static int date_to_ob_time(int32_t value, ObTime &ob_time); static int time_to_ob_time(int64_t value, ObTime &ob_time); // int / string <- ObTime -> datetime(timestamp) / date / time. @@ -218,17 +338,33 @@ public: char *buf, int64_t buf_len, int64_t &pos, bool with_delim); static int ob_time_to_str_format(const ObTime &ob_time, const ObString &format, char *buf, int64_t buf_len, int64_t &pos); + static int calc_tz_offset_by_tz_name(const ObTimeConvertCtx &cvrt_ctx, ObTime &ob_time); + static int ob_time_to_utc(const ObObjType obj_type, const ObTimeConvertCtx &cvrt_ctx, ObTime &ob_time); + static bool valid_oracle_year(const ObTime &ob_time); + static void calc_iso_week(bool &is_iso_week_calced, int32_t &iso_week, const ObTime &ob_time, + ObDTMode mode, int32_t &delta); + static int ob_time_to_str_oracle_dfm(const ObTime &ob_time, ObScale scale, const ObString &format, + char *buf, int64_t buf_len, int64_t &pos); static int ob_time_to_datetime(ObTime &ob_time, const ObTimeZoneInfo *sp, int64_t &value); static int32_t ob_time_to_date(ObTime &ob_time); + static int ob_time_to_otimestamp(ObTime &ob_time, ObOTimestampData &value); static int64_t ob_time_to_time(const ObTime &ob_time); static int ob_interval_to_interval(const ObInterval &ob_interval, int64_t &value); // year / month / day / quarter / week / hour / minite / second / microsecond. static int32_t ob_time_to_week(const ObTime &ob_time, ObDTMode mode); static int32_t ob_time_to_week(const ObTime &ob_time, ObDTMode mode, int32_t &delta); static void get_first_day_of_isoyear(ObTime &ob_time); - + static int validate_oracle_date(const ObTime &ob_time); + static bool is_valid_otimestamp(const int64_t time_us, const int32_t tail_nsec); + public: // other functions. + static int set_ob_time_part_directly( + ObTime &ob_time, int64_t &conflict_bitset, const int64_t part_offset, const int32_t part_value); + static int set_ob_time_part_may_conflict( + ObTime &ob_time, int64_t &conflict_bitset, const int64_t part_offset, const int32_t part_value); + static int32_t calc_max_name_length(const ObTimeConstStr names[], const int64_t size); + static ObOTimestampData round_otimestamp(const int16_t scale, const ObOTimestampData &in_ot_data); static int time_overflow_trunc(int64_t &value); static void round_datetime(int16_t scale, int64_t &value); static void trunc_datetime(int16_t scale, int64_t &value); @@ -236,22 +372,20 @@ public: { return (ZERO_DATE == value) || (ZERO_DATETIME == value); } + static int get_oracle_err_when_datetime_parts_conflict(int64_t part_idx); + struct ObTimeDigits { - ObTimeDigits() - : ptr_(NULL), len_(0), value_(0) + ObTimeDigits() : ptr_(NULL), len_(0), value_(0) {} + VIRTUAL_TO_STRING_KV(K(ptr_), K(len_), K(value_)); const char *ptr_; int32_t len_; int32_t value_; }; struct ObTimeDelims { - ObTimeDelims() - : ptr_(NULL), len_(0) + ObTimeDelims() : ptr_(NULL), len_(0) {} - const char *ptr_; - int32_t len_; - }; - struct ObTimeConstStr { + VIRTUAL_TO_STRING_KV(K(ptr_), K(len_)); const char *ptr_; int32_t len_; }; @@ -271,6 +405,8 @@ private: static int validate_datetime(ObTime &ob_time); static int validate_time(ObTime &ob_time); static int validate_year(int64_t year); + static int validate_oracle_timestamp(const ObTime &ob_time); + static int check_leading_precision(const ObTimeDigits &digits); static int get_datetime_digits(const char *&str, const char *end, int32_t max_len, ObTimeDigits &digits); static int get_datetime_delims(const char *&str, const char *end, ObTimeDelims &delims); static int get_datetime_digits_delims(const char *&str, const char *end, @@ -296,17 +432,87 @@ private: // static int find_transition_type(int64_t t, const ObTimeZoneInfo *sp, TRAN_TYPE_INFO *& result); static int add_timezone_offset(const ObTimeZoneInfo *tz_info, int64_t &value); static int sub_timezone_offset(const ObTimeZoneInfo *tz_info, int64_t &value); + static int sub_timezone_offset(const ObTimeZoneInfo *tz_info, bool is_timestamp, const ObString &tz_abbr_str, + int64_t &value, const bool is_oracle_mode = false); + static int sub_timezone_offset(const ObTimeZoneInfo &tz_info, const ObString &tz_abbr_str, int64_t &value_us, + int32_t &offset_min, int32_t &tz_id, int32_t &tran_type_id); static int get_str_array_idx(const ObString &str, const ObTimeConstStr *array, int32_t count, int32_t &idx); static int data_fmt_nd(char *buffer, int64_t buf_len, int64_t &pos, const int64_t n, int64_t target); static int data_fmt_d(char *buffer, int64_t buf_len, int64_t &pos, int64_t target); static int data_fmt_s(char *buffer, int64_t buf_len, int64_t &pos, const char *ptr); + static int get_day_and_month_from_year_day(const int32_t yday, const int32_t year, int32_t &month, int32_t &day); + static int set_ob_time_year_may_conflict(ObTime &ob_time, int32_t &julian_year_value, + int32_t check_year, int32_t set_year, bool overwrite); + private: ObTimeConverter(); virtual ~ObTimeConverter(); DISALLOW_COPY_AND_ASSIGN(ObTimeConverter); }; +enum ObNLSFormatEnum { + NLS_DATE = 0, + NLS_TIMESTAMP, + NLS_TIMESTAMP_TZ, + NLS_MAX, +}; + +/** + * @brief The ObDataTypeCastParams struct + * include session variables for SQL resolver + * especially for oracle mode, including time zone, datetime format and charset + */ +struct ObDataTypeCastParams { + ObDataTypeCastParams() + : tz_info_(NULL), + force_use_standard_format_(false), + nls_collation_(CS_TYPE_INVALID), + nls_collation_nation_(CS_TYPE_INVALID), + connection_collation_(CS_TYPE_UTF8MB4_BIN) + {} + ObDataTypeCastParams(const ObTimeZoneInfo *tz_info, bool force_use_standard_format = true) + : tz_info_(tz_info), + force_use_standard_format_(force_use_standard_format), + nls_collation_(CS_TYPE_INVALID), + nls_collation_nation_(CS_TYPE_INVALID), + connection_collation_(CS_TYPE_UTF8MB4_BIN) + {} + ObDataTypeCastParams(const ObTimeZoneInfo *tz_info, + const ObString *nls_formats, + const ObCollationType nls_collation, + const ObCollationType nls_collation_nation, + const ObCollationType connection_collation, + const bool force_use_standard_format = false) + : tz_info_(tz_info), + force_use_standard_format_(force_use_standard_format), + nls_collation_(nls_collation), + nls_collation_nation_(nls_collation_nation), + connection_collation_(connection_collation) + { + for (int64_t i = 0; NULL != nls_formats && i < NLS_MAX; ++i) { + session_nls_formats_[i] = nls_formats[i]; + } + } + + ObString get_nls_format(const ObObjType input_type) const; + void set_nls_date_format(ObString str); + void set_nls_timestamp_format(ObString str); + void set_nls_timestamp_tz_format(ObString str); + int set_nls_format_by_type(ObObjType type, ObString str); + + const ObTimeZoneInfo *tz_info_; + ObString session_nls_formats_[NLS_MAX]; + + // only user related str depend nls_format. others do not care it, such as ob_print_sql... + bool force_use_standard_format_; + + ObCollationType nls_collation_; + ObCollationType nls_collation_nation_; + ObCollationType connection_collation_; // as client cs for now +}; + + }// end of common }// end of oceanbase diff --git a/src/lib/timezone/ob_timezone_info.cpp b/src/lib/timezone/ob_timezone_info.cpp index be1d0861a0a3f872f42303e47c46fdcf917feac3..7373f0f54a8c9be9a6e929615e9e7c16bd204436 100644 --- a/src/lib/timezone/ob_timezone_info.cpp +++ b/src/lib/timezone/ob_timezone_info.cpp @@ -15,6 +15,7 @@ #include "lib/timezone/ob_timezone_info.h" #include "lib/timezone/ob_time_convert.h" #include "lib/oblog/ob_log.h" +#include "lib/ob_proxy_worker.h" namespace oceanbase { @@ -38985,19 +38986,376 @@ ObTimeZoneTrans ObTimeZoneInfo::TIME_ZONE_TRANS[] = {INT64_MIN, 0, 573}, }; -int ObTimeZoneInfo::set_timezone(const ObString &str) +DEF_TO_STRING(ObOTimestampData) +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(time_us), + "tail_nsec", + time_ctx_.tail_nsec_, + "version", + time_ctx_.version_, + "is_null", + time_ctx_.is_null_, + "store_tz_id", + time_ctx_.store_tz_id_); + if (time_ctx_.store_tz_id_ != 0) { + J_COMMA(); + J_KV("tz_id", time_ctx_.tz_id_, "tran_type_id", time_ctx_.tran_type_id_); + } else { + J_COMMA(); + J_KV("is_neg_offset", time_ctx_.is_neg_offset_, "offset_min", time_ctx_.offset_min_); + } + J_OBJ_END(); + return pos; +} + +ObTZNameKey::ObTZNameKey(const ObString &tz_key_str) +{ + int64_t len = tz_key_str.length(); + if (OB_UNLIKELY(len + 1 > OB_MAX_TZ_NAME_LEN)) { + LOG_ERROR("invalid tz_key_str", K(tz_key_str)); + } else { + for (int64_t i = 0; i < len; ++i) { + tz_name_[i] = static_cast(tolower(tz_key_str[i])); + } + tz_name_[len] = 0; + } +} + +void ObTZNameKey::reset() +{ + MEMSET(tz_name_, 0, OB_MAX_TZ_NAME_LEN); +} + +int ObTZNameKey::assign(const ObTZNameKey &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + this->reset(); + MEMCPY(tz_name_, src.tz_name_, OB_MAX_TZ_NAME_LEN); + } + return ret; +} + +void ObTZNameKey::operator=(const ObTZNameKey &key) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(assign(key))) { + LOG_ERROR("fail to assign ObTZNameKey", K(key), K(ret)); + } +} + +ObTZNameKey::ObTZNameKey(const ObTZNameKey &key) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(assign(key))) { + LOG_ERROR("fail to assign ObTZNameKey", K(key), K(ret)); + } +} + +uint64_t ObTZNameKey::hash(uint64_t seed) const +{ + uint64_t seed_ret = 0; + int32_t str_len = static_cast(strlen(tz_name_)); + if (OB_ISNULL(tz_name_) || OB_UNLIKELY(str_len > OB_MAX_TZ_NAME_LEN)) { + LOG_WARN("invalid tz_name", K(str_len)); + } else { + seed_ret = murmurhash(tz_name_, str_len, seed); + } + return seed_ret; +} + +void ObTZTransitionTypeInfo::reset() +{ + lower_time_ = OB_INVALID_TZ_TRAN_TIME; + info_.reset(); +} + +int ObTZTransitionTypeInfo::assign(const ObTZTransitionTypeInfo &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + this->reset(); + lower_time_ = src.lower_time_; + ret = info_.assign(src.info_); + } + return ret; +} + +int ObTZTransitionTypeInfo::get_offset_according_abbr(const ObString &tz_abbr_str, + int32_t &offset_sec, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + if (0 == tz_abbr_str.case_compare(info_.abbr_)) { + offset_sec = info_.offset_sec_; + tran_type_id = info_.tran_type_id_; + } else { + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + LOG_WARN("invalid abbr", K(tz_abbr_str), KPC(this), K(ret)); + } + return ret; +} + +OB_DEF_SERIALIZE(ObTZTransitionTypeInfo) +{ + int ret = OB_SUCCESS; + ObString abbr_str(static_cast(strlen(info_.abbr_)), info_.abbr_); + LST_DO_CODE(OB_UNIS_ENCODE, lower_time_, info_.offset_sec_, info_.is_dst_, abbr_str, info_.tran_type_id_); + return ret; +} + +OB_DEF_DESERIALIZE(ObTZTransitionTypeInfo) { int ret = OB_SUCCESS; - if (OB_FAIL(get_timezone_id(str, tz_id_))) { - LOG_WARN("failed get time zone id", K(ret), K(str)); - } else if (0 == tz_id_) { - if (OB_FAIL(ObTimeConverter::str_to_offset(str, offset_))) { - LOG_WARN("invalid time zone offset", K(ret), K(str)); + info_.reset(); + ObString abbr_str; + LST_DO_CODE(OB_UNIS_DECODE, lower_time_, info_.offset_sec_, info_.is_dst_, abbr_str, info_.tran_type_id_); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(abbr_str.length() + 1 > OB_MAX_TZ_ABBR_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid abbr_str", K(abbr_str), K(ret)); + } else { + MEMCPY(info_.abbr_, abbr_str.ptr(), abbr_str.length()); + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObTZTransitionTypeInfo) +{ + int64_t len = 0; + ObString abbr_str(static_cast(strlen(info_.abbr_)), info_.abbr_); + LST_DO_CODE(OB_UNIS_ADD_LEN, lower_time_, info_.offset_sec_, info_.is_dst_, abbr_str, info_.tran_type_id_); + return len; +} + +void ObTZRevertTypeInfo::reset() +{ + ObTZTransitionTypeInfo::reset(); + type_class_ = NONE; + extra_info_.reset(); +} + +int ObTZRevertTypeInfo::assign_normal(const ObTZTransitionTypeInfo &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + ret = info_.assign(src.info_); + } + return ret; +} + +int ObTZRevertTypeInfo::assign_extra(const ObTZTransitionTypeInfo &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + ret = extra_info_.assign(src.info_); + } + return ret; +} + +int ObTZRevertTypeInfo::assign(const ObTZRevertTypeInfo &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + this->reset(); + if (OB_FAIL(ObTZTransitionTypeInfo::assign(src))) { + LOG_WARN("fail to assign tran type info", K(ret)); + } else if (OB_FAIL(extra_info_.assign(src.extra_info_))) { + LOG_WARN("fail to assign tran extra info", K(ret)); + } else { + type_class_ = src.type_class_; } } return ret; } +int ObTZRevertTypeInfo::get_offset_according_abbr(const ObString &tz_abbr_str, + int32_t &offset_sec, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + if (0 == tz_abbr_str.case_compare(info_.abbr_)) { + offset_sec = info_.offset_sec_; + tran_type_id = info_.tran_type_id_; + } else if (0 == tz_abbr_str.case_compare(extra_info_.abbr_)) { + offset_sec = extra_info_.offset_sec_; + tran_type_id = extra_info_.tran_type_id_; + } else { + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + LOG_WARN("invalid abbr", K(tz_abbr_str), KPC(this), K(ret)); + } + return ret; +} + +OB_DEF_SERIALIZE(ObTZRevertTypeInfo) +{ + int ret = ObTZTransitionTypeInfo::serialize(buf, buf_len, pos); + ObString abbr_str(static_cast(strlen(extra_info_.abbr_)), extra_info_.abbr_); + LST_DO_CODE( + OB_UNIS_ENCODE, type_class_, extra_info_.offset_sec_, extra_info_.is_dst_, abbr_str, extra_info_.tran_type_id_); + return ret; +} + +OB_DEF_DESERIALIZE(ObTZRevertTypeInfo) +{ + int ret = ObTZTransitionTypeInfo::deserialize(buf, data_len, pos); + extra_info_.reset(); + ObString abbr_str; + LST_DO_CODE( + OB_UNIS_DECODE, type_class_, extra_info_.offset_sec_, extra_info_.is_dst_, abbr_str, extra_info_.tran_type_id_); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(abbr_str.length() + 1 > OB_MAX_TZ_ABBR_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid abbr_str", K(abbr_str), K(ret)); + } else { + MEMCPY(extra_info_.abbr_, abbr_str.ptr(), abbr_str.length()); + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObTZRevertTypeInfo) +{ + int64_t len = ObTZTransitionTypeInfo::get_serialize_size(); + ObString abbr_str(static_cast(strlen(extra_info_.abbr_)), extra_info_.abbr_); + LST_DO_CODE( + OB_UNIS_ADD_LEN, type_class_, extra_info_.offset_sec_, extra_info_.is_dst_, abbr_str, extra_info_.tran_type_id_); + return len; +} + +/* +int ObTZInfoMap::init(const lib::ObLabel& label) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_FAIL(id_map_.init(label))) { + LOG_WARN("fail to init id map", K(ret)); + } else if (OB_FAIL(name_map_.init(label))) { + LOG_WARN("fail to init name map", K(ret)); + } else { + inited_ = true; + } + return ret; +} +*/ + +int ObTZInfoMap::reset() +{ + int ret = OB_SUCCESS; + id_map_.reset(); + name_map_.reset(); + return ret; +} + +static bool print_tz_info(ObTZIDKey &key, ObTimeZoneInfoPos *tz_info) +{ + int ret = OB_SUCCESS; + UNUSED(key); + if (OB_ISNULL(tz_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is NULL", K(tz_info)); + } else { + LOG_INFO("dump current time zone info", KPC(tz_info)); + } + return OB_SUCCESS == ret; +} + +int ObTZInfoMap::print_tz_info_map() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(id_map_.for_each(print_tz_info))) { + LOG_WARN("fail to call for_each", K(ret)); + } + return ret; +} + +int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_info_by_id) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(tz_info_by_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz_info_by_id should be null here", K(ret)); + } else if (OB_FAIL(id_map_.get(tz_id, tz_info_by_id))) { + LOG_WARN("fail to get tz_info_by_id, should not happened", K(tz_id), K(ret)); + } else { + LOG_DEBUG("succ to get tz_info_by_id", K(tz_id), KPC(tz_info_by_id), K(ret)); + } + return ret; +} + +int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos *&tz_info_by_name) +{ + int ret = OB_SUCCESS; + ObTZNameIDInfo *name_id_info = NULL; + if (OB_NOT_NULL(tz_info_by_name)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("v should be null here", K(ret)); + } else if (OB_FAIL(name_map_.get(ObTZNameKey(tz_name), name_id_info))) { + LOG_WARN("fail to get get_tz_info_by_name", K(tz_name), K(ret)); + } else if (OB_FAIL(get_tz_info_by_id(name_id_info->tz_id_, tz_info_by_name))) { + LOG_WARN("fail to get get_tz_info_by_name", KPC(name_id_info), K(ret)); + } else { + LOG_DEBUG("succ to get get_tz_info_by_name", K(tz_name), KPC(name_id_info), KPC(tz_info_by_name), K(ret)); + } + + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_ERR_UNKNOWN_TIME_ZONE; + } + + if (NULL != name_id_info) { + name_map_.revert(name_id_info); + name_id_info = NULL; + } + + return ret; +} + +ObTZMapWrap::~ObTZMapWrap() +{ + if (!OB_ISNULL(tz_info_map_)) { + tz_info_map_->dec_ref_count(); + } +} + +void ObTZMapWrap::set_tz_map(const common::ObTZInfoMap *timezone_info_map) +{ + if (OB_NOT_NULL(tz_info_map_)) { + tz_info_map_->dec_ref_count(); + } + ObTZInfoMap *non_const_tz_map = const_cast(timezone_info_map); + if (!OB_ISNULL(timezone_info_map)) { + non_const_tz_map->inc_ref_count(); + } + tz_info_map_ = non_const_tz_map; +} + +int ObTimeZoneInfo::assign(const ObTimeZoneInfo &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + this->reset(); + tz_id_ = src.tz_id_; + offset_ = src.offset_; + error_on_overlap_time_ = src.error_on_overlap_time_; + } + return ret; +} + +int ObTimeZoneInfo::set_timezone(const ObString &str) +{ + int ret = OB_SUCCESS; + int ret_more = OB_SUCCESS; + if (OB_FAIL(ObTimeConverter::str_to_offset(str, offset_, ret_more, true, true))) { + LOG_WARN("invalid time zone offset", K(ret), K(str)); + } else { + tz_id_ = 0; + } + return ret; +} + int ObTimeZoneInfo::get_timezone_offset(int64_t value, int32_t &offset) const { int ret = OB_SUCCESS; @@ -39010,15 +39368,511 @@ int ObTimeZoneInfo::get_timezone_offset(int64_t value, int32_t &offset) const return ret; } -int ObTimeZoneInfo::get_timezone_id(const ObString &str, int32_t &tz_id) +int ObTimeZoneInfo::get_timezone_sub_offset(int64_t value, + const ObString &tz_abbr_str, + int32_t &offset_sec, + int32_t &tz_id, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + UNUSED(tz_abbr_str); + if (tz_id_ > 0) { + ret = OB_NOT_SUPPORTED; + } else { + tz_id = OB_INVALID_INDEX; + tran_type_id = OB_INVALID_INDEX; + ret = get_timezone_offset(value, offset_sec); + } + return ret; +} + +int ObTimeZoneInfo::get_timezone_offset(int64_t value, + int32_t &offset_sec, + common::ObString &tz_abbr_str, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + UNUSED(value); + offset_sec = offset_; + tz_abbr_str.reset(); + tran_type_id = common::OB_INVALID_INDEX; + return ret; +} + +int ObTimeZoneInfo::timezone_to_str(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; - UNUSED(str); - tz_id = 0; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < 0) || OB_UNLIKELY(buf_len < pos)) { + ret = OB_ERR_UNEXPECTED; + } else { + int32_t offset_min = static_cast(SEC_TO_MIN(offset_)); + const char *fmt_str = (offset_min < 0 ? "-%02d:%02d" : "+%02d:%02d"); + ret = databuff_printf(buf, buf_len, pos, fmt_str, abs(offset_min) / 60, abs(offset_min) % 60); + } return ret; } OB_SERIALIZE_MEMBER(ObTimeZoneInfo, tz_id_, offset_); + +void ObTimeZoneInfoPos::reset() +{ + tz_id_ = OB_INVALID_TZ_ID; + default_type_.reset(); + curr_idx_ = 0; + tz_tran_types_[0].reset(); + tz_tran_types_[1].reset(); + tz_revt_types_[0].reset(); + tz_revt_types_[1].reset(); + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); +} + +int ObTimeZoneInfoPos::assign(const ObTimeZoneInfoPos &src) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + this->reset(); + tz_id_ = src.tz_id_; + curr_idx_ = src.get_curr_idx(); + if (OB_FAIL(default_type_.assign(src.default_type_))) { + LOG_WARN("fail to assign default type", K(ret)); + } else if (OB_FAIL(tz_tran_types_[get_curr_idx() % 2].assign(src.get_tz_tran_types()))) { + LOG_WARN("fail to assign tz_tran_types", K(ret)); + } else if (OB_FAIL(tz_revt_types_[get_curr_idx() % 2].assign(src.get_tz_revt_types()))) { + LOG_WARN("fail to assign tz_revt_types", K(ret)); + } else { + MEMCPY(tz_name_, src.tz_name_, OB_MAX_TZ_NAME_LEN); + } + } + return ret; +} + +int ObTimeZoneInfoPos::add_tran_type_info(const ObTZTransitionTypeInfo &type_info) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(tz_tran_types_[get_curr_idx() % 2].push_back(type_info))) { + LOG_WARN("fail to push back type info", K(type_info), K(ret)); + } + return ret; +} + +int ObTimeZoneInfoPos::set_default_tran_type(const ObTZTransitionTypeInfo &tran_type) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(default_type_.assign(tran_type))) { + LOG_WARN("fail to assign tran type", K(tran_type), K(ret)); + } + return ret; +} + +int ObTimeZoneInfoPos::get_tz_name(ObString &tz_name) const +{ + int ret = OB_SUCCESS; + int64_t str_length = strlen(tz_name_); + if (OB_UNLIKELY(false == is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is invalid", K(ret)); + } else if (OB_UNLIKELY(str_length + 1 > OB_MAX_TZ_NAME_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tz_name", K(str_length), K(OB_MAX_TZ_NAME_LEN), K(ret)); + } else { + tz_name.assign_ptr(tz_name_, static_cast(str_length)); + } + return ret; +} + +int ObTimeZoneInfoPos::set_tz_name(const char *name, int64_t name_len) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(name) + || OB_ISNULL(tz_name_) + || OB_UNLIKELY(name_len < 0 + || name_len + 1 > OB_MAX_TZ_NAME_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid parameter", K(name_len), K(OB_MAX_TZ_NAME_LEN), K(ret)); + } else { + MEMCPY(tz_name_, name, name_len); + tz_name_[name_len] = 0; + } + return ret; +} + +int ObTimeZoneInfoPos::compare_upgrade(const ObTimeZoneInfoPos &other, bool &is_equal) const +{ + int ret = OB_SUCCESS; + is_equal = false; + const common::ObSArray &other_type = other.get_tz_tran_types(); + const common::ObSArray &tz_tran_types = get_tz_tran_types(); + + if (OB_UNLIKELY(tz_id_ != other.get_tz_id()) + || OB_UNLIKELY(0 != other.get_tz_name().compare(tz_name_)) + || OB_UNLIKELY(default_type_ != other.get_default_trans_type()) + || OB_UNLIKELY(tz_tran_types.count() > other_type.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("it should not happened", KPC(this), K(other), K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < tz_tran_types.count(); i++) { + if (OB_UNLIKELY(other_type[i] != tz_tran_types[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("it should not happened", K(other_type[i]), K(tz_tran_types[i]), K(ret)); + } + } + + if (OB_SUCC(ret) && tz_tran_types.count() == other.get_tz_tran_types().count()) { + is_equal = true; + } + } + return ret; +} + +int ObTimeZoneInfoPos::find_time_range(int64_t value, + const common::ObIArray &tz_tran_types, + int64_t &type_idx) const +{ + int ret = OB_SUCCESS; + int64_t higher_bound = tz_tran_types.count(); + int64_t lower_bound = 0; + int64_t i = 0; + while (higher_bound - lower_bound > 1) { + i = (lower_bound + higher_bound) >> 1; + if (tz_tran_types.at(i).lower_time_ <= value) { + lower_bound = i; + } else { + higher_bound = i; + } + } + type_idx = lower_bound; + return ret; +} + +int ObTimeZoneInfoPos::find_revt_time_range(int64_t value, + const common::ObIArray &tz_revt_types, + int64_t &type_idx) const +{ + int ret = OB_SUCCESS; + int64_t higher_bound = tz_revt_types.count(); + int64_t lower_bound = 0; + int64_t i = 0; + while (higher_bound - lower_bound > 1) { + i = (lower_bound + higher_bound) >> 1; + if (tz_revt_types.at(i).lower_time_ <= value) { + lower_bound = i; + } else { + higher_bound = i; + } + } + type_idx = lower_bound; + return ret; +} + +int ObTimeZoneInfoPos::get_timezone_offset(int64_t value, + int32_t &offset_sec, + common::ObString &tz_abbr_str, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + const common::ObSArray &tz_tran_types = get_tz_tran_types(); + int64_t type_cnt = tz_tran_types.count(); + int64_t type_idx = 0; + if (OB_UNLIKELY(false == is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is invalid", K(ret)); + } else if (0 == type_cnt || value < tz_tran_types.at(0).lower_time_) { + offset_sec = default_type_.info_.offset_sec_; + tz_abbr_str.assign_ptr(default_type_.info_.abbr_, static_cast(strlen(default_type_.info_.abbr_))); + tran_type_id = default_type_.info_.tran_type_id_; + } else if (OB_FAIL(find_time_range(value, tz_tran_types, type_idx))) { + LOG_WARN("fail to find time range", K(ret)); + } else if (OB_UNLIKELY(type_idx < 0 || type_idx >= tz_tran_types.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type idx", K(type_idx), K(tz_tran_types), K(ret)); + } else { + const ObTZTransitionStruct &info = tz_tran_types.at(type_idx).info_; + offset_sec = info.offset_sec_; + tz_abbr_str.assign_ptr(info.abbr_, static_cast(strlen(info.abbr_))); + tran_type_id = info.tran_type_id_; + } + return ret; +} + +int ObTimeZoneInfoPos::get_timezone_offset(int64_t value, int32_t &offset_sec) const +{ + common::ObString tz_abbr_str; + int32_t tran_type_id = OB_INVALID_INDEX; + return get_timezone_offset(value, offset_sec, tz_abbr_str, tran_type_id); +} + +int ObTimeZoneInfoPos::find_offset_range(const int32_t tran_type_id, + const common::ObIArray &tz_tran_types, + int64_t &type_idx) const +{ + int ret = OB_SUCCESS; + int64_t count = tz_tran_types.count(); + bool found = false; + type_idx = OB_INVALID_INDEX; + for (int64_t i = 0; !found && i < count; ++i) { + if (tz_tran_types.at(i).info_.tran_type_id_ == tran_type_id) { + found = true; + type_idx = i; + } + } + + return ret; +} + +int ObTimeZoneInfoPos::get_timezone_offset(const int32_t tran_type_id, + common::ObString &tz_abbr_str, + int32_t &offset_sec) const +{ + int ret = OB_SUCCESS; + const common::ObSArray &tz_tran_types = get_tz_tran_types(); + int64_t type_idx = 0; + if (OB_UNLIKELY(false == is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is invalid", K(ret)); + } else if (tran_type_id == default_type_.info_.tran_type_id_) { + offset_sec = default_type_.info_.offset_sec_; + tz_abbr_str.assign_ptr(default_type_.info_.abbr_, static_cast(strlen(default_type_.info_.abbr_))); + } else if (OB_FAIL(find_offset_range(tran_type_id, tz_tran_types, type_idx))) { + LOG_WARN("fail to find time range", K(ret)); + } else if (OB_UNLIKELY(type_idx < 0 || type_idx >= tz_tran_types.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type idx", K(type_idx), K(tz_tran_types), K(ret)); + } else { + const ObTZTransitionStruct &info = tz_tran_types.at(type_idx).info_; + offset_sec = info.offset_sec_; + tz_abbr_str.assign_ptr(info.abbr_, static_cast(strlen(info.abbr_))); + } + + return ret; +} + +int ObTimeZoneInfoPos::get_timezone_sub_offset(int64_t value, + const ObString &tz_abbr_str, + int32_t &offset_sec, + int32_t &tz_id, + int32_t &tran_type_id) const +{ + int ret = OB_SUCCESS; + const common::ObSArray &tz_revt_types = get_tz_revt_types(); + tz_id = static_cast(tz_id_); + int64_t type_idx = 0; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is invalid", K(ret)); + } else if (OB_FAIL(find_revt_time_range(value, tz_revt_types, type_idx))) { + LOG_WARN("fail to find time range", K(ret)); + } else if (OB_UNLIKELY(type_idx < 0 || type_idx >= tz_revt_types.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected type idx", K(type_idx), K(tz_revt_types), K(ret)); + } else { + const ObTZRevertTypeInfo &revt_type_info = tz_revt_types.at(type_idx); + if (OB_UNLIKELY(revt_type_info.is_gap())) { // gap + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + LOG_WARN("fail to get offset, value may be in gap range", K(value), K(type_idx), K(revt_type_info), K(ret)); + } else if (OB_UNLIKELY(revt_type_info.is_overlap())) { // overlap + if (OB_LIKELY(tz_abbr_str.empty())) { + if (error_on_overlap_time_) { + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + LOG_WARN( + "fail to get offset, value may be in overlap range", K(value), K(type_idx), K(revt_type_info), K(ret)); + } else { // if error_on_overlap_time_ == false, + // oracle mode : use standard offset, mysql mode: use daylight saving time + if (lib::is_oracle_mode()) { + offset_sec = revt_type_info.extra_info_.offset_sec_; + tran_type_id = revt_type_info.extra_info_.tran_type_id_; + } else { + offset_sec = revt_type_info.info_.offset_sec_; + tran_type_id = revt_type_info.info_.tran_type_id_; + } + } + } else if (OB_FAIL(revt_type_info.get_offset_according_abbr(tz_abbr_str, offset_sec, tran_type_id))) { + LOG_WARN("fail to get offset according to abbr", K(tz_abbr_str), K(revt_type_info), K(ret)); + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + } + } else if (revt_type_info.is_normal()) { // normal + if (OB_LIKELY(tz_abbr_str.empty())) { + offset_sec = revt_type_info.info_.offset_sec_; + tran_type_id = revt_type_info.info_.tran_type_id_; + } else if (OB_FAIL(revt_type_info.get_offset_according_abbr(tz_abbr_str, offset_sec, tran_type_id))) { + LOG_WARN("fail to get offset according to abbr", K(tz_abbr_str), K(revt_type_info), K(ret)); + ret = OB_ERR_UNEXPECTED_TZ_TRANSITION; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected revert type info", K(revt_type_info), K(ret)); + } + } + return ret; +} + +int ObTimeZoneInfoPos::calc_revt_types() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(false == is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tz info is invalid", K(ret)); + } else { + common::ObSArray &tz_revt_types = tz_revt_types_[get_curr_idx() % 2]; + tz_revt_types.reset(); + ObTZRevertTypeInfo revt_type_info; + const common::ObSArray &tz_tran_types = get_tz_tran_types(); + + // add first revert type, type info is from default type + revt_type_info.type_class_ = ObTZRevertTypeInfo::NORMAL; + revt_type_info.lower_time_ = DATETIME_MIN_VAL; + if (OB_FAIL(revt_type_info.assign_normal(default_type_))) { + LOG_WARN("fail to assign transition type info", K(ret)); + } else if (OB_FAIL(tz_revt_types.push_back(revt_type_info))) { + LOG_WARN("fail to push back revert type info", K(revt_type_info), K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < tz_tran_types.count(); ++i) { + const ObTZTransitionTypeInfo &cur_tran_type_info = tz_tran_types.at(i); + int64_t last_high_time = (cur_tran_type_info.lower_time_ - 1) + revt_type_info.info_.offset_sec_; + int64_t cur_lower_time = cur_tran_type_info.lower_time_ + cur_tran_type_info.info_.offset_sec_; + if (last_high_time + 1 > cur_lower_time) { // add overlap revert_type_info + revt_type_info.type_class_ = + ObTZRevertTypeInfo::OVERLAP; // no need reset revt_type_info, reuse the normal_type_info + revt_type_info.lower_time_ = cur_lower_time; + if (OB_FAIL(revt_type_info.assign_extra(cur_tran_type_info))) { + LOG_WARN("fail to assign extra type info", K(cur_tran_type_info), K(ret)); + } else if (OB_FAIL(tz_revt_types.push_back(revt_type_info))) { + LOG_WARN("fail to push back revert type info", K(revt_type_info), K(ret)); + } + } else if (last_high_time + 1 < cur_lower_time) { // add gap revert_type_info + revt_type_info.reset(); // type_info should be empty + revt_type_info.type_class_ = ObTZRevertTypeInfo::GAP; + revt_type_info.lower_time_ = last_high_time + 1; + if (OB_FAIL(tz_revt_types.push_back(revt_type_info))) { + LOG_WARN("fail to push back revert type info", K(revt_type_info), K(ret)); + } + } else { /*do nothing*/ + } + + if (OB_SUCC(ret)) { + // add normal revert_type_info + bool is_overlap = ObTZRevertTypeInfo::OVERLAP == revt_type_info.type_class_; + revt_type_info.reset(); + revt_type_info.type_class_ = ObTZRevertTypeInfo::NORMAL; + revt_type_info.lower_time_ = is_overlap ? last_high_time + 1 : cur_lower_time; + if (OB_FAIL(revt_type_info.assign_normal(cur_tran_type_info))) { + LOG_WARN("fail to assign normal type info", K(ret)); + } else if (OB_FAIL(tz_revt_types.push_back(revt_type_info))) { + LOG_WARN("fail to push back revert type info", K(revt_type_info), K(ret)); + } + } + } // for + } + return ret; +} + +int ObTimeZoneInfoPos::timezone_to_str(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < 0) || OB_UNLIKELY(buf_len < pos)) { + ret = OB_ERR_UNEXPECTED; + } else { + const size_t tz_len = strlen(tz_name_); + if (OB_UNLIKELY((pos + tz_len + 1) > buf_len)) { + ret = OB_BUF_NOT_ENOUGH; + LOG_WARN("buff size is not enough", K(pos), K(tz_len), K(buf_len), KPC(this), K(ret)); + } else { + memcpy(buf + pos, tz_name_, tz_len); + pos += tz_len; + buf[tz_len] = '\0'; + } + } + return ret; +} + +OB_DEF_SERIALIZE(ObTimeZoneInfoPos) +{ + int ret = OB_SUCCESS; + ObString tz_name_str(static_cast(strlen(tz_name_)), tz_name_); + const common::ObSArray &tz_tran_types = get_tz_tran_types(); + const common::ObSArray &tz_revt_types = get_tz_revt_types(); + LST_DO_CODE(OB_UNIS_ENCODE, tz_id_, default_type_, tz_tran_types, tz_revt_types, tz_name_str); + return ret; +} + +OB_DEF_DESERIALIZE(ObTimeZoneInfoPos) +{ + int ret = OB_SUCCESS; + ObString tz_name_str; + curr_idx_ = 0; + common::ObSArray &tz_tran_types = tz_tran_types_[0]; + common::ObSArray &tz_revt_types = tz_revt_types_[0]; + LST_DO_CODE(OB_UNIS_DECODE, tz_id_, default_type_, tz_tran_types, tz_revt_types, tz_name_str); + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(tz_name_str.length() + 1 > OB_MAX_TZ_NAME_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tz_name_str", K(tz_name_str)); + } else { + MEMCPY(tz_name_, tz_name_str.ptr(), tz_name_str.length()); + tz_name_[tz_name_str.length()] = 0; + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObTimeZoneInfoPos) +{ + int64_t len = 0; + ObString tz_name_str(static_cast(strlen(tz_name_)), tz_name_); + LST_DO_CODE(OB_UNIS_ADD_LEN, tz_id_, default_type_, get_tz_tran_types(), get_tz_revt_types(), tz_name_str); + return len; +} + +ObTimeZoneInfoPos* ObTZIDPosAlloc::alloc_value() +{ + return op_alloc(ObTimeZoneInfoPos); +} + +void ObTZIDPosAlloc::free_value(ObTimeZoneInfoPos *tz_info) +{ + op_free(tz_info); + tz_info = NULL; +} + +ObTZIDHashNode* ObTZIDPosAlloc::alloc_node(ObTimeZoneInfoPos *value) +{ + UNUSED(value); + return op_alloc(ObTZIDHashNode); +} + +void ObTZIDPosAlloc::free_node(ObTZIDHashNode *node) +{ + if (NULL != node) { + op_free(node); + node = NULL; + } +} + +ObTZNameIDInfo* ObTZNameIDAlloc::alloc_value() +{ + return op_alloc(ObTZNameIDInfo); +} + +void ObTZNameIDAlloc::free_value(ObTZNameIDInfo *info) +{ + op_free(info); + info = NULL; +} + +ObTZNameHashNode* ObTZNameIDAlloc::alloc_node(ObTZNameIDInfo *value) +{ + UNUSED(value); + return op_alloc(ObTZNameHashNode); +} + +void ObTZNameIDAlloc::free_node(ObTZNameHashNode *node) +{ + if (NULL != node) { + op_free(node); + node = NULL; + } +} + + } // end of common } // end of oceanbase diff --git a/src/lib/timezone/ob_timezone_info.h b/src/lib/timezone/ob_timezone_info.h index a4fb8d8997dfbcbde9b8d62e5b67597bbe9f2888..0c899cf814b42640f78a0cc363bbbcde2c574301 100644 --- a/src/lib/timezone/ob_timezone_info.h +++ b/src/lib/timezone/ob_timezone_info.h @@ -15,12 +15,182 @@ #include "lib/string/ob_string.h" #include "lib/utility/ob_print_utils.h" +#include "lib/hash/ob_link_hashmap.h" +#include "lib/container/ob_array.h" +#include "lib/container/ob_array_serialization.h" +#include "lib/number/ob_number_v2.h" +#include "lib/alloc/alloc_struct.h" namespace oceanbase { namespace common { +class ObOTimestampData { +public: + static const int32_t MIN_OFFSET_MINUTES = -15 * 60 - 59; + static const int32_t MAX_OFFSET_MINUTES = 15 * 60 + 59; + static const int32_t MAX_OFFSET_MINUTES_STRICT = 15 * 60; + static const int32_t MIN_TAIL_NSEC = 0; + static const int32_t MAX_TAIL_NSEC = 999; + static const int32_t BASE_TAIL_NSEC = 1000; + static const int32_t MAX_BIT_OF_TAIL_NSEC = ((1 << 10) - 1); + static const int32_t MAX_BIT_OF_OFFSET_MIN = ((1 << 11) - 1); + static const int32_t MAX_BIT_OF_TZ_ID = ((1 << 11) - 1); + static const int32_t MAX_BIT_OF_TRAN_TYPE_ID = ((1 << 5) - 1); + static const int32_t MIN_TZ_ID = 1; + static const int32_t MAX_TZ_ID = MAX_BIT_OF_TZ_ID; + static const int32_t MIN_TRAN_TYPE_ID = 0; + static const int32_t MAX_TRAN_TYPE_ID = MAX_BIT_OF_TRAN_TYPE_ID; + + struct UnionTZCtx { + void set_tail_nsec(const int32_t value) + { + tail_nsec_ = static_cast(value & MAX_BIT_OF_TAIL_NSEC); + } + void set_tz_id(const int32_t value) + { + tz_id_ = static_cast(value & MAX_BIT_OF_TZ_ID); + } + void set_tran_type_id(const int32_t value) + { + tran_type_id_ = static_cast(value & MAX_BIT_OF_TRAN_TYPE_ID); + } + int32_t get_offset_min() const + { + return (is_neg_offset_ ? (0 - static_cast(offset_min_)) : static_cast(offset_min_)); + } + void set_offset_min(const int32_t value) + { + int32_t tmp_value = value; + if (value < 0) { + tmp_value = 0 - value; + is_neg_offset_ = 1; + } else { + is_neg_offset_ = 0; + } + offset_min_ = static_cast(tmp_value & MAX_BIT_OF_OFFSET_MIN); + } + + union { + uint32_t desc_; + struct { + union { + uint16_t time_desc_; + struct { + uint16_t tail_nsec_ : 10; // append nanosecond to the tailer, [0, 999] + uint16_t version_ : 2; // default 0 + uint16_t store_tz_id_ : 1; // true mean store tz_id + uint16_t is_null_ : 1; // oracle null timestamp + uint16_t time_reserved_ : 2; // reserved + }; + }; + union { + uint16_t tz_desc_; + struct { + uint16_t is_neg_offset_ : 1; // reserved + uint16_t offset_min_ : 11; // tz offset min + uint16_t offset_reserved_ : 4; // reserved + }; + struct { + uint16_t tz_id_ : 11; // Time_zone_id of oceanbase.__all_time_zone_transition_type, [1, 2047] + uint16_t tran_type_id_ : 5; // Transition_type_id of oceanbase.__all_time_zone_transition_type, [0,31] + }; + }; + }; + }; + } __attribute__((packed)); + +public: + ObOTimestampData() : time_ctx_(), time_us_(0) + {} + ObOTimestampData(const int64_t time_us, const UnionTZCtx tz_ctx) : time_ctx_(tz_ctx), time_us_(time_us) + {} + ~ObOTimestampData() + {} + void reset() + { + memset(this, 0, sizeof(ObOTimestampData)); + } + bool is_null_value() const + { + return time_ctx_.is_null_; + } + void set_null_value() + { + time_ctx_.is_null_ = 1; + } + static bool is_valid_offset_min(const int32_t offset_min) + { + return MIN_OFFSET_MINUTES <= offset_min && offset_min <= MAX_OFFSET_MINUTES; + } + static bool is_valid_offset_min_strict(const int32_t offset_min) + { + return MIN_OFFSET_MINUTES <= offset_min && offset_min <= MAX_OFFSET_MINUTES_STRICT; + } + static bool is_valid_tz_id(const int32_t tz_id) + { + return (MIN_TZ_ID <= tz_id && tz_id <= MAX_TZ_ID); + } + static bool is_valid_tran_type_id(const int32_t tran_type_id) + { + return (MIN_TRAN_TYPE_ID <= tran_type_id && tran_type_id <= MAX_TRAN_TYPE_ID); + } + inline int compare(const ObOTimestampData &other) const + { + int result = 0; + if (time_us_ > other.time_us_) { + result = 1; + } else if (time_us_ < other.time_us_) { + result = -1; + } else if (time_ctx_.tail_nsec_ > other.time_ctx_.tail_nsec_) { + result = 1; + } else if (time_ctx_.tail_nsec_ < other.time_ctx_.tail_nsec_) { + result = -1; + } else { + result = 0; + } + return result; + } + + inline bool operator<(const ObOTimestampData &other) const + { + return compare(other) < 0; + } + + inline bool operator<=(const ObOTimestampData &other) const + { + return compare(other) <= 0; + } + + inline bool operator>(const ObOTimestampData &other) const + { + return compare(other) > 0; + } + + inline bool operator>=(const ObOTimestampData &other) const + { + return compare(other) >= 0; + } + + inline bool operator==(const ObOTimestampData &other) const + { + return compare(other) == 0; + } + + inline bool operator!=(const ObOTimestampData &other) const + { + return compare(other) != 0; + } + + DECLARE_TO_STRING; + +public: + UnionTZCtx time_ctx_; // time ctx, such as version, tail_ns, tz_id... + int64_t time_us_; // full time with usec, same as timestamp +} __attribute__((packed)); + + struct ObTimeZoneName { ObString name_; @@ -38,34 +208,598 @@ struct ObTimeZoneTrans static const int32_t INVALID_TZ_OFF = INT32_MAX; +struct ObTZTransitionStruct { + ObTZTransitionStruct() : offset_sec_(0), tran_type_id_(common::OB_INVALID_INDEX), is_dst_(false) + { + MEMSET(abbr_, 0, common::OB_MAX_TZ_ABBR_LEN); + } + ~ObTZTransitionStruct() + {} + int assign(const ObTZTransitionStruct &src) + { + int ret = OB_SUCCESS; + if (OB_LIKELY(this != &src)) { + MEMCPY(this, &src, sizeof(ObTZTransitionStruct)); + } + return ret; + } + void reset() + { + offset_sec_ = 0; + tran_type_id_ = common::OB_INVALID_INDEX; + is_dst_ = false; + MEMSET(abbr_, 0, common::OB_MAX_TZ_ABBR_LEN); + } + void set_tz_abbr(const common::ObString &abbr) + { + MEMSET(abbr_, 0, common::OB_MAX_TZ_ABBR_LEN); + if (!abbr.empty()) { + const int64_t min_len = std::min(static_cast(abbr.length()), common::OB_MAX_TZ_ABBR_LEN - 1); + MEMCPY(abbr_, abbr.ptr(), min_len); + } + } + bool operator==(const ObTZTransitionStruct &other) const + { + return (offset_sec_ == other.offset_sec_ + && tran_type_id_ == other.tran_type_id_ + && is_dst_ == other.is_dst_ + && 0 == MEMCMP(abbr_, other.abbr_, common::OB_MAX_TZ_ABBR_LEN)); + } + bool operator!=(const ObTZTransitionStruct &other) const + { + return !(*this == other); + } + + TO_STRING_KV(K_(offset_sec), K_(tran_type_id), K_(is_dst), "abbr", common::ObString(abbr_)); + + int32_t offset_sec_; + int32_t tran_type_id_; + bool is_dst_; + char abbr_[common::OB_MAX_TZ_ABBR_LEN]; +}; + +class ObTZTransitionTypeInfo { + OB_UNIS_VERSION(1); + +public: + ObTZTransitionTypeInfo() : lower_time_(common::OB_INVALID_TZ_TRAN_TIME), info_() + {} + virtual ~ObTZTransitionTypeInfo() + {} + int assign(const ObTZTransitionTypeInfo &src); + void reset(); + + void set_tz_abbr(const common::ObString &abbr) + { + info_.set_tz_abbr(abbr); + } + OB_INLINE bool is_dst() const + { + return info_.is_dst_; + } + bool operator==(const ObTZTransitionTypeInfo &other) const + { + return (lower_time_ == other.lower_time_ && info_ == other.info_); + } + bool operator!=(const ObTZTransitionTypeInfo &other) const + { + return !(*this == other); + } + virtual int get_offset_according_abbr( + const common::ObString &tz_abbr_str, int32_t &offset_sec, int32_t &tran_type_id) const; + VIRTUAL_TO_STRING_KV(K_(lower_time), K_(info)); + +private: + DISALLOW_COPY_AND_ASSIGN(ObTZTransitionTypeInfo); + +public: + int64_t lower_time_; + ObTZTransitionStruct info_; +}; + +class ObTZRevertTypeInfo : public ObTZTransitionTypeInfo { + OB_UNIS_VERSION(1); + +public: + enum TypeInfoClass { NONE = 0, NORMAL, OVERLAP, GAP }; + +public: + ObTZRevertTypeInfo() : ObTZTransitionTypeInfo(), type_class_(NONE), extra_info_() + {} + virtual ~ObTZRevertTypeInfo() + {} + void reset(); + int assign_normal(const ObTZTransitionTypeInfo &src); + int assign_extra(const ObTZTransitionTypeInfo &src); + int assign(const ObTZRevertTypeInfo &src); + OB_INLINE bool is_normal() const + { + return NORMAL == type_class_; + } + OB_INLINE bool is_gap() const + { + return GAP == type_class_; + } + OB_INLINE bool is_overlap() const + { + return OVERLAP == type_class_; + } + int get_offset_according_abbr(const common::ObString &tz_abbr_str, int32_t &offset_sec, int32_t &tran_type_id) const; + bool operator==(const ObTZRevertTypeInfo &other) const + { + return (type_class_ == other.type_class_ + && extra_info_ == other.extra_info_ + && lower_time_ == other.lower_time_ + && info_ == other.info_); + } + bool operator!=(const ObTZRevertTypeInfo &other) const + { + return !(*this == other); + } + + VIRTUAL_TO_STRING_KV(K_(lower_time), K_(info), K_(type_class), K_(extra_info)); + +private: + DISALLOW_COPY_AND_ASSIGN(ObTZRevertTypeInfo); + +public: + TypeInfoClass type_class_; + ObTZTransitionStruct extra_info_; +}; + +class ObTZIDKey { +public: + ObTZIDKey() : tz_id_(0) + {} + ObTZIDKey(const int64_t tz_id) : tz_id_(tz_id) + {} + uint64_t hash() const + { + return common::murmurhash(&tz_id_, sizeof(tz_id_), 0); + }; + int compare(const ObTZIDKey& r) + { + int cmp = 0; + if (tz_id_ < r.tz_id_) { + cmp = -1; + } else if (tz_id_ == r.tz_id_) { + cmp = 0; + } else { + cmp = 1; + } + return cmp; + } + TO_STRING_KV(K_(tz_id)); + +public: + int64_t tz_id_; +}; + +class ObTZNameKey { +public: + ObTZNameKey(const common::ObString &tz_key_str); + ObTZNameKey(const ObTZNameKey &key); + ObTZNameKey() + { + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + } + ~ObTZNameKey() + {} + void reset(); + int assign(const ObTZNameKey &key); + void operator=(const ObTZNameKey &key); + int compare(const ObTZNameKey &key) const + { + ObString self_str(strlen(tz_name_), tz_name_); + return self_str.case_compare(key.tz_name_); + } + bool operator==(const ObTZNameKey &key) const + { + return 0 == compare(key); + } + + uint64_t hash(uint64_t seed = 0) const; + TO_STRING_KV("tz_name", common::ObString(common::OB_MAX_TZ_NAME_LEN, tz_name_)); + +private: + char tz_name_[common::OB_MAX_TZ_NAME_LEN]; +}; + + +class ObTZInfoMap; + +class ObTZMapWrap { +public: + ObTZMapWrap() : tz_info_map_(NULL) + {} + ~ObTZMapWrap(); + const ObTZInfoMap* get_tz_map() const + { + return tz_info_map_; + } + void set_tz_map(const common::ObTZInfoMap *tz_info_map); + +private: + ObTZInfoMap *tz_info_map_; +}; + class ObTimeZoneInfo { OB_UNIS_VERSION(1); public: ObTimeZoneInfo() - : tz_id_(0), offset_(0) + : error_on_overlap_time_(false), + tz_map_wrap_(), + tz_id_(common::OB_INVALID_TZ_ID), + offset_(0) {} + int assign(const ObTimeZoneInfo &src); int set_timezone(const ObString &str); - int get_timezone_offset(int64_t value, int32_t &offset) const; + int32_t get_tz_id() const + { + return tz_id_; + } + void set_offset(int32_t offset) + { + offset_ = offset; + } + virtual int get_timezone_offset(int64_t value, int32_t &offset_sec) const; + virtual int get_timezone_sub_offset( + int64_t value, const ObString &tz_abbr_str, int32_t &offset_sec, int32_t &tz_id, int32_t &tran_type_id) const; + virtual int get_timezone_offset( + int64_t value, int32_t &offset_sec, common::ObString &tz_abbr_str, int32_t &tran_type_id) const; + virtual int timezone_to_str(char *buf, const int64_t len, int64_t &pos) const; + + void set_error_on_overlap_time(bool is_error) + { + error_on_overlap_time_ = is_error; + } + bool is_error_on_overlap_time() const + { + return error_on_overlap_time_; + } + void set_tz_info_map(const ObTZInfoMap *tz_info_map) + { + tz_map_wrap_.set_tz_map(tz_info_map); + } + ObTZMapWrap& get_tz_map_wrap() + { + return tz_map_wrap_; + } + const ObTZInfoMap* get_tz_info_map() const + { + return tz_map_wrap_.get_tz_map(); + } + void reset() { - tz_id_ = 0; + tz_id_ = common::OB_INVALID_TZ_ID; offset_ = 0; + error_on_overlap_time_ = false; + tz_map_wrap_.set_tz_map(NULL); } - TO_STRING_KV(N_ID, tz_id_, N_OFFSET, offset_); + TO_STRING_KV(N_ID, tz_id_, N_OFFSET, offset_, N_ERROR_ON_OVERLAP_TIME, error_on_overlap_time_); private: static ObTimeZoneName TIME_ZONE_NAMES[]; static ObTimeZoneTrans TIME_ZONE_TRANS[]; -private: - static int get_timezone_id(const ObString &str, int32_t &tz_id); +protected: + bool error_on_overlap_time_; + + // do not serialize it + ObTZMapWrap tz_map_wrap_; private: int32_t tz_id_; int32_t offset_; }; +typedef common::LinkHashNode ObTZIDHashNode; +typedef common::LinkHashNode ObTZNameHashNode; +typedef common::LinkHashValue ObTZIDHashValue; +typedef common::LinkHashValue ObTZNameHashValue; + +class ObTimeZoneInfo; +class ObTimeZoneInfoPos : public ObTimeZoneInfo, public ObTZIDHashValue { + OB_UNIS_VERSION(1); + +public: + ObTimeZoneInfoPos() + : tz_id_(common::OB_INVALID_TZ_ID), + default_type_(), + tz_tran_types_(), + tz_revt_types_(), + curr_idx_(0) + { + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + } + virtual ~ObTimeZoneInfoPos() + {} + int assign(const ObTimeZoneInfoPos &src); + void reset(); + bool is_valid() const + { + return tz_id_ != common::OB_INVALID_TZ_ID; + } + int compare_upgrade(const ObTimeZoneInfoPos &other, bool &is_equal) const; + + virtual int get_timezone_offset( + int64_t value, int32_t &offset_sec, common::ObString &tz_abbr_str, int32_t &tran_type_id) const; + int get_timezone_offset(const int32_t tran_type_id, common::ObString &tz_abbr_str, int32_t &offset_sec) const; + virtual int get_timezone_offset(int64_t value, int32_t &offset_sec) const; + virtual int get_timezone_sub_offset(int64_t value, const common::ObString &tz_abbr_str, int32_t &offset_sec, + int32_t &tz_id, int32_t &tran_type_id) const; + + inline void set_tz_id(int64_t tz_id) + { + tz_id_ = tz_id; + } + inline int64_t get_tz_id() const + { + return tz_id_; + } + int set_tz_name(const char *name, int64_t name_len); + int get_tz_name(common::ObString &tz_name) const; + common::ObString get_tz_name() const + { + return common::ObString(tz_name_); + } + int add_tran_type_info(const ObTZTransitionTypeInfo &type_info); + int set_default_tran_type(const ObTZTransitionTypeInfo &tran_type); + inline const ObTZTransitionTypeInfo &get_default_trans_type() const + { + return default_type_; + } + inline int32_t get_curr_idx() const + { + return curr_idx_; + } + inline int32_t get_next_idx() const + { + return curr_idx_ + 1; + } + inline void inc_curr_idx() + { + ++curr_idx_; + } + const common::ObSArray& get_tz_tran_types() const + { + return tz_tran_types_[get_curr_idx() % 2]; + } + const common::ObSArray& get_tz_revt_types() const + { + return tz_revt_types_[get_curr_idx() % 2]; + } + const common::ObSArray& get_next_tz_tran_types() const + { + return tz_tran_types_[get_next_idx() % 2]; + } + const common::ObSArray& get_next_tz_revt_types() const + { + return tz_revt_types_[get_next_idx() % 2]; + } + common::ObSArray& get_next_tz_tran_types() + { + return tz_tran_types_[get_next_idx() % 2]; + } + common::ObSArray& get_next_tz_revt_types() + { + return tz_revt_types_[get_next_idx() % 2]; + } + int calc_revt_types(); + virtual int timezone_to_str(char *buf, const int64_t len, int64_t &pos) const; + VIRTUAL_TO_STRING_KV("tz_name", common::ObString(common::OB_MAX_TZ_NAME_LEN, tz_name_), "tz_id", tz_id_, + "default_transition_type", default_type_, "tz_transition_types", get_tz_tran_types(), "tz_revert_types", + get_tz_revt_types(), K_(curr_idx), "next_tz_transition_types", get_next_tz_tran_types(), "next_tz_revert_types", + get_next_tz_revt_types()); + +private: + int find_time_range(int64_t value, + const common::ObIArray &tz_tran_types, + int64_t &type_idx) const; + int find_offset_range(const int32_t tran_type_id, + const common::ObIArray &tz_tran_types, + int64_t &type_idx) const; + int find_revt_time_range(int64_t value, + const common::ObIArray &tz_revt_types, + int64_t &type_idx) const; + +private: + int64_t tz_id_; + /*default_type_ is used for times smaller than first transition or if + there are no transitions at all.*/ + ObTZTransitionTypeInfo default_type_; + // used for utc time -> local time + common::ObSArray tz_tran_types_[2]; + // used for local time -> utc time + common::ObSArray tz_revt_types_[2]; + uint32_t curr_idx_; + char tz_name_[common::OB_MAX_TZ_NAME_LEN]; +}; + +class ObTZNameIDInfo : public ObTZNameHashValue { +public: + ObTZNameIDInfo() : tz_id_(common::OB_INVALID_TZ_ID) + { + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + } + void set(const int64_t tz_id, const common::ObString &tz_name) + { + tz_id_ = tz_id; + if (tz_name.empty()) { + MEMSET(tz_name_, 0, common::OB_MAX_TZ_NAME_LEN); + } else { + const int64_t min_len = std::min(static_cast(tz_name.length()), common::OB_MAX_TZ_ABBR_LEN - 1); + MEMCPY(tz_name_, tz_name.ptr(), min_len); + tz_name_[min_len] = '\0'; + } + } + + ObTZNameIDInfo(const int64_t tz_id, const common::ObString &tz_name) : tz_id_(tz_id) + { + set(tz_id, tz_name); + } + ~ObTZNameIDInfo() + {} + TO_STRING_KV(K_(tz_id), K_(tz_name)); + +public: + int64_t tz_id_; + char tz_name_[common::OB_MAX_TZ_NAME_LEN]; +}; + +class ObTZIDPosAlloc { +public: + ObTZIDPosAlloc() + {} + ~ObTZIDPosAlloc() + {} + ObTimeZoneInfoPos* alloc_value(); + void free_value(ObTimeZoneInfoPos *tz_info); + + ObTZIDHashNode* alloc_node(ObTimeZoneInfoPos *value); + void free_node(ObTZIDHashNode *node); +}; + +class ObTZNameIDAlloc { +public: + ObTZNameIDAlloc() + {} + ~ObTZNameIDAlloc() + {} + ObTZNameIDInfo* alloc_value(); + void free_value(ObTZNameIDInfo *info); + + ObTZNameHashNode* alloc_node(ObTZNameIDInfo *value); + void free_node(ObTZNameHashNode *node); +}; + +typedef common::ObLinkHashMap ObTZInfoIDPosMap; +typedef common::ObLinkHashMap ObTZInfoNameIDMap; + +class ObTZInfoMap { +public: + ObTZInfoMap() : inited_(false), id_map_(), name_map_(), ref_count_(0) + {} + ~ObTZInfoMap() + {} + //int init(const lib::ObLabel& label); + int reset(); + int print_tz_info_map(); + bool is_inited() + { + return inited_; + } + int get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_info_by_id); + int get_tz_info_by_name(const common::ObString &tz_name, ObTimeZoneInfoPos *&tz_info_by_name); + void free_tz_info_pos(ObTimeZoneInfoPos *&tz_info) + { + id_map_.revert(tz_info); + tz_info = NULL; + } + void inc_ref_count() + { + ATOMIC_INC(&ref_count_); + } + void dec_ref_count() + { + ATOMIC_DEC(&ref_count_); + } + int64_t get_ref_count() + { + return ATOMIC_LOAD64(&ref_count_); + } + +public: + bool inited_; + ObTZInfoIDPosMap id_map_; + ObTZInfoNameIDMap name_map_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObTZInfoMap); + int64_t ref_count_; +}; + +class ObTimeZoneInfoWrap { + enum ObTZInfoClass { NONE = 0, POSITION = 1, OFFSET = 2 }; + OB_UNIS_VERSION(1); + +public: + ObTimeZoneInfoWrap() + : tz_info_pos_(), + tz_info_offset_(), + tz_info_(NULL), + class_(NONE), + cur_version_(0), + error_on_overlap_time_(false) + {} + ~ObTimeZoneInfoWrap() + {} + void reset(); + const ObTimeZoneInfo* get_time_zone_info() const + { + return tz_info_; + } + ObTimeZoneInfoPos& get_tz_info_pos() + { + return tz_info_pos_; + } + const ObTimeZoneInfo& get_tz_info_offset() const + { + return tz_info_offset_; + } + int64_t get_cur_version() const + { + return cur_version_; + } + void set_cur_version(const int64_t version) + { + cur_version_ = version; + } + ObTZInfoClass get_tz_info_class() const + { + return class_; + } + bool is_position_class() const + { + return POSITION == class_; + } + bool is_error_on_overlap_time() const + { + return error_on_overlap_time_; + } + int set_error_on_overlap_time(bool is_error); + int init_time_zone(const ObString &str_val, const int64_t curr_version, ObTZInfoMap &tz_info_map); + void set_tz_info_map(const ObTZInfoMap *tz_info_map); + int deep_copy(const ObTimeZoneInfoWrap &tz_inf_wrap); + void set_tz_info_offset(const int32_t offset) + { + tz_info_offset_.set_offset(offset); + tz_info_ = &tz_info_offset_; + tz_info_->set_error_on_overlap_time(error_on_overlap_time_); + class_ = OFFSET; + } + void set_tz_info_position() + { + tz_info_ = &tz_info_pos_; + tz_info_->set_error_on_overlap_time(error_on_overlap_time_); + class_ = POSITION; + } + VIRTUAL_TO_STRING_KV( + K_(cur_version), "class", class_, KP_(tz_info), K_(error_on_overlap_time), K_(tz_info_pos), K_(tz_info_offset)); + +private: + common::ObTimeZoneInfoPos tz_info_pos_; + common::ObTimeZoneInfo tz_info_offset_; + common::ObTimeZoneInfo *tz_info_; + ObTZInfoClass class_; + int64_t cur_version_; + bool error_on_overlap_time_; +}; + + + } // end of common } // end of oceanbase diff --git a/src/lib/utility/ob_print_utils.cpp b/src/lib/utility/ob_print_utils.cpp index fd69ceb89c0042687ee4cee42114c0e2f3bddab7..a01c9fd80318a25c6e71e27795baba222fac43e2 100644 --- a/src/lib/utility/ob_print_utils.cpp +++ b/src/lib/utility/ob_print_utils.cpp @@ -17,6 +17,20 @@ namespace oceanbase { namespace common { + +char get_xdigit(const char c1) +{ + char ret_char = 0; + if (c1 >= 'a' && c1 <= 'f') { + ret_char = (char)(c1 - 'a' + 10); + } else if (c1 >= 'A' && c1 <= 'F') { + ret_char = (char)(c1 - 'A' + 10); + } else { + ret_char = (char)(c1 - '0'); + } + return (ret_char & 0x0F); +} + char *hex2str(const void *data, const int32_t size) { //TODO: change the log buffer diff --git a/src/lib/utility/ob_print_utils.h b/src/lib/utility/ob_print_utils.h index 312f2b55e2a0dc23155f61105ff29377764a90f9..28d6f12ad7f8e838b0c044111389035a3b5ab2a2 100644 --- a/src/lib/utility/ob_print_utils.h +++ b/src/lib/utility/ob_print_utils.h @@ -26,6 +26,8 @@ namespace common #define COMMA_FORMAT ", " #define WITH_COMMA(format) (with_comma ? COMMA_FORMAT format: format) +char get_xdigit(const char c1); + char *hex2str(const void *data, const int32_t size); int32_t hex_to_str(const void *in_data, const int32_t data_length, void *buff, const int32_t buff_size); @@ -99,15 +101,11 @@ const char *to_cstring(const T &obj) if (OB_ISNULL(buffer)) { LIB_LOG(ERROR, "buffer is NULL"); } else { - if (NULL == &obj) { - snprintf(buffer, BUFFER_SIZE, "NULL"); + pos = to_string(obj, buffer, BUFFER_SIZE -1); + if (pos >= 0 && pos < BUFFER_SIZE) { + buffer[pos] = '\0'; } else { - pos = to_string(obj, buffer, BUFFER_SIZE -1); - if (pos >= 0 && pos < BUFFER_SIZE) { - buffer[pos] = '\0'; - } else { - buffer[0] = '\0'; - } + buffer[0] = '\0'; } } } diff --git a/src/lib/utility/serialization.h b/src/lib/utility/serialization.h index 61800439f279d8471c5e889e309301114b258de5..13b817a9ab8647e6fd9e756e759b51716fabdbf7 100644 --- a/src/lib/utility/serialization.h +++ b/src/lib/utility/serialization.h @@ -1234,6 +1234,63 @@ inline int encode_number_type(char *buf, return err; } +inline int64_t encode_length_otimestamp_tz_type() +{ + return (sizeof(int64_t) + sizeof(uint32_t)); +} +inline int decode_otimestamp_tz_type(const char *buf, + const int64_t buf_len, + int64_t &pos, + int64_t &time_us, + uint32_t &tz_ctx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(decode_i64(buf, buf_len, pos, &time_us))) { + } else if (OB_FAIL(decode_i32(buf, buf_len, pos, (int32_t*)(&tz_ctx)))) { + } + return ret; +} +inline int encode_otimestamp_tz_type(char *buf, + const int64_t buf_len, + int64_t &pos, + const int64_t time_us, + const uint32_t tz_ctx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(encode_i64(buf, buf_len, pos, time_us))) { + } else if (OB_FAIL(encode_i32(buf, buf_len, pos, tz_ctx))) { + } + return ret; +} +inline int64_t encode_length_otimestamp_type() +{ + return (sizeof(int64_t) + sizeof(uint16_t)); +} +inline int decode_otimestamp_type(const char *buf, + const int64_t buf_len, + int64_t &pos, + int64_t &time_us, + uint16_t &time_desc) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(decode_i64(buf, buf_len, pos, &time_us))) { + } else if (OB_FAIL(decode_i16(buf, buf_len, pos, (int16_t*)(&time_desc)))) { + } + return ret; +} +inline int encode_otimestamp_type(char *buf, + const int64_t buf_len, + int64_t &pos, + const int64_t time_us, + const uint16_t time_desc) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(encode_i64(buf, buf_len, pos, time_us))) { + } else if (OB_FAIL(encode_i16(buf, buf_len, pos, time_desc))) { + } + return ret; +} + inline int encode_decimal_type(char *buf, const int64_t buf_len, int64_t &pos, bool is_add, int8_t precision, int8_t scale, int8_t vscale, int8_t nwords, const uint32_t *words) { diff --git a/src/obproxy/Makemodule.am b/src/obproxy/Makemodule.am index e64e30f3206c39c6d009a53ae7f4cd05f0b7faf1..df2de637cb227d566b19745ffea6ee9f05c805c4 100644 --- a/src/obproxy/Makemodule.am +++ b/src/obproxy/Makemodule.am @@ -33,6 +33,7 @@ include obproxy/engine/Makemodule.am include obproxy/optimizer/Makemodule.am include obproxy/executor/Makemodule.am include obproxy/qos/Makemodule.am +include obproxy/omt/Makemodule.am #include obproxy/tests/Makemodule.am pub_source = \ @@ -68,7 +69,8 @@ ${prometheus_sources}\ ${obproxy_executor_sources}\ ${obproxy_optimizer_sources}\ ${obproxy_engine_sources}\ -${qos_sources} +${qos_sources}\ +${omt_sources} obproxy_libobproxy_la_SOURCES :=\ ${obproxy_lib_sources}\ @@ -81,7 +83,7 @@ ${obproxy_sources} nodist_obproxy_libobproxy_la_SOURCES = build_version.c obproxy_obproxy_SOURCES := obproxy/main.cpp ${pub_source} -obproxy_obproxy_LDADD_BASE := obproxy/libobproxy.la lib/compress/libzlib_1.0.la ${BIN_LDFLAGS} ${DEP_DIR}/lib64/libprotobuf.a ${DEP_DIR}/lib/libgrpc.a ${DEP_DIR}/lib/libgrpc++.a ${DEP_DIR}/lib/libgrpc_unsecure.a ${DEP_DIR}/lib/libgrpc++_unsecure.a ${DEP_DIR}/lib/libgrpc++_reflection.a ${DEP_DIR}/lib/libgrpcpp_channelz.a ${DEP_DIR}/lib/libgrpc++_error_details.a ${DEP_DIR}/lib/libgrpc_cronet.a ${DEP_DIR}/lib/libgrpc++_cronet.a ${DEP_DIR}/lib/libgpr.a ${DEP_DIR}/lib/libaddress_sorting.a ${DEP_DIR}/lib/libcares.a ${DEP_DIR}/lib64/libprometheus-cpp-pull.a ${DEP_DIR}/lib64/libprometheus-cpp-core.a ${RUNTIME_DIR}/lib/libob_sql_proxy_parser_static.a ${DEP_DIR}/lib/libssl.a ${DEP_DIR}/lib/libcrypto.a -lpthread ${DEP_DIR}/lib/libcurl.a ${DEP_DIR}/lib/libcrypto.a +obproxy_obproxy_LDADD_BASE := obproxy/libobproxy.la lib/compress/libzlib_1.0.la lib/compress/libzstd_1.3.8.la ${BIN_LDFLAGS} ${DEP_DIR}/lib64/libprotobuf.a ${DEP_DIR}/lib/libgrpc.a ${DEP_DIR}/lib/libgrpc++.a ${DEP_DIR}/lib/libgrpc_unsecure.a ${DEP_DIR}/lib/libgrpc++_unsecure.a ${DEP_DIR}/lib/libgrpc++_reflection.a ${DEP_DIR}/lib/libgrpcpp_channelz.a ${DEP_DIR}/lib/libgrpc++_error_details.a ${DEP_DIR}/lib/libgrpc_cronet.a ${DEP_DIR}/lib/libgrpc++_cronet.a ${DEP_DIR}/lib/libgpr.a ${DEP_DIR}/lib/libaddress_sorting.a ${DEP_DIR}/lib/libcares.a ${DEP_DIR}/lib64/libprometheus-cpp-pull.a ${DEP_DIR}/lib64/libprometheus-cpp-core.a ${RUNTIME_DIR}/lib/libob_sql_proxy_parser_static.a ${DEP_DIR}/lib/libssl.a ${DEP_DIR}/lib/libcrypto.a -lpthread ${DEP_DIR}/lib/libcurl.a ${DEP_DIR}/lib/libcrypto.a ${DEP_DIR}/lib/sqlite/libsqlite3.a obproxy_obproxy_LDFLAGS :=${AM_LDFLAGS} -lpthread -lc -lm -lrt -ldl -L${DEP_DIR}/lib obproxy_obproxy_LDADD := ${obproxy_obproxy_LDADD_BASE} diff --git a/src/obproxy/cmd/Makemodule.am b/src/obproxy/cmd/Makemodule.am index a4b3d27c5edcecbf421fe934f15003c8e680781e..29c172e408a37fe867cec76c6f3a8ff9f056ffcd 100644 --- a/src/obproxy/cmd/Makemodule.am +++ b/src/obproxy/cmd/Makemodule.am @@ -9,6 +9,8 @@ obproxy/cmd/ob_alter_config_handler.h\ obproxy/cmd/ob_alter_config_handler.cpp\ obproxy/cmd/ob_dds_config_handler.h\ obproxy/cmd/ob_dds_config_handler.cpp\ +obproxy/cmd/ob_config_v2_handler.h\ +obproxy/cmd/ob_config_v2_handler.cpp\ obproxy/cmd/ob_alter_resource_handler.h\ obproxy/cmd/ob_alter_resource_handler.cpp\ obproxy/cmd/ob_show_warning_handler.h\ diff --git a/src/obproxy/cmd/ob_alter_config_handler.cpp b/src/obproxy/cmd/ob_alter_config_handler.cpp index f53b211dcc46ed883b83262cf67ffc0a714f0967..5181bcdbde57da3d4a20addeac53c22da447929a 100644 --- a/src/obproxy/cmd/ob_alter_config_handler.cpp +++ b/src/obproxy/cmd/ob_alter_config_handler.cpp @@ -20,6 +20,7 @@ #include "obutils/ob_proxy_config.h" #include "obutils/ob_proxy_reload_config.h" #include "lib/encrypt/ob_encrypted_helper.h" +#include "obutils/ob_config_processor.h" using namespace oceanbase::common; using namespace oceanbase::obmysql; @@ -129,6 +130,16 @@ int ObAlterConfigSetHandler::handle_set_config(int event, void *data) } } + // If it is an ssl switch, it needs to be synchronized to ssl_config_table for compatibility + if (OB_SUCC(ret)) { + if (0 == key_string.case_compare("enable_client_ssl") + || 0 == key_string.case_compare("enable_server_ssl")) { + if (OB_FAIL(get_global_config_processor().store_cloud_config("ssl_config", "*", "*", key_string, value_string))) { + LOG_WARN("store cloud ssl config failed", K(ret)); + } + } + } + //5. rollback if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; @@ -163,7 +174,7 @@ int ObAlterConfigSetHandler::handle_set_config(int event, void *data) if (OB_FAIL(encode_ok_packet(0, capability_))) { WARN_ICMD("fail to encode ok packet", K(ret)); } else { - INFO_ICMD("succ to update config", K(key_string), K(value_string)); + INFO_ICMD("succ to update config", K(key_string)); event_ret = handle_callback(INTERNAL_CMD_EVENTS_SUCCESS, NULL); } } else { diff --git a/src/obproxy/cmd/ob_config_v2_handler.cpp b/src/obproxy/cmd/ob_config_v2_handler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e916b3d2070ea7ab313a1cdfc86f6dc12e522634 --- /dev/null +++ b/src/obproxy/cmd/ob_config_v2_handler.cpp @@ -0,0 +1,197 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cmd/ob_config_v2_handler.h" +#include "obutils/ob_config_processor.h" +#include "proxy/mysql/ob_mysql_sm.h" +#include "rpc/obmysql/ob_mysql_global.h" + +using namespace oceanbase::common; +using namespace oceanbase::obmysql; +using namespace oceanbase::obproxy::event; +using namespace oceanbase::obproxy::proxy; +using namespace oceanbase::obproxy::obutils; + +namespace oceanbase +{ +namespace obproxy +{ + +ObConfigV2Handler::ObConfigV2Handler(ObContinuation *cont, ObMIOBuffer *buf, + const ObInternalCmdInfo &info) + : ObInternalCmdHandler(cont, buf, info), sqlite3_column_name_(), + sqlite3_column_value_(), cmd_type_(info.get_cmd_type()), + capability_(info.get_capability()) +{ + sm_ = reinterpret_cast(cont); + SET_HANDLER(&ObConfigV2Handler::main_handler); +} + +int ObConfigV2Handler::main_handler(int event, void *data) +{ + UNUSED(event); + UNUSED(data); + int event_ret = EVENT_DONE; + int ret = OB_SUCCESS; + switch (cmd_type_) { + case OBPROXY_T_SELECT: + event_ret = handle_select_stmt(); + break; + case OBPROXY_T_REPLACE: + case OBPROXY_T_DELETE: + event_ret = handle_dml_stmt(); + break; + default: + ret = OB_ERR_UNEXPECTED; + WARN_ICMD("unknown type", "cmd_type", get_print_stmt_name(cmd_type_)); + event_ret = internal_error_callback(ret); + } + + return event_ret; +} + +int ObConfigV2Handler::handle_select_stmt() +{ + int event_ret = EVENT_DONE; + int ret = OB_SUCCESS; + ObString sql = sm_->trans_state_.trans_info_.client_request_.get_sql(); + DEBUG_ICMD("handle select stmt", K(sql)); + if (OB_FAIL(get_global_config_processor().execute(sql, cmd_type_, this))) { + WARN_ICMD("execute sql faield", K(sql), K(ret)); + } else { + const int64_t column_num = sqlite3_column_name_.count(); + if (OB_UNLIKELY(column_num > OB_PROXY_MAX_INNER_TABLE_COLUMN_NUM)) { + ret = OB_ERR_UNEXPECTED; + WARN_ICMD("column num is bigger than OB_PROXY_MAX_INNER_TABLE_COLUMN_NUM", K(ret), K(column_num)); + } else { + ObProxyColumnSchema schema_array[OB_PROXY_MAX_INNER_TABLE_COLUMN_NUM]; + for (int i = 0; i < column_num; i++) { + schema_array[i] = ObProxyColumnSchema::make_schema(0, sqlite3_column_name_.at(i).string_, OB_MYSQL_TYPE_VARCHAR); + } + + if (0 == column_num) { + if (OB_FAIL(encode_ok_packet(0, capability_))) { + WARN_ICMD("fail to encode eof packet", K(ret)); + } else { + event_ret = handle_callback(INTERNAL_CMD_EVENTS_SUCCESS, NULL); + } + } else if (OB_FAIL(encode_header(schema_array, column_num))) { + WARN_ICMD("fail to encode header", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < sqlite3_column_value_.count(); i++) { + ObIArray &tmp_array = sqlite3_column_value_.at(i); + ObNewRow row; + ObObj cells[64]; + for (int64_t j = 0; j < column_num; j++) { + cells[j].set_varchar(tmp_array.at(j).ptr()); + } + row.cells_ = cells; + row.count_ = column_num; + if (OB_FAIL(encode_row_packet(row))) { + WARN_ICMD("fail to encode row packet", K(row), K(ret)); + } + } + sqlite3_column_name_.reset(); + sqlite3_column_value_.reset(); + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(encode_eof_packet())) { + WARN_ICMD("fail to encode eof packet", K(ret)); + } else { + event_ret = handle_callback(INTERNAL_CMD_EVENTS_SUCCESS, NULL); + } + } + } + } + + if (OB_FAIL(ret)) { + event_ret = internal_error_callback(ret); + } + + return event_ret; +} + +int ObConfigV2Handler::handle_dml_stmt() +{ + int event_ret = EVENT_DONE; + int ret = OB_SUCCESS; + ObString sql = sm_->trans_state_.trans_info_.client_request_.get_sql(); + DEBUG_ICMD("handle dml stmt", K(sql)); + if (OB_FAIL(get_global_config_processor().execute(sql, cmd_type_, this))) { + WARN_ICMD("fail to execute sql", K(sql), K(ret)); + } else if (OB_FAIL(encode_ok_packet(0, capability_))) { + WARN_ICMD("fail to encode_ok_packet", K(ret)); + } else { + event_ret = handle_callback(INTERNAL_CMD_EVENTS_SUCCESS, NULL); + } + + if (OB_FAIL(ret)) { + event_ret = internal_error_callback(ret); + } + + return event_ret; +} + +int ObConfigV2Handler::config_v2_cmd_callback(ObContinuation *cont, ObInternalCmdInfo &info, + ObMIOBuffer *buf, ObAction *&action) +{ + int ret = OB_SUCCESS; + action = NULL; + ObConfigV2Handler *handler = NULL; + if (OB_UNLIKELY(!ObInternalCmdHandler::is_constructor_argument_valid(cont, buf))) { + ret = OB_INVALID_ARGUMENT; + WARN_ICMD("constructor argument is invalid", K(cont), K(buf), K(ret)); + } else if (OB_ISNULL(handler = new(std::nothrow) ObConfigV2Handler(cont, buf, info))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + ERROR_ICMD("fail to new ObConfigV2Handler", K(ret)); + } else if (OB_FAIL(handler->init())) { + WARN_ICMD("fail to init for ObConfigV2Handler", K(ret)); + } else { + action = &handler->get_action(); + if (OB_ISNULL(g_event_processor.schedule_imm(handler, ET_TASK))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + ERROR_ICMD("fail to schedule ObConfigV2Handler", K(ret)); + action = NULL; + } else { + DEBUG_ICMD("succ to schedule ObConfigV2Handler"); + } + } + + if (OB_FAIL(ret) && OB_LIKELY(NULL != handler)) { + delete handler; + handler = NULL; + } + + return ret; +} + +} // end of obproxy +} // end of oceanbase diff --git a/src/obproxy/cmd/ob_config_v2_handler.h b/src/obproxy/cmd/ob_config_v2_handler.h new file mode 100644 index 0000000000000000000000000000000000000000..192e7898e01568e18dddfc58297b291c0557c36c --- /dev/null +++ b/src/obproxy/cmd/ob_config_v2_handler.h @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_CONFIG_V2_HANDLER_H_ +#define OB_CONFIG_V2_HANDLER_H_ + +#include "cmd/ob_internal_cmd_handler.h" +#include "obutils/ob_proxy_sql_parser.h" +#include "lib/container/ob_se_array.h" +#include "obutils/ob_proxy_string_utils.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace proxy +{ + class ObMysqlSM; +} +class ObConfigV2Handler : public ObInternalCmdHandler +{ +public: + ObConfigV2Handler(event::ObContinuation *cont, + event::ObMIOBuffer *buf, + const ObInternalCmdInfo &info); + + ~ObConfigV2Handler() {} + int main_handler(int event, void *data); + +private: + int handle_select_stmt(); + int handle_dml_stmt(); + +public: + common::ObSEArray, 4> sqlite3_column_name_; + common::ObSEArray, 4> sqlite3_column_value_; +public: + static int config_v2_cmd_callback(event::ObContinuation *cont, ObInternalCmdInfo &info, + event::ObMIOBuffer *buf, event::ObAction *&action); +private: + proxy::ObMysqlSM* sm_; + ObProxyBasicStmtType cmd_type_; + obmysql::ObMySQLCapabilityFlags capability_; + + DISALLOW_COPY_AND_ASSIGN(ObConfigV2Handler); +}; + +} // end of obproxy +} // end of oceanbase + +#endif diff --git a/src/obproxy/cmd/ob_dds_config_handler.cpp b/src/obproxy/cmd/ob_dds_config_handler.cpp index 71e0d264745537b0831469ccccb35fff842d333e..58cd32ca7a89287f9b5d76f53ea357599ab0c232 100644 --- a/src/obproxy/cmd/ob_dds_config_handler.cpp +++ b/src/obproxy/cmd/ob_dds_config_handler.cpp @@ -341,7 +341,7 @@ int ObDdsConfigHandler::handle_parse_where_fields(ObArenaAllocator* allocator, O { int ret = OB_SUCCESS; bool need_parse_fields = true; - ObExprParseMode parse_mode = INVLIAD_PARSE_MODE; + ObExprParseMode parse_mode = INVALID_PARSE_MODE; ObProxyMysqlRequest &client_request = sm_->trans_state_.trans_info_.client_request_; ObSqlParseResult &sql_parse_result = client_request.get_parse_result(); ObString sql = client_request.get_sql(); @@ -362,6 +362,7 @@ int ObDdsConfigHandler::handle_parse_where_fields(ObArenaAllocator* allocator, O } else { ObExprParser expr_parser(*allocator, parse_mode); expr_result.part_key_info_.key_num_ = 0; + expr_result.target_mask_ = 0; if (OB_FAIL(expr_parser.parse_reqsql(sql, sql_parse_result.get_parsed_length(), expr_result, sql_parse_result.get_stmt_type(), connection_collation))) { @@ -494,7 +495,7 @@ static int dds_config_cmd_callback(ObContinuation *cont, ObInternalCmdInfo &info ret = OB_ALLOCATE_MEMORY_FAILED; ERROR_ICMD("fail to new ObDdsConfigHandler", K(ret)); } else if (OB_FAIL(handler->init())) { - WARN_ICMD("fail to init for ObDdsConfigHandler"); + WARN_ICMD("fail to init for ObDdsConfigHandler", K(ret)); } else { action = &handler->get_action(); if (OB_ISNULL(g_event_processor.schedule_imm(handler, ET_TASK))) { @@ -515,21 +516,20 @@ static int dds_config_cmd_callback(ObContinuation *cont, ObInternalCmdInfo &info int dds_config_cmd_init() { int ret = OB_SUCCESS; - // for dds_config use interal account, skip cmd type check if (OB_FAIL(get_global_internal_cmd_processor().register_cmd(OBPROXY_T_SHOW, &dds_config_cmd_callback, true))) { WARN_ICMD("fail to register_cmd for OBPROXY_T_SHOW", K(ret)); } else if (OB_FAIL(get_global_internal_cmd_processor().register_cmd(OBPROXY_T_SET_NAMES, - &dds_config_cmd_callback, true))) { + &dds_config_cmd_callback, true))) { WARN_ICMD("fail to register_cmd for OBPROXY_T_SET_NAMES", K(ret)); } else if (OB_FAIL(get_global_internal_cmd_processor().register_cmd(OBPROXY_T_SET, - &dds_config_cmd_callback, true))) { + &dds_config_cmd_callback, true))) { WARN_ICMD("fail to register_cmd for OBPROXY_T_SET", K(ret)); } else if (OB_FAIL(get_global_internal_cmd_processor().register_cmd(OBPROXY_T_SELECT, - &dds_config_cmd_callback, true))) { + &dds_config_cmd_callback, true))) { WARN_ICMD("fail to register_cmd for OBPROXY_T_SELECT", K(ret)); } else if (OB_FAIL(get_global_internal_cmd_processor().register_cmd(OBPROXY_T_UPDATE, - &dds_config_cmd_callback, true))) { + &dds_config_cmd_callback, true))) { WARN_ICMD("fail to register_cmd for OBPROXY_T_UPDATE", K(ret)); } return ret; diff --git a/src/obproxy/cmd/ob_internal_cmd_handler.h b/src/obproxy/cmd/ob_internal_cmd_handler.h index 715946155acac85dfb66f59d73a2c77a2691cfd0..5c17953c4762e87f246192406af3b6390bc4bdd6 100644 --- a/src/obproxy/cmd/ob_internal_cmd_handler.h +++ b/src/obproxy/cmd/ob_internal_cmd_handler.h @@ -41,6 +41,8 @@ struct ObProxyColumnSchema ObProxyColumnSchema() : cid_(-1), cname_(), ctype_(obmysql::OB_MYSQL_TYPE_NOT_DEFINED) {} ObProxyColumnSchema(const int64_t cid, const char *cname, const obmysql::EMySQLFieldType ctype) : cid_(cid), cname_(cname), ctype_(ctype) {} + ObProxyColumnSchema(const int64_t cid, const int64_t length, const char *cname, const obmysql::EMySQLFieldType ctype) + : cid_(cid), cname_(length, cname), ctype_(ctype) {} ~ObProxyColumnSchema() {} static ObProxyColumnSchema make_schema(const int64_t cid, const char *cname, @@ -49,11 +51,17 @@ struct ObProxyColumnSchema return (OB_ISNULL(cname) ? ObProxyColumnSchema() : ObProxyColumnSchema(cid, cname, ctype)); } + static ObProxyColumnSchema make_schema(const int64_t cid, common::ObString &cname, + const obmysql::EMySQLFieldType ctype) + { + return (cname.empty() ? ObProxyColumnSchema() : ObProxyColumnSchema(cid, cname.length(), cname.ptr(), ctype)); + } + TO_STRING_KV(K_(cid), K_(cname), K_(ctype)); - const int64_t cid_; - const common::ObString cname_; - const obmysql::EMySQLFieldType ctype_; + int64_t cid_; + common::ObString cname_; + obmysql::EMySQLFieldType ctype_; }; struct ObCSIDHandler diff --git a/src/obproxy/cmd/ob_internal_cmd_processor.cpp b/src/obproxy/cmd/ob_internal_cmd_processor.cpp index 2fcd2e47fbc9805ddec5c19f2d9134c65c4868c1..8d33a8b605971e88851465ac0751edd5b5810110 100644 --- a/src/obproxy/cmd/ob_internal_cmd_processor.cpp +++ b/src/obproxy/cmd/ob_internal_cmd_processor.cpp @@ -13,11 +13,15 @@ #define USING_LOG_PREFIX PROXY_ICMD #include "cmd/ob_internal_cmd_processor.h" +#include "cmd/ob_config_v2_handler.h" +#include "obutils/ob_config_processor.h" +#include "proxy/mysql/ob_mysql_sm.h" using namespace oceanbase::common; using namespace oceanbase::obproxy; using namespace oceanbase::obproxy::event; using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy::proxy; namespace oceanbase { @@ -51,12 +55,34 @@ int ObInternalCmdProcessor::execute_cmd(ObContinuation *cont, ObInternalCmdInfo int ret = OB_SUCCESS; action = NULL; const ObProxyBasicStmtType type = info.get_cmd_type(); + ObMysqlSM *sm = reinterpret_cast(cont); + ObString table_name; if (OB_UNLIKELY(!info.is_internal_cmd())) { ret = OB_ERR_UNEXPECTED; WARN_ICMD("invalid cmd type", K(type), K(ret)); + } else if (NULL == sm) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sm is null unexpected", K(ret)); + } else { + ObProxyMysqlRequest &client_request = sm->trans_state_.trans_info_.client_request_; + table_name = client_request.get_parse_result().get_table_name(); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (get_global_config_processor().is_table_in_service(table_name) + || OBPROXY_T_DELETE == type || OBPROXY_T_REPLACE == type) { + // Take the new configuration parsing framework + DEBUG_ICMD("begin to handle table", K(table_name)); + if (OB_FAIL(ObConfigV2Handler::config_v2_cmd_callback(cont, info, buf, action))) { + WARN_ICMD("fail to call config_v2_cmd_callback", K(ret)); + } else if (OB_ISNULL(action)) { + ret = OB_ERR_UNEXPECTED; + WARN_ICMD("action is still null, it should not happend", K(ret)); + } } else if (OB_ISNULL(cmd_table_[type].func_)) { ret = OB_ERR_UNEXPECTED; - WARN_ICMD("func is null, it should not happened", K(type), K(ret)); + WARN_ICMD("func is null, it should not happend", K(type), K(ret)); } else if (OB_FAIL((*(cmd_table_[type].func_))(cont, info, buf, action))) { WARN_ICMD("fail to call ObInternalCmdCallbackFunc", K(type), K(ret)); } else if (OB_ISNULL(action)) { @@ -70,6 +96,7 @@ int ObInternalCmdProcessor::execute_cmd(ObContinuation *cont, ObInternalCmdInfo WARN_ICMD("action is not null, but ret is fail, it should not happened", K(type), K(ret)); action = NULL; } + return ret; } diff --git a/src/obproxy/cmd/ob_show_session_handler.cpp b/src/obproxy/cmd/ob_show_session_handler.cpp index 15e0eb4fffdd78b13cda52db6586437a822d8ff1..679eb6aa739f98794400cc44ea9e6daff0204693 100644 --- a/src/obproxy/cmd/ob_show_session_handler.cpp +++ b/src/obproxy/cmd/ob_show_session_handler.cpp @@ -57,6 +57,7 @@ enum OB_SILC_STATE, OB_SILC_TID, OB_SILC_PID, + OB_SILC_USING_SSL, OB_SILC_MAX_SLIST_COLUMN_ID, }; @@ -114,6 +115,7 @@ const ObProxyColumnSchema INTERNAL_LIST_COLUMN_ARRAY[OB_SILC_MAX_SLIST_COLUMN_ID ObProxyColumnSchema::make_schema(OB_SILC_STATE, "state", OB_MYSQL_TYPE_VARCHAR), ObProxyColumnSchema::make_schema(OB_SILC_TID, "tid", OB_MYSQL_TYPE_LONGLONG), ObProxyColumnSchema::make_schema(OB_SILC_PID, "pid", OB_MYSQL_TYPE_LONG), + ObProxyColumnSchema::make_schema(OB_SILC_USING_SSL, "using_ssl", OB_MYSQL_TYPE_LONG), }; const ObProxyColumnSchema ATTRIBUTE_COLUMN_ARRAY[OB_SLC_MAX_SLIST_COLUMN_ID] = { @@ -752,6 +754,7 @@ int ObShowSessionHandler::dump_cs_list(const ObMysqlClientSession &cs) cells[OB_SILC_STATE].set_varchar(cs.get_read_state_str()); cells[OB_SILC_TID].set_int(cs.get_current_tid()); cells[OB_SILC_PID].set_mediumint(getpid()); + cells[OB_SILC_USING_SSL].set_int(static_cast(cs.get_netvc())->using_ssl()); row.cells_ = cells; row.count_ = OB_SILC_MAX_SLIST_COLUMN_ID; if (OB_FAIL(encode_row_packet(row))) { diff --git a/src/obproxy/cmd/ob_show_sqlaudit_handler.h b/src/obproxy/cmd/ob_show_sqlaudit_handler.h index bc4dd321871cc813e123c1c1b10fd9b375935d68..7f9a44ea488eaf8d121806815baa87fdca6ac9a2 100644 --- a/src/obproxy/cmd/ob_show_sqlaudit_handler.h +++ b/src/obproxy/cmd/ob_show_sqlaudit_handler.h @@ -60,7 +60,7 @@ public: bool is_overlap() const { return is_overlap_; } private: - static const double RETAIN_RATIO = 0.1; + static constexpr double RETAIN_RATIO = 0.1; ObSqlauditRecordQueue &sql_audit_record_queue_; bool is_overlap_; diff --git a/src/obproxy/dbconfig/ob_proxy_db_config_info.cpp b/src/obproxy/dbconfig/ob_proxy_db_config_info.cpp index 5e8d93ba34ee60348c4f2dddba66b0b39d4eb959..fe753d86a21ecc44ecb25eb8cbeb9bd33403a0a2 100644 --- a/src/obproxy/dbconfig/ob_proxy_db_config_info.cpp +++ b/src/obproxy/dbconfig/ob_proxy_db_config_info.cpp @@ -20,12 +20,16 @@ #include "dbconfig/ob_proxy_db_config_processor.h" #include "utils/ob_proxy_utils.h" #include "utils/ob_proxy_blowfish.h" +#include "lib/hash/ob_hashset.h" #include "lib/number/ob_number_v2.h" #include "lib/hash_func/murmur_hash.h" #include "lib/encrypt/ob_encrypted_helper.h" +#include "lib/container/ob_se_array_iterator.h" #include "iocore/eventsystem/ob_buf_allocator.h" #include "opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.h" #include "obutils/ob_proxy_sequence_utils.h" +#include "obutils/ob_proxy_stmt.h" +#include "proxy/shard/obproxy_shard_utils.h" using namespace obsys; using namespace oceanbase::json; @@ -159,6 +163,8 @@ static const char *SHARD_DISTS = "distributions"; static const char *SHARD_DISTS_DIST = "distribution"; static const char *SHARD_DISTS_MARK = "mark"; +static const int BUCKET_SIZE = 8; + ObDbConfigCache &get_global_dbconfig_cache() { @@ -414,21 +420,21 @@ int ObShardUserPrivInfo::parse_from_json(const json::Value &json_value) } else if (it->name_ == SHARDS_PASSWORD) { ret = set_password(it->value_->get_string().ptr(), it->value_->get_string().length()); } else if (it->name_ == AUTH_ALTER_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_ALTER_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_ALTER_SHIFT); } else if (it->name_ == AUTH_CREATE_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_CREATE_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_CREATE_SHIFT); } else if (it->name_ == AUTH_DELETE_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_DELETE_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_DELETE_SHIFT); } else if (it->name_ == AUTH_DROP_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_DROP_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_DROP_SHIFT); } else if (it->name_ == AUTH_INSERT_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_INSERT_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_INSERT_SHIFT); } else if (it->name_ == AUTH_UPDATE_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_UPDATE_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_UPDATE_SHIFT); } else if (it->name_ == AUTH_SELECT_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_SELECT_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_SELECT_SHIFT); } else if (it->name_ == AUTH_INDEX_PRIV) { - set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), OB_PRIV_INDEX_SHIFT); + set_user_priv(it->value_->get_string().ptr(), it->value_->get_string().length(), ::OB_PRIV_INDEX_SHIFT); } } } // end JT_OBJECT @@ -462,35 +468,35 @@ int ObShardUserPrivInfo::set_password(const char *password, int64_t len) return ret; } -void ObShardUserPrivInfo::set_user_priv(const char *priv_str, int64_t len, OB_PRIV_SHIFT type) +void ObShardUserPrivInfo::set_user_priv(const char *priv_str, int64_t len, ::OB_PRIV_SHIFT type) { ObString priv_value(len, priv_str); bool has_priv = priv_value.length() > 0 && priv_value.case_compare(AUTH_HAS_PRIV) == 0; if (has_priv) { switch(type) { - case OB_PRIV_ALTER_SHIFT: + case ::OB_PRIV_ALTER_SHIFT: priv_set_ |= OB_PRIV_ALTER; break; - case OB_PRIV_CREATE_SHIFT: + case ::OB_PRIV_CREATE_SHIFT: priv_set_ |= OB_PRIV_CREATE; break; - case OB_PRIV_DELETE_SHIFT: + case ::OB_PRIV_DELETE_SHIFT: priv_set_ |= OB_PRIV_DELETE; break; - case OB_PRIV_DROP_SHIFT: + case ::OB_PRIV_DROP_SHIFT: priv_set_ |= OB_PRIV_DROP; break; - case OB_PRIV_INSERT_SHIFT: + case ::OB_PRIV_INSERT_SHIFT: priv_set_ |= OB_PRIV_INSERT; break; - case OB_PRIV_UPDATE_SHIFT: + case ::OB_PRIV_UPDATE_SHIFT: priv_set_ |= OB_PRIV_UPDATE; break; - case OB_PRIV_SELECT_SHIFT: + case ::OB_PRIV_SELECT_SHIFT: priv_set_ |= OB_PRIV_SELECT; break; - case OB_PRIV_INDEX_SHIFT: + case ::OB_PRIV_INDEX_SHIFT: priv_set_ |= OB_PRIV_INDEX; break; default: @@ -1256,6 +1262,36 @@ void ObShardTpo::parse_tpo_specification(const ObString &specification) } } +int ObDbConfigLogicDb::get_shard_rule(ObShardRule *&shard_rule, const ObString &table_name) +{ + int ret = OB_SUCCESS; + ObShardRouter *shard_router = NULL; + shard_rule = NULL; + + // table_name_len must be less than OB_MAX_TABLE_NAME_LENGTH + char table_name_str[OB_MAX_TABLE_NAME_LENGTH]; + memcpy(table_name_str, table_name.ptr(), table_name.length()); + table_name_str[table_name.length()] = '\0'; + string_to_upper_case(table_name_str, table_name.length()); + ObString upper_table_name(table_name.length(), table_name_str); + + if (OB_FAIL(get_shard_router(upper_table_name, shard_router))) { + LOG_WARN("fail to get shard router", K_(sr_array), K(upper_table_name), K(ret)); + } else if (OB_FAIL(shard_router->get_shard_rule(upper_table_name, shard_rule))) { + LOG_WARN("fail to get logic tb info", KPC(shard_router), K(upper_table_name), K(ret)); + } else if (OB_ISNULL(shard_rule)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("logic tb info is null", K(ret)); + } + + if (NULL != shard_router) { + shard_router->dec_ref(); + shard_router = NULL; + } + + return ret; +} + int ObDbConfigLogicDb::get_shard_tpo(ObShardTpo *&shard_tpo) { int ret = OB_SUCCESS; @@ -1536,8 +1572,8 @@ int ObShardRule::get_real_name_by_index(const int64_t size, const int64_t suffix char *buf, const int64_t buf_len) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(index >= size)) { - ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; + if (OB_UNLIKELY(size > 1 && index >= size)) { + ret = OB_EXPR_CALC_ERROR; LOG_WARN("shard index is larger than shard count", K(size), K(index), K(name_prefix), K(ret)); } else { snprintf(buf, buf_len, "%.*s", name_prefix.length(), name_prefix.ptr()); @@ -1566,7 +1602,7 @@ int ObShardRule::get_physic_index_random(const int64_t physic_size,int64_t &inde } else if (physic_size == 1) { // for elastic id, physic_size = max elastic id + 1 index = 0; - } else if (OB_FAIL(ObRandomNumUtils::get_random_num(0, physic_size, index))) { + } else if (OB_FAIL(ObRandomNumUtils::get_random_num(0, physic_size - 1, index))) { LOG_DEBUG("fail to get random num", K(index), K(physic_size), K(ret)); } @@ -1646,7 +1682,7 @@ int ObShardRule::get_physic_index(const SqlFieldResult &sql_result, if (OB_SUCC(ret)) { if (OBPROXY_MAX_DBMESH_ID == last_index) { - ret = OB_ERR_COULUMN_VALUE_NOT_MATCH; + ret = OB_EXPR_CALC_ERROR; LOG_DEBUG("not match any shard rule", K(ret)); } else { index = last_index; @@ -1671,12 +1707,21 @@ int ObShardRule::get_physic_index_array(const SqlFieldResult &sql_result, int ret = OB_SUCCESS; ObArenaAllocator allocator; ObSEArray last_index_array; + ObSEArray sort_index_array; + ObHashSet index_set; LOG_DEBUG("get physic index", K(type), K(physic_size), K(is_elastic_index)); for (int i = 0; i < sql_result.field_num_; i++) { LOG_DEBUG("SqlField is ", K(i), K(sql_result.fields_[i])); } + if (OB_SUCC(ret)) { + if (OB_FAIL(index_set.create(BUCKET_SIZE, ObModIds::OB_PROXY_SHARDING_CONFIG, + ObModIds::OB_PROXY_SHARDING_CONFIG))) { + LOG_WARN("fail to create index hashset", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < rules.count(); i++) { const ObProxyShardRuleInfo &rule = rules.at(i); ObProxyExprCtx expr_ctx(physic_size, type, is_elastic_index, &allocator); @@ -1690,15 +1735,17 @@ int ObShardRule::get_physic_index_array(const SqlFieldResult &sql_result, ret = OB_ERR_UNEXPECTED; LOG_WARN("proxy expr is null unexpected", K(ret)); } else if (OB_FAIL(proxy_expr->calc(expr_ctx, calc_item, result_obj_array))) { - if (OB_EXPR_COLUMN_NOT_EXIST == ret) { - ret = OB_SUCCESS; - continue; - } else { + // Ignore the return value and scan the table directly + if (OB_EXPR_COLUMN_NOT_EXIST != ret) { LOG_WARN("calc proxy expr failed", K(ret)); } + ret = OB_SUCCESS; + continue; } else { - index_array.reset(); - // get value from result_obj_array, covert to int, save to index_array + sort_index_array.reset(); + index_set.reuse(); + // Because the obj in result_obj_array may not be an integer, + // it is stored in index_array after conversion. for (int64_t i = 0; OB_SUCC(ret) && i < result_obj_array.count(); i++) { ObObj tmp_obj; int64_t tmp_index; @@ -1709,35 +1756,53 @@ int ObShardRule::get_physic_index_array(const SqlFieldResult &sql_result, } else if (tmp_index < 0 || tmp_index >= physic_size) { ret = OB_EXPR_CALC_ERROR; LOG_WARN("invalid index", K(tmp_index), K(physic_size), K(ret)); - } else if (OB_FAIL(index_array.push_back(tmp_index))) { - LOG_WARN("push back index failed", K(ret)); + } else if (OB_FAIL(index_set.set_refactored(tmp_index))) { + LOG_WARN("fail to add index to hash set", K(tmp_index), K(ret)); + } + } + + ObHashSet::iterator iter = index_set.begin(); + ObHashSet::iterator end = index_set.end(); + for (; OB_SUCC(ret) && iter != end; iter++) { + if (OB_FAIL(sort_index_array.push_back(iter->first))) { + LOG_WARN("push back index failed", "index", iter->first, K(ret)); } } + // sort + if (OB_SUCC(ret)) { + std::sort(sort_index_array.begin(), sort_index_array.end()); + } + if (OB_SUCC(ret)) { if (!last_index_array.empty()) { - if (last_index_array.count() != index_array.count()) { + if (last_index_array.count() != sort_index_array.count()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("count not equal", K(ret), K(last_index_array.count()), K(index_array.count())); + LOG_WARN("count not equal", "last count", last_index_array.count(), + "sort count", sort_index_array.count(), K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < last_index_array.count(); i++) { - if (last_index_array.at(i) != index_array.at(i)) { + if (last_index_array.at(i) != sort_index_array.at(i)) { ret = OB_ERR_DISTRIBUTED_NOT_SUPPORTED; - LOG_WARN("different physcic index is not supported", K(ret), K(i), - K(last_index_array.at(i)), K(index_array.at(i))); + LOG_WARN("different physcic index is not supported", K(i), "last index", last_index_array.at(i), + "sort index", sort_index_array.at(i), K(ret)); } } } else if (rules.count() > 1) { - for (int64_t i = 0; OB_SUCC(ret) && i < index_array.count(); i++) { - if (OB_FAIL(last_index_array.push_back(index_array.at(i)))) { - LOG_WARN("push back index failed", K(index_array.at(i)), K(ret)); - } + if (OB_FAIL(last_index_array.assign(sort_index_array))) { + LOG_WARN("fail assign to last index array", K(sort_index_array), K(ret)); } } } } } + if (OB_SUCC(ret)) { + if (OB_FAIL(index_array.assign(sort_index_array))) { + LOG_WARN("fail assgin to index array", K(sort_index_array), K(ret)); + } + } + return ret; } @@ -1896,8 +1961,6 @@ int ObShardRouter::get_shard_rule(const ObString &tb_name, ObShardRule *&shard_r shard_rule = NULL; if (OB_FAIL(mr_map_.get_refactored(tb_name, shard_rule))) { ret = OB_ENTRY_NOT_EXIST; - } else { - LOG_DEBUG("succ to get shard rule", K(tb_name), KPC(shard_rule)); } return ret; } @@ -2826,6 +2889,22 @@ int ObDbConfigLogicTenant::load_local_db(const Value &json_value) ObString db_version; bool is_new_db = false; bool is_new_db_version = false; + + hash::ObHashSet new_db_names; + bool is_need_delete_db = false; + const ObString runtime_env = get_global_proxy_config().runtime_env.str(); + + if (0 == runtime_env.case_compare(OB_PROXY_DBP_RUNTIME_ENV)) { + int tmp_ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = new_db_names.create(BUCKET_SIZE, ObModIds::OB_PROXY_SHARDING_CONFIG, + ObModIds::OB_PROXY_SHARDING_CONFIG)))) { + LOG_WARN("hash set init failed", K(tmp_ret)); + is_need_delete_db = false; + } else { + is_need_delete_db = true; + } + } + DLIST_FOREACH(it, json_value.get_array()) { new_db_info = NULL; db_info = NULL; @@ -2849,6 +2928,16 @@ int ObDbConfigLogicTenant::load_local_db(const Value &json_value) } else if (db_info->is_version_changed(db_version)) { is_new_db_version = true; } + + if (is_need_delete_db) { + int tmp_ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = new_db_names.set_refactored(db_name)))) { + if (OB_UNLIKELY(OB_HASH_EXIST != tmp_ret)) { + LOG_WARN("fail to add db name", K(db_name), K(tmp_ret)); + is_need_delete_db = false; + } + } + } } if (is_new_db_version) { ObDataBaseKey db_info_key; @@ -2887,6 +2976,34 @@ int ObDbConfigLogicTenant::load_local_db(const Value &json_value) new_db_info = NULL; } } // end loop array + + if (OB_SUCC(ret) && is_need_delete_db) { + ObDbConfigLogicTenant *cur_tenant_info = NULL; + if (OB_ISNULL(cur_tenant_info = dbconfig_cache.get_exist_tenant(tenant_name))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("logic tenant does not exist", K(tenant_name), K(ret)); + } else { + obsys::CWLockGuard guard(dbconfig_cache.rwlock_); + ObDbConfigLogicTenant::LDHashMap &ld_map = const_cast(cur_tenant_info->ld_map_); + ObDbConfigLogicTenant::LDHashMap::iterator end = ld_map.end(); + ObDbConfigLogicTenant::LDHashMap::iterator tmp_iter; + for (ObDbConfigLogicTenant::LDHashMap::iterator iter = ld_map.begin(); iter != end;) { + if (OB_HASH_NOT_EXIST == new_db_names.exist_refactored(iter->db_name_.config_string_)) { + tmp_iter = iter; + ++iter; + ld_map.remove(&(*tmp_iter)); + tmp_iter->dec_ref(); + } else { + ++iter; + } + } + } + + if (NULL != cur_tenant_info) { + cur_tenant_info->dec_ref(); + cur_tenant_info = NULL; + } + } } return ret; } @@ -3579,6 +3696,32 @@ int ObDbConfigLogicDb::get_real_info(const common::ObString &table_name, return ret; } +int ObDbConfigLogicDb::get_real_table_name(const ObString &table_name, SqlFieldResult &sql_result, + char *real_table_name, int64_t tb_name_len, int64_t &tb_index, + const ObString &hint_table, ObTestLoadType testload_type) +{ + int ret = OB_SUCCESS; + + ObShardRule *logic_tb_info = NULL; + + if (!hint_table.empty()) { + snprintf(real_table_name, tb_name_len, "%.*s", static_cast(hint_table.length()), hint_table.ptr()); + } else if (OB_FAIL(get_shard_rule(logic_tb_info, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); + } else if (OB_FAIL(ObShardRule::get_physic_index(sql_result, logic_tb_info->tb_rules_, + logic_tb_info->tb_size_, testload_type, tb_index))) { + LOG_WARN("fail to get physic tb index", K(table_name), KPC(logic_tb_info), K(ret)); + } else if (OB_FAIL(logic_tb_info->get_real_name_by_index(logic_tb_info->tb_size_, logic_tb_info->tb_suffix_len_, + tb_index, logic_tb_info->tb_prefix_.config_string_, + logic_tb_info->tb_tail_.config_string_, real_table_name, tb_name_len))) { + LOG_WARN("fail to get real table name", K(tb_index), KPC(logic_tb_info), K(ret)); + } else if (TESTLOAD_NON != testload_type) { + snprintf(real_table_name + strlen(real_table_name), tb_name_len - strlen(real_table_name), "_T"); + } + + return ret; +} + int ObDbConfigLogicDb::get_shard_table_info(const ObString &table_name, SqlFieldResult &sql_result, ObShardConnector *&shard_conn, @@ -3591,32 +3734,16 @@ int ObDbConfigLogicDb::get_shard_table_info(const ObString &table_name, int ret = OB_SUCCESS; ObShardRule *logic_tb_info = NULL; - ObShardRouter *shard_router = NULL; ObShardTpo *shard_tpo = NULL; ObGroupCluster *gc_info = NULL; - // table_name_len must be less than OB_MAX_TABLE_NAME_LENGTH - char table_name_str[OB_MAX_TABLE_NAME_LENGTH]; - memcpy(table_name_str, table_name.ptr(), table_name.length()); - table_name_str[table_name.length()] = '\0'; - string_to_upper_case(table_name_str, table_name.length()); - ObString upper_table_name(table_name.length(), table_name_str); - if (OB_FAIL(get_shard_tpo(shard_tpo))) { LOG_WARN("fail to get shard tpo info", K(ret)); } else if (OB_ISNULL(shard_tpo)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("shard tpo info is null", K(ret)); - } else if (OB_FAIL(get_shard_router(upper_table_name, shard_router))) { - LOG_WARN("fail to get shard router", K_(sr_array), K(upper_table_name), K(ret)); - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_ERR_NO_TABLE_RULE; - } - } else if (OB_FAIL(shard_router->get_shard_rule(upper_table_name, logic_tb_info))) { - LOG_WARN("fail to get logic tb info", KPC(shard_router), K(upper_table_name), K(ret)); - } else if (OB_ISNULL(logic_tb_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("logic tb info is null", K(ret)); + } else if (OB_FAIL(get_shard_rule(logic_tb_info, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); // get group_id } else if (OB_FAIL(ObShardRule::get_physic_index(sql_result, logic_tb_info->db_rules_, logic_tb_info->db_size_, testload_type, group_index))) { @@ -3648,7 +3775,7 @@ int ObDbConfigLogicDb::get_shard_table_info(const ObString &table_name, testload_type, es_index, is_elastic_index))) { LOG_WARN("fail to calculate elastic index", K(table_name), KPC(logic_tb_info), K(ret)); } else if (OB_UNLIKELY(es_index >= es_size)) { - ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("es index is larger than elastic array", K(es_index), K(es_size), K(ret)); } @@ -3689,10 +3816,6 @@ int ObDbConfigLogicDb::get_shard_table_info(const ObString &table_name, } } - if (NULL != shard_router) { - shard_router->dec_ref(); - shard_router = NULL; - } if (NULL != shard_tpo) { shard_tpo->dec_ref(); shard_tpo = NULL; @@ -3752,7 +3875,7 @@ int ObDbConfigLogicDb::get_single_table_info(const ObString &table_name, LOG_WARN("fail to get random eid", K(gc_info)); } } else if (OB_UNLIKELY(es_id >= es_size)) { - ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("es index is larger than elastic array", K(es_id), K(es_size), K(ret)); } if (OB_SUCC(ret)) { @@ -4212,238 +4335,256 @@ int ObDbConfigCache::handle_new_db_info(ObDbConfigLogicDb &db_info, bool is_from return ret; } -int ObDbConfigLogicDb::get_sharding_select_info(const common::ObString &table_name, - ObSqlParseResult &parse_result, - ObTestLoadType testload_type, - bool is_read_stmt, - int64_t es_index, - ObIAllocator &allocator, - ObIArray &shard_connector_array, - ObIArray &physical_table_name_array) +int ObDbConfigLogicDb::get_es_index_by_gc(ObGroupCluster *gc_info, ObShardRule *shard_rule, + ObTestLoadType testload_type, bool is_read_stmt, + SqlFieldResult &sql_result, int64_t &es_index) { int ret = OB_SUCCESS; - SqlFieldResult &sql_result = parse_result.get_sql_filed_result(); - ObShardRule *shard_rule = NULL; - ObShardRouter *shard_router = NULL; - ObShardTpo *shard_tpo = NULL; - ObSEArray tmp_shard_connector_array; - ObSEArray tmp_physical_table_name_array; - bool need_scan_all = false; - // table_name_len must be less than OB_MAX_TABLE_NAME_LENGTH - char table_name_str[OB_MAX_TABLE_NAME_LENGTH]; - memcpy(table_name_str, table_name.ptr(), table_name.length()); - table_name_str[table_name.length()] = '\0'; - string_to_upper_case(table_name_str, table_name.length()); - ObString upper_table_name(table_name.length(), table_name_str); + // Each elastic bit will only be calculated once, and for subsequent reuse, + // it is necessary to ensure that the following elastic bits are valid. + if (-1 == es_index || OBPROXY_MAX_DBMESH_ID == es_index) { + int64_t es_size = gc_info->get_es_size(); + ObSEArray es_index_array; + if (-1 == es_index || (OBPROXY_MAX_DBMESH_ID == es_index && shard_rule->es_rules_.empty())) { + if (OB_FAIL(gc_info->get_elastic_id_by_weight(es_index, is_read_stmt))) { + LOG_WARN("fail to get eid by read weight", K(is_read_stmt), KPC(gc_info)); + } else { + LOG_DEBUG("succ to get eid by weight", K(es_index)); + } + } else if (sql_result.field_num_ > 0 + && OB_FAIL(ObShardRule::get_physic_index_array(sql_result, shard_rule->es_rules_, es_size, + testload_type, es_index_array, true))) { + LOG_WARN("fail to calculate elastic index", KPC(shard_rule), K(ret)); + } else if (es_index_array.empty()) { + if (OB_FAIL(gc_info->get_elastic_id_by_weight(es_index, is_read_stmt))) { + LOG_WARN("fail to get eid by read weight", K(is_read_stmt), KPC(gc_info)); + } else { + LOG_DEBUG("succ to get eid by weight", K(es_index)); + } + } else { + es_index = es_index_array.at(0); + if (es_index >= es_size) { + ret = OB_EXPR_CALC_ERROR; + LOG_WARN("es index is larger than elastic array", K(es_index), K(es_size), K(ret)); + } - if (OB_FAIL(get_shard_tpo(shard_tpo))) { - LOG_WARN("fail to get shard tpo info", K(ret)); - } else if (OB_ISNULL(shard_tpo)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("shard tpo info is null", K(ret)); - } else if (OB_FAIL(get_shard_router(upper_table_name, shard_router))) { - LOG_WARN("fail to get shard router", K_(sr_array), K(upper_table_name), K(ret)); - } else if (OB_FAIL(shard_router->get_shard_rule(upper_table_name, shard_rule))) { - LOG_WARN("fail to get logic tb info", K(upper_table_name), K(ret)); - } else if (OB_ISNULL(shard_rule)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("logic tb info is null", K(ret)); + for (int64_t j = 1; OB_SUCC(ret) && j < es_index_array.count(); j++) { + if (es_index != es_index_array.at(j)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get too many es index", K(es_index), K(j), K(es_index_array.at(j)), K(ret)); + } + } + } } - // get shard_connector - if (OB_SUCC(ret)) { - ObSEArray group_index_array; - if (sql_result.field_num_ > 0 && OB_FAIL(ObShardRule::get_physic_index_array(sql_result, - shard_rule->db_rules_, shard_rule->db_size_, testload_type, group_index_array))) { - LOG_WARN("fail to get physic db index", K(table_name), KPC(shard_rule), K(ret)); + return ret; +} + +int ObDbConfigLogicDb::get_db_and_table_index(ObShardRule *shard_rule, + ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + ObIArray &group_index_array, + ObIArray &table_index_array) +{ + int ret = OB_SUCCESS; + + SqlFieldResult &sql_result = parse_result.get_sql_filed_result(); + + if (shard_rule->tb_size_ == 1) { + // Sub-libraries are not divided into tables, calculated according to the library + // Note that the sub-library single table will not go here + if (sql_result.field_num_ > 0 + && OB_FAIL(ObShardRule::get_physic_index_array(sql_result, shard_rule->db_rules_, + shard_rule->db_size_, testload_type, + group_index_array))) { + LOG_WARN("fail to get physic db index", KPC(shard_rule), K(ret)); } // if do not get sharding info, scall all table if (OB_SUCC(ret) && group_index_array.empty()) { - need_scan_all = true; for (int64_t i = 0; OB_SUCC(ret) && i < shard_rule->db_size_; i++) { if (OB_FAIL(group_index_array.push_back(i))) { - LOG_WARN("push back group index failed", K(ret), K(i)); + LOG_WARN("push back group index failed", K(i), K(ret)); } } } for (int64_t i = 0; OB_SUCC(ret) && i < group_index_array.count(); i++) { - int64_t group_index = group_index_array.at(i); - char group_name[OB_MAX_DATABASE_NAME_LENGTH]; - ObGroupCluster *gc_info = NULL; - if (OB_FAIL(shard_rule->get_real_name_by_index(shard_rule->db_size_, - shard_rule->db_suffix_len_, group_index, - shard_rule->db_prefix_.config_string_, - shard_rule->db_tail_.config_string_, - group_name, OB_MAX_DATABASE_NAME_LENGTH))) { - LOG_WARN("fail to get real group name", K(group_index), KPC(shard_rule), K(ret)); - } else if (OB_FAIL(shard_tpo->get_group_cluster(ObString::make_string(static_cast(group_name)), gc_info))) { - LOG_WARN("group does not exist", K(ret)); - } else if (OB_ISNULL(gc_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group cluster info is null", K(ret)); - } else { - // es_id only calc once - if (-1 == es_index || OBPROXY_MAX_DBMESH_ID == es_index) { - int64_t es_size = gc_info->get_es_size(); - ObSEArray es_index_array; - if (-1 == es_index || (OBPROXY_MAX_DBMESH_ID == es_index && shard_rule->es_rules_.empty())) { - if (OB_FAIL(gc_info->get_elastic_id_by_weight(es_index, is_read_stmt))) { - LOG_WARN("fail to get eid by read weight", KPC(gc_info)); - } else { - LOG_DEBUG("succ to get eid by weight", K(es_index)); - } - } else if (sql_result.field_num_ > 0 && OB_FAIL(ObShardRule::get_physic_index_array(sql_result, shard_rule->es_rules_, es_size, - testload_type, es_index_array, true))) { - LOG_WARN("fail to calculate elastic index", K(table_name), KPC(shard_rule), K(ret)); - } else if (es_index_array.empty()) { - if (OB_FAIL(gc_info->get_elastic_id_by_weight(es_index, is_read_stmt))) { - LOG_WARN("fail to get eid by read weight", KPC(gc_info)); - } else { - LOG_DEBUG("succ to get eid by weight", K(es_index)); - } - } else { - es_index = es_index_array.at(0); - if (es_index >= es_size) { - ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; - LOG_WARN("es index is larger than elastic array", K(es_index), K(es_size), K(ret)); - } - - for (int64_t j = 1; OB_SUCC(ret) && j < es_index_array.count(); j++) { - if (es_index != es_index_array.at(j)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get too many es index", K(ret), K(es_index), K(j), K(es_index_array.at(j))); - } - } - } - } - - if (OB_SUCC(ret)) { - ObString shard_name = gc_info->get_shard_name_by_eid(es_index); - ObShardConnector *shard_conn = NULL; - if (TESTLOAD_NON != testload_type) { - if (OB_FAIL(get_testload_shard_connector(shard_name, testload_prefix_.config_string_, shard_conn))) { - LOG_WARN("testload shard connector not exist", K(ret), K(shard_name), K(testload_prefix_)); - } - } else if (OB_FAIL(get_shard_connector(shard_name, shard_conn))) { - LOG_WARN("shard connector does not exist", K(shard_name), K(ret)); - } else if (NULL != shard_conn && OB_FAIL(tmp_shard_connector_array.push_back(shard_conn))) { - LOG_WARN("push back shard conn failed", K(shard_conn), K(ret)); - } - } + if (OB_FAIL(table_index_array.push_back(group_index_array.at(i)))) { + LOG_WARN("push back group index failed", "group index", group_index_array.at(i), K(ret)); } - } // end group_index_array - } - - if (OB_SUCC(ret)) { - ObSEArray table_index_array; - char real_table_name[OB_MAX_TABLE_NAME_LENGTH]; - if (OB_FAIL(sql_result.field_num_ > 0 && ObShardRule::get_physic_index_array(sql_result, shard_rule->tb_rules_, - shard_rule->tb_size_, testload_type, table_index_array))) { - LOG_WARN("fail to get physic tb index", K(table_name), KPC(shard_rule), K(ret)); + } + } else { + // Sub-library and sub-table, according to table calculation + // We only support shard-rules that satisfied 'db_index = tb_index / (tb_size / db_size)' + int64_t route_hint = shard_rule->tb_size_ / shard_rule->db_size_; + // If you need to scan the database, then the table should not be able to calculate + if (sql_result.field_num_ > 0 + && OB_FAIL(ObShardRule::get_physic_index_array(sql_result, shard_rule->tb_rules_, + shard_rule->tb_size_, testload_type, + table_index_array))) { + LOG_WARN("fail to get physic tb index", KPC(shard_rule), K(ret)); } // if do not get sharding info, scall all table if (OB_SUCC(ret) && table_index_array.empty()) { - need_scan_all = true; for (int64_t i = 0; OB_SUCC(ret) && i < shard_rule->tb_size_; i++) { if (OB_FAIL(table_index_array.push_back(i))) { - LOG_WARN("table index array push back failed", K(ret), K(i)); + LOG_WARN("table index array push back failed", K(i), K(ret)); } } } for (int64_t i = 0; OB_SUCC(ret) && i < table_index_array.count(); i++) { - int64_t tb_index = table_index_array.at(i); - if (OB_FAIL(shard_rule->get_real_name_by_index(shard_rule->tb_size_, shard_rule->tb_suffix_len_, - tb_index, shard_rule->tb_prefix_.config_string_, shard_rule->tb_tail_.config_string_, - real_table_name, OB_MAX_TABLE_NAME_LENGTH))) { - LOG_WARN("fail to get real table name", K(tb_index), KPC(shard_rule), K(ret)); + if (OB_FAIL(group_index_array.push_back(table_index_array.at(i) / route_hint))) { + LOG_WARN("push back group index failed", "table index", table_index_array.at(i), + K(route_hint), K(ret)); } + } + } - if (OB_SUCC(ret) && TESTLOAD_NON != testload_type) { - snprintf(real_table_name + strlen(real_table_name), OB_MAX_TABLE_NAME_LENGTH - strlen(real_table_name), "_T"); + return ret; +} + +int ObDbConfigLogicDb::get_shard_prop_by_connector(ObIArray &shard_connector_array, + ObIArray &shard_prop_array) +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < shard_connector_array.count(); i++) { + ObShardConnector *shard_connctor = shard_connector_array.at(i); + ObShardProp *shard_prop = NULL; + if (OB_FAIL(get_shard_prop(shard_connctor->shard_name_, shard_prop))) { + LOG_DEBUG("fail to get shard prop", "shard name", shard_connctor->shard_name_, K(ret)); + ret = OB_SUCCESS; + } + + if (OB_FAIL(shard_prop_array.push_back(shard_prop))) { + LOG_WARN("push back shard prop failed", KP(shard_prop), K(ret)); + } + } + + return ret; +} + +int ObDbConfigLogicDb::get_shard_connector_by_index(ObShardRule *shard_rule, + ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + bool is_read_stmt, + int64_t es_index, + ObIArray &group_index_array, + ObIArray &shard_connector_array) +{ + int ret = OB_SUCCESS; + + SqlFieldResult &sql_result = parse_result.get_sql_filed_result(); + ObShardTpo *shard_tpo = NULL; + + if (OB_FAIL(get_shard_tpo(shard_tpo))) { + LOG_WARN("fail to get shard tpo info", K(ret)); + } else if (OB_ISNULL(shard_tpo)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("shard tpo info is null", K(ret)); + } + + // acquire shard_connector + for (int64_t i = 0; OB_SUCC(ret) && i < group_index_array.count(); i++) { + int64_t group_index = group_index_array.at(i); + char group_name[OB_MAX_DATABASE_NAME_LENGTH]; + ObGroupCluster *gc_info = NULL; + if (OB_FAIL(shard_rule->get_real_name_by_index(shard_rule->db_size_, + shard_rule->db_suffix_len_, group_index, + shard_rule->db_prefix_.config_string_, + shard_rule->db_tail_.config_string_, + group_name, OB_MAX_DATABASE_NAME_LENGTH))) { + LOG_WARN("fail to get real group name", K(group_index), KPC(shard_rule), K(ret)); + } else if (OB_FAIL(shard_tpo->get_group_cluster(ObString::make_string(static_cast(group_name)), gc_info))) { + LOG_WARN("group does not exist", K(group_name), K(ret)); + } else if (OB_ISNULL(gc_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group cluster info is null", K(ret)); + } else if (-1 == es_index || OBPROXY_MAX_DBMESH_ID == es_index) { + if (OB_FAIL(get_es_index_by_gc(gc_info, shard_rule, testload_type, is_read_stmt, sql_result, es_index))) { + LOG_WARN("fail to get es index", K(gc_info), K(shard_rule), K(testload_type), K(is_read_stmt), K(ret)); } + } - void *buf = NULL; - int64_t name_len = strlen(real_table_name); - if (NULL == (buf = allocator.alloc(name_len))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc table name failed", K(ret), K(name_len)); - } else { - MEMCPY(buf, real_table_name, name_len); - ObString table_name; - table_name.assign(static_cast(buf), static_cast(name_len)); - if (OB_FAIL(tmp_physical_table_name_array.push_back(table_name))) { - LOG_WARN("physical table name push back failed", K(ret), K(table_name)); + if (OB_SUCC(ret)) { + ObString shard_name = gc_info->get_shard_name_by_eid(es_index); + ObShardConnector *shard_conn = NULL; + if (TESTLOAD_NON != testload_type) { + if (OB_FAIL(get_testload_shard_connector(shard_name, testload_prefix_.config_string_, shard_conn))) { + LOG_WARN("testload shard connector not exist", K(ret), K(shard_name), K(testload_prefix_)); } + } else if (OB_FAIL(get_shard_connector(shard_name, shard_conn))) { + LOG_WARN("shard connector does not exist", K(shard_name), K(ret)); + } else if (NULL != shard_conn && OB_FAIL(shard_connector_array.push_back(shard_conn))) { + LOG_WARN("push back shard conn failed", K(shard_conn), K(ret)); } } + } // end group_index_array + + if (NULL != shard_tpo) { + shard_tpo->dec_ref(); + shard_tpo = NULL; } - if (OB_SUCC(ret)) { - struct RealConnector { - RealConnector() : connector_(NULL), table_name_() {} - ~RealConnector() {} - - ObShardConnector *connector_; - ObString table_name_; - LINK(RealConnector, real_connector_link_); - }; - - struct ObShardConnectorHashing - { - typedef const RealConnector &Key; - typedef RealConnector Value; - typedef ObDLList(RealConnector, real_connector_link_) ListHead; - - static uint64_t hash(Key key) { return (key.connector_->shard_name_).hash(); } - static Key key(Value *value) { return *value; } - static bool equal(Key lhs, Key rhs) - { - return lhs.connector_->shard_name_ == rhs.connector_->shard_name_ - && lhs.table_name_ == rhs.table_name_; - } - }; - - static const int64_t SC_HASH_BUCKET_SIZE = 32; - typedef common::hash::ObBuildInHashMap ObShardConnectorHashMap; - - ObShardConnectorHashMap connector_map; - if (!need_scan_all) { // execute it when has the split key - for(int64_t i = 0; OB_SUCC(ret) && i < tmp_shard_connector_array.count(); i++) { - RealConnector tmp_connector; - tmp_connector.connector_ = tmp_shard_connector_array.at(i); - tmp_connector.table_name_ = tmp_physical_table_name_array.at(i); - if (OB_HASH_EXIST == connector_map.unique_set(&tmp_connector)) { - // do nothing - } else if (OB_FAIL(shard_connector_array.push_back(tmp_connector.connector_))) { - LOG_WARN("shard_connector_array push back failed", K(ret)); - } else if (OB_FAIL(physical_table_name_array.push_back(tmp_connector.table_name_))) { - LOG_WARN("physical_table_name_array push back failed", K(ret)); + return ret; +} + +int ObDbConfigLogicDb::get_table_name_by_index(ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + ObIAllocator &allocator, + ObIArray &table_index_array, + ObIArray > &table_name_map_array) +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(parse_result.get_proxy_stmt()); + ObProxySelectStmt::ExprMap &table_exprs_map = select_stmt->get_table_exprs_map(); + SqlFieldResult &sql_result = parse_result.get_sql_filed_result(); + char real_table_name[OB_MAX_TABLE_NAME_LENGTH]; + + ObHashMapWrapper table_name_map_wrapper; + if (OB_FAIL(table_name_map_wrapper.init(OB_ALIAS_TABLE_MAP_MAX_BUCKET_NUM, + ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { + LOG_WARN("fail to init table name map", K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < table_index_array.count(); i++) { + int64_t tb_index = table_index_array.at(i); + ObProxySelectStmt::ExprMap::iterator iter = table_exprs_map.begin(); + ObProxySelectStmt::ExprMap::iterator end = table_exprs_map.end(); + table_name_map_wrapper.reuse(); + + for (; OB_SUCC(ret) && iter != end; iter++) { + ObProxyExpr *expr = iter->second; + ObProxyExprTable *table_expr = NULL; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null, unexpected", K(ret)); + } else if (OB_ISNULL(table_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to cast to table expr", K(expr), K(ret)); + } else { + ObString &sql_table_name = table_expr->get_table_name(); + ObString hint_table; + + if (OB_FAIL(get_real_table_name(sql_table_name, sql_result, + real_table_name, OB_MAX_TABLE_NAME_LENGTH, + tb_index, hint_table, testload_type))) { + LOG_WARN("fail to get real table name", K(sql_table_name), K(tb_index), K(testload_type), K(ret)); + } else if (OB_FAIL(ObProxyShardUtils::add_table_name_to_map(allocator, table_name_map_wrapper.get_hash_map(), + sql_table_name, real_table_name))) { + LOG_WARN("fail to add table name to map", K(sql_table_name), K(real_table_name), K(ret)); } } - } else { - // We only support shard-rules that satisfied 'db_index = tb_index / (tb_size / db_size)' - int64_t route_hint = shard_rule->tb_size_ / shard_rule->db_size_; - int64_t group_index = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < tmp_physical_table_name_array.count(); i++) { - group_index = i / route_hint; - if (group_index >= tmp_shard_connector_array.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get more tables base on table index", K(ret), K(i), K(group_index)); - } else { - RealConnector tmp_connector; - tmp_connector.connector_ = tmp_shard_connector_array.at(group_index); - tmp_connector.table_name_ = tmp_physical_table_name_array.at(i); - if (OB_FAIL(shard_connector_array.push_back(tmp_connector.connector_))) { - LOG_WARN("shard_connector_array push back failed", K(ret)); - } else if (OB_FAIL(physical_table_name_array.push_back(tmp_connector.table_name_))) { - LOG_WARN("physical_table_name_array push back failed", K(ret)); - } - } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_name_map_array.push_back(table_name_map_wrapper))) { + LOG_WARN("fail to push back table name map", K(ret)); } } } @@ -4451,6 +4592,40 @@ int ObDbConfigLogicDb::get_sharding_select_info(const common::ObString &table_na return ret; } +int ObDbConfigLogicDb::get_sharding_select_info(const common::ObString &table_name, + ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + bool is_read_stmt, + int64_t es_index, + ObIAllocator &allocator, + ObIArray &shard_connector_array, + ObIArray &shard_prop_array, + ObIArray > &table_name_map_array) +{ + int ret = OB_SUCCESS; + + ObShardRule *shard_rule = NULL; + ObSEArray group_index_array; + ObSEArray table_index_array; + + if (OB_FAIL(get_shard_rule(shard_rule, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); + } else if (OB_FAIL(get_db_and_table_index(shard_rule, parse_result, testload_type, + group_index_array, table_index_array))) { + LOG_WARN("fail to get db and table index", KPC(shard_rule), K(testload_type), K(ret)); + } else if (OB_FAIL(get_shard_connector_by_index(shard_rule, parse_result, testload_type, is_read_stmt, + es_index, group_index_array, shard_connector_array))) { + LOG_WARN("fail to get shard connector", KPC(shard_rule), K(testload_type), K(is_read_stmt), K(es_index), K(ret)); + } else if (OB_FAIL(get_shard_prop_by_connector(shard_connector_array, shard_prop_array))) { + LOG_WARN("fail to get shard prop", K(ret)); + } else if (OB_FAIL(get_table_name_by_index(parse_result, testload_type, + allocator, table_index_array, table_name_map_array))) { + LOG_WARN("fail to get real table name", KPC(shard_rule), K(testload_type), K(ret)); + } + + return ret; +} + } // end namespace dbconfig } // end namespace proxy } // end namespace oceanbase diff --git a/src/obproxy/dbconfig/ob_proxy_db_config_info.h b/src/obproxy/dbconfig/ob_proxy_db_config_info.h index fad0a0556d7c7912ab940a583f5b5ca42f5badaf..7465e7e4a9cf314e1c1c62f3fd695634a40a2aa2 100644 --- a/src/obproxy/dbconfig/ob_proxy_db_config_info.h +++ b/src/obproxy/dbconfig/ob_proxy_db_config_info.h @@ -743,8 +743,8 @@ struct ConnectionProperties { class ObShardProp : public ObDbConfigChild { public: - ObShardProp() : ObDbConfigChild(TYPE_SHARDS_PROP), shard_name_(), connect_timeout_(0), - socket_timeout_(0), idle_timeout_ms_(0), blocking_timeout_ms_(0), need_prefill_(false), read_consistency_(), + ObShardProp() : ObDbConfigChild(TYPE_SHARDS_PROP), shard_name_(), connect_timeout_(500), + socket_timeout_(5000), idle_timeout_ms_(0), blocking_timeout_ms_(0), need_prefill_(false), read_consistency_(), kv_kv_map_(), kv_vec_map_(), cur_conn_prop_(), conn_prop_type_(TYPE_SHARD_PROP), current_zone_(), conn_prop_map_() {} virtual ~ObShardProp() {} @@ -886,6 +886,7 @@ public: int get_shard_tpo(ObShardTpo *&shard_tpo); int get_all_shard_table(common::ObIArray &all_table); int get_shard_router(const common::ObString &tb_name, ObShardRouter *&shard_router); + int get_shard_rule(ObShardRule *&shard_rule, const common::ObString &table_name); bool is_shard_rule_empty(); int get_shard_connector(const common::ObString &shard_name, ObShardConnector *&shard_conn); int get_testload_shard_connector(const common::ObString &shard_name, @@ -946,6 +947,32 @@ public: ObTestLoadType testload_type, const bool is_read_stmt); int get_shard_prop(const common::ObString & shard_name, ObShardProp* &shard_prop); + int get_real_table_name(const ObString &table_name, obutils::SqlFieldResult &sql_result, + char *real_table_name, int64_t tb_name_len, int64_t &tb_index, + const ObString &hint_table, ObTestLoadType testload_type); + int get_es_index_by_gc(ObGroupCluster *gc_info, ObShardRule *shard_rule, + ObTestLoadType testload_type, bool is_read_stmt, + obutils::SqlFieldResult &sql_result, int64_t &es_index); + int get_db_and_table_index(ObShardRule *shard_rule, + obutils::ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + common::ObIArray &group_index_array, + common::ObIArray &table_index_array); + + int get_shard_prop_by_connector(ObIArray &shard_connector_array, + ObIArray &shard_prop_array); + int get_shard_connector_by_index(ObShardRule *shard_rule, + obutils::ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + bool is_read_stmt, + int64_t es_index, + common::ObIArray &group_index_array, + common::ObIArray &shard_connector_array); + int get_table_name_by_index(obutils::ObSqlParseResult &parse_result, + ObTestLoadType testload_type, + ObIAllocator &allocator, + common::ObIArray &table_index_array, + common::ObIArray > &table_name_map_array); const common::ObString get_sequence_table_name(); const common::ObString get_sequence_table_name_from_router(); @@ -966,7 +993,8 @@ public: int64_t es_index, common::ObIAllocator &allocator, common::ObIArray &shard_connector_array, - common::ObIArray &physical_table_name_array); + common::ObIArray &shard_prop_array, + common::ObIArray > &table_name_map_array); int init_test_load_table_map(); int check_and_get_testload_table(std::string origin_name, std::string &real_name); diff --git a/src/obproxy/dbconfig/ob_proxy_pb_utils.cpp b/src/obproxy/dbconfig/ob_proxy_pb_utils.cpp index d4e6591c8bfd90cab10a78c5a22ffa69191f8291..dd0a3f4115d111a44aeebb115351d2e832dc1bf2 100644 --- a/src/obproxy/dbconfig/ob_proxy_pb_utils.cpp +++ b/src/obproxy/dbconfig/ob_proxy_pb_utils.cpp @@ -390,7 +390,7 @@ void ObProxyPbUtils::get_str_value_by_name(const std::string &shard_url, } } // shard_url format: jdbc:oceanbase://127.0.0.1:18587/group_00?useUnicode=true&characterEncoding=utf8 -// or ocj shard url format: http://x.x.x.x/services?Action=ObRootServiceInfo&User_ID=ocptest&UID=gengping.zj&ObRegion=ant_dev_g&database=thin&k=v +// or ocj shard url format: http://x.x.x.x/services?Action=xxx&User_ID=xxx&UID=xxx&ObRegion=xxx&database=xxx&k=v int ObProxyPbUtils::parse_shard_url(const std::string &shard_url, ObShardConnector &conn_info) { int ret = OB_SUCCESS; diff --git a/src/obproxy/engine/ob_proxy_operator.cpp b/src/obproxy/engine/ob_proxy_operator.cpp index 86bd7e22a356f62cac597d5363f9f4602f3779aa..6fc55855e27035ec5128755d35a84a7761ea81c9 100644 --- a/src/obproxy/engine/ob_proxy_operator.cpp +++ b/src/obproxy/engine/ob_proxy_operator.cpp @@ -346,26 +346,28 @@ int ObProxyOperator::put_result_row(ResultRow *row) int ObProxyOperator::process_ready_data(void *data, int &event) { int ret = OB_SUCCESS; + bool is_final = false; ObProxyResultResp *result = NULL; - ObProxyResultResp *res = NULL; if (OB_ISNULL(data)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param, data is null", "op_name", op_name(), K(ret)); - } else if (OB_ISNULL(res = reinterpret_cast(data))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid input, pres type is not match", K(ret), KP(data)); - } else if (OB_FAIL(handle_result(res, false, result))) { - LOG_WARN("fail to handle_result", "op_name", op_name(), K(ret)); - } else if (OB_NOT_NULL(result) && result->is_error_resp()) { + } else if (OB_FAIL(handle_result(data, is_final, result))) { + LOG_WARN("fail to handle result", "op_name", op_name(), K(ret)); + } else if (!is_final && OB_ISNULL(result)) { + event = VC_EVENT_CONT; + } else if (is_final || result->is_error_resp()) { result_ = result; event = VC_EVENT_READ_COMPLETE; + } else { + result_ = result; + event = VC_EVENT_READ_READY; } LOG_DEBUG("finish process_ready_data", K(event), K(ret)); return ret; } -int ObProxyOperator::handle_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyOperator::handle_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; @@ -391,7 +393,7 @@ int ObProxyOperator::handle_result(void *data, bool is_final, ObProxyResultResp return ret; } -int ObProxyOperator::handle_response_result(void *src, bool is_final, ObProxyResultResp *&result) +int ObProxyOperator::handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result) { UNUSED(src); UNUSED(is_final); @@ -402,15 +404,12 @@ int ObProxyOperator::handle_response_result(void *src, bool is_final, ObProxyRes int ObProxyOperator::process_complete_data(void *data) { int ret = OB_SUCCESS; + bool is_final = true; ObProxyResultResp *result = NULL; - ObProxyResultResp *res = NULL; if (OB_ISNULL(data)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid param, data is null", K(ret)); - } else if (OB_ISNULL(res = reinterpret_cast(data))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid input, pres type is not match", K(ret)); - } else if (OB_FAIL(handle_result(res, true, result))) { + } else if (OB_FAIL(handle_result(data, is_final, result))) { LOG_WARN("fail to handle result", K(ret)); } else if (OB_ISNULL(result)) { ret = OB_ERR_UNEXPECTED; @@ -453,7 +452,7 @@ int ObProxyOperator::init_row(ResultRow *&row) ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRow))); } else { - row = new (buf) ResultRow(array_new_alloc_size, allocator_); + row = new (buf) ResultRow(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); } return ret; } @@ -467,7 +466,7 @@ int ObProxyOperator::init_row_set(ResultRows *&rows) ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); } else { - rows = new (buf) ResultRows(array_new_alloc_size, allocator_); + rows = new (buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); } return ret; } @@ -593,13 +592,6 @@ void ObProxyOperator::set_op_type(ObPhyOperatorType type) type_ = type; } -void ObProxyOpInput::set_op_limit(int64_t limit, int64_t offset) -{ - LOG_DEBUG("set_op_limit:", K(limit), K(offset)); - limit_ = limit; - offset_ = offset; -} - } } } diff --git a/src/obproxy/engine/ob_proxy_operator.h b/src/obproxy/engine/ob_proxy_operator.h index 2e8549c82dab668b9d0624f8b3ee2fc801acc02d..1f4bd2d67160f9df714b4ae2380391b53ff9104c 100644 --- a/src/obproxy/engine/ob_proxy_operator.h +++ b/src/obproxy/engine/ob_proxy_operator.h @@ -39,8 +39,12 @@ enum ObPhyOperatorType PHY_PROJECTION, PHY_MERGE_AGG, PHY_HASH_AGG, + PHY_STREAM_AGG, + PHY_MEM_MERGE_AGG, PHY_MERGE_SORT, PHY_TOPK, + PHY_STREAM_SORT, + PHY_MEM_MERGE_SORT, PHY_MAX }; @@ -72,6 +76,18 @@ const char* get_op_name(ObPhyOperatorType type) case PHY_TOPK: char_ret = "PHY_TOPK"; break; + case PHY_STREAM_AGG: + char_ret = "PHY_STREAM_AGG"; + break; + case PHY_MEM_MERGE_AGG: + char_ret = "PHY_MEM_MERGE_AGG"; + break; + case PHY_STREAM_SORT: + char_ret = "PHY_STREAM_SORT"; + break; + case PHY_MEM_MERGE_SORT: + char_ret = "PHY_MEM_MERGE_SORT"; + break; default: char_ret = "UNKOWN_OPERATOR"; break; @@ -79,8 +95,6 @@ const char* get_op_name(ObPhyOperatorType type) return char_ret; } -typedef dbconfig::ObShardConnector dbKeyName; - /* Default child_cnt_ for each Operator */ const uint32_t OP_MAX_CHILDREN_NUM = 10; const uint32_t OP_HANDLER_TIMEOUT_MS = 5000 * 1000; @@ -136,8 +150,8 @@ public: virtual int process_ready_data(void *data, int &event); virtual int process_complete_data(void *data); - virtual int handle_result(void *data, bool is_final, ObProxyResultResp *&result); - virtual int handle_response_result(void *data, bool is_final, ObProxyResultResp *&result); + virtual int handle_result(void *data, bool &is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result); virtual int calc_result(ResultRow &row, ResultRow &result, common::ObIArray &exprs, const int64_t first_res_offset); @@ -205,42 +219,27 @@ protected: class ObProxyOpInput { public: - ObProxyOpInput() : select_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - limit_(-1), offset_(-1), added_row_count_(0), - first_field_offset_(0) {} - - ObProxyOpInput(const common::ObSEArray &select_exprs) - : select_exprs_(select_exprs), limit_(-1), offset_(-1), added_row_count_(0), - first_field_offset_(0) { - } + ObProxyOpInput() : limit_offset_(0), limit_size_(-1), + select_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} virtual ~ObProxyOpInput() {} - void set_select_exprs(const common::ObSEArray &select_exprs) { - select_exprs_.reset(); - select_exprs_ = select_exprs; + int set_select_exprs(const common::ObIArray &select_exprs) { + return select_exprs_.assign(select_exprs); } + void set_limit_offset(int64_t limit_offset) { limit_offset_ = limit_offset; } + int64_t get_limit_offset() { return limit_offset_; } + void set_limit_size(int64_t limit_size) { limit_size_ = limit_size; } + int64_t get_limit_size() { return limit_size_; } + common::ObSEArray& get_select_exprs() { return select_exprs_; } - void set_op_limit(int64_t limit, int64_t offset = -1); - int64_t get_op_limit_value() { return limit_; } - int64_t get_op_offset_value() { return offset_; } - int64_t get_op_top_value() { - return (offset_ != -1) ? (limit_ + offset_) : ((limit_ == 0)? -1 : limit_); - } - void set_added_row_count(int64_t offset) { added_row_count_ = offset; } - int64_t get_added_row_count() { return added_row_count_; } - void set_first_field_offset(int64_t offset) { first_field_offset_ = offset; } - int64_t get_first_field_offset() { return first_field_offset_; } + int64_t get_op_top_value() { return 0; } protected: + int64_t limit_offset_; + int64_t limit_size_; /* All operations need the select_expr, so need put it inito here. */ common::ObSEArray select_exprs_; - - /* All operations need the limit info. */ - int64_t limit_; //ObExpr *limit_; - int64_t offset_; //ObExpr *offset_; - int64_t added_row_count_; - int64_t first_field_offset_; //TODO will be delete }; } diff --git a/src/obproxy/engine/ob_proxy_operator_agg.cpp b/src/obproxy/engine/ob_proxy_operator_agg.cpp index 1371ada7b16f127c855f589ced37823617a605c4..959d9f4f6d7864e0143e6998ef0ed97d43e30f8f 100644 --- a/src/obproxy/engine/ob_proxy_operator_agg.cpp +++ b/src/obproxy/engine/ob_proxy_operator_agg.cpp @@ -54,13 +54,13 @@ int ObProxyAggOp::init() if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(ObColInfoArray)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("init not have enough memory", K(ret), K(sizeof(ObColInfoArray))); - } else if (OB_ISNULL(hash_col_idxs_ = new (tmp_buf) ObColInfoArray(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(hash_col_idxs_ = new (tmp_buf) ObColInfoArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init construct error", K(ret), K(hash_col_idxs_)); } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(SortColumnArray)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("init not have enough memory", K(ret), K(sizeof(SortColumnArray))); - } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init construct error", K(ret), K(sort_columns_)); } else { @@ -94,15 +94,13 @@ int ObProxyAggOp::get_next_row() int ObProxyAggOp::init_group_by_columns() { int ret = common::OB_SUCCESS; - ObProxyAggInput *input = - dynamic_cast(get_input()); + ObProxyAggInput *input = dynamic_cast(get_input()); if (OB_ISNULL(input)) { ret = common::OB_INVALID_ARGUMENT; LOG_WARN("input is invalid in ObProxyAggOp", K(ret), K(input_)); } else { - const ObSEArray& group_by_expr = input->get_group_by_exprs(); - const ObSEArray& select_expr = input->get_select_exprs(); + const ObSEArray& group_by_expr = input->get_group_by_exprs(); ObProxyExpr* expr_ptr = NULL; void *tmp_ptr = NULL; @@ -118,26 +116,6 @@ int ObProxyAggOp::init_group_by_columns() ObSortColumn *new_col= new (tmp_ptr) ObSortColumn(); if (expr_ptr->index_ >= 0) { new_col->index_ = expr_ptr->index_; - } else if (expr_ptr->index_ == -1 && expr_ptr->is_alias()) { - // For 'SLELCT c1 AS a FROM ta GROUP BY a', the expr of 'a' not set index for a not put into - // forehead in select expr. We need to calc correct index for expr of 'a'. - int64_t l = 0; - ObProxyExprShardingConst *shard_const_expr = dynamic_cast(expr_ptr); - if (OB_NOT_NULL(shard_const_expr)) { - for (l=0; lexpr_ == select_expr.at(l)) { - break; - } - } - } - if (OB_ISNULL(shard_const_expr) || l == select_expr.count()) { - //not found expr in select_expr - ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("ObProxyAggOp::init_group_by_columns not found group by expression with alias in" - " select expressions", K(ret), KP(expr_ptr)); - } else { - new_col->index_ = select_expr.count() - 1 - l; - } } else { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("ObProxyAggOp::init_group_by_columns error group by expression", K(ret), KP(expr_ptr)); @@ -191,7 +169,6 @@ int ObProxyAggOp::process_exprs_in_agg(ResultRows *src_rows, ResultRows *obj_row LOG_DEBUG("get one recored from res", K(ret), K(row)); if (OB_FAIL(ObProxyOperator::calc_result(*row, *new_row, select_exprs, added_row_count))) { LOG_WARN("ObProxyProOp::process_ready_data calc result error", K(ret)); - //} else if (OB_FAIL(put_result_row(new_row))) { } else if (OB_FAIL(obj_rows->push_back(new_row))) { LOG_WARN("ObProxyProOp::process_ready_data put row error", K(ret)); } @@ -204,7 +181,7 @@ int ObProxyAggOp::process_exprs_in_agg(ResultRows *src_rows, ResultRows *obj_row return ret; } -int ObProxyAggOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyAggOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; LOG_DEBUG("Enter ObProxyAggOp::handle_response_result", K(op_name()), K(data)); @@ -287,7 +264,7 @@ int ObProxyHashAggOp::init() { return ret; } -int ObProxyHashAggOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyHashAggOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; LOG_DEBUG("Enter ObProxyHashAggOp::handle_response_result", K(op_name()), K(data)); @@ -371,14 +348,14 @@ int ObProxyMergeAggOp::init() ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(tmp_buf), K(sizeof(ResultFlagArray))); } else if (OB_ISNULL(result_rows_flag_array_ - = new (tmp_buf) ResultFlagArray(array_new_alloc_size, allocator_))) { + = new (tmp_buf) ResultFlagArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("ObProxyAggOp::init error for construct", K(ret), K(result_rows_flag_array_)); } else if (OB_ISNULL(tmp_buf = (char*)allocator_.alloc(sizeof(ResultRespArray)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(tmp_buf), K(sizeof(ResultRespArray))); } else if (OB_ISNULL(regions_results_ - = new (tmp_buf) ResultRespArray(array_new_alloc_size, allocator_))) { + = new (tmp_buf) ResultRespArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("ObProxyAggOp::init error for construct", K(ret), K(regions_results_)); } @@ -386,7 +363,7 @@ int ObProxyMergeAggOp::init() return ret; } -int ObProxyMergeAggOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyMergeAggOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; LOG_DEBUG("Enter ObProxyMergeAggOp::handle_response_result", K(op_name()), K(data)); @@ -484,7 +461,7 @@ int ObProxyMergeAggOp::fetch_all_result(ResultRows *rows) if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); - } else if (OB_ISNULL(new_rows = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(new_rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init ResultRows failed", K(ret), K(op_name()), K(new_rows)); } else if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObMemorySort)))) { @@ -583,6 +560,502 @@ int ObProxyMergeAggOp::fetch_all_result(ResultRows *rows) return ret; } +ObProxyStreamAggOp::~ObProxyStreamAggOp() +{ + if (NULL != current_group_unit_) { + ObProxyGroupUnit::destroy_group_unit(allocator_, current_group_unit_); + current_group_unit_ = NULL; + } +} + +int ObProxyStreamAggOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) +{ + int ret = OB_SUCCESS; + + ObProxyResultResp *opres = NULL; + ObProxyAggInput *input = NULL; + + if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input, data is NULL", K(ret)); + } else if (OB_ISNULL(opres = reinterpret_cast(data))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input, opres type is not match", K(ret)); + } else if (!opres->is_resultset_resp()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("resp is not resultset", K(opres), K(ret)); + } else if (OB_ISNULL(input = dynamic_cast(get_input()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input is invalid", K(ret)); + } else { + const ObSEArray& group_exprs = input->get_group_by_exprs(); + const ObSEArray& agg_exprs = input->get_agg_exprs(); + int64_t limit_offset = get_input()->get_limit_offset(); + int64_t limit_offset_size = get_input()->get_limit_size() + limit_offset; + + if (OB_ISNULL(get_result_fields())) { + result_fields_ = opres->get_fields(); + if (OB_ISNULL(result_fields_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no result field, unexpected", K(ret)); + } + } + + ResultRow *row = NULL; + while (OB_SUCC(ret) && OB_SUCC(opres->next(row))) { + ObProxyGroupUnit group_unit(allocator_); + if (OB_FAIL(group_unit.init(row, group_exprs))) { + LOG_WARN("fail to init group unit", K(ret)); + } else if (NULL == current_group_unit_) { + if (OB_FAIL(ObProxyGroupUnit::create_group_unit(allocator_, current_group_unit_, group_unit))) { + LOG_WARN("fail to create group unit", K(ret)); + } + } else if (*current_group_unit_ == group_unit) { + if (OB_FAIL(current_group_unit_->aggregate(group_unit, agg_exprs))) { + LOG_WARN("fail to aggregate", K(ret)); + } + } else { + if (OB_FAIL(current_group_unit_->set_agg_value())) { + LOG_WARN("fail to set agg value", K(ret)); + } else if (OB_FAIL(current_rows_.push_back(current_group_unit_->get_row()))) { + LOG_WARN("fail to push back row", K(ret)); + } else { + ObProxyGroupUnit::destroy_group_unit(allocator_, current_group_unit_); + current_group_unit_ = NULL; + if (limit_offset_size > 0 && current_rows_.count() == limit_offset_size) { + is_final = true; + break; + } + + if (OB_FAIL(ObProxyGroupUnit::create_group_unit(allocator_, current_group_unit_, group_unit))) { + LOG_WARN("fail to create group unit", K(ret)); + } + } + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + if (is_final && NULL != current_group_unit_) { + if (OB_FAIL(current_group_unit_->set_agg_value())) { + LOG_WARN("fail to set agg value", K(ret)); + } else if (OB_FAIL(current_rows_.push_back(current_group_unit_->get_row()))) { + LOG_WARN("fail to push back row", K(ret)); + } else { + ObProxyGroupUnit::destroy_group_unit(allocator_, current_group_unit_); + current_group_unit_ = NULL; + } + } + } + + if (OB_SUCC(ret) && (!current_rows_.empty() || is_final)) { + ObProxyResultResp *res = NULL; + ResultRows *rows = NULL; + void *tmp_buf = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ResultRows), K(ret)); + } else if (FALSE_IT(rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { + // impossible + } else { + int64_t count = current_rows_.count(); + if (limit_offset_size > 0) { + count = count > limit_offset_size ? limit_offset_size : count; + } + + for (int64_t i = limit_offset; OB_SUCC(ret) && i < count; i++) { + if (OB_FAIL(rows->push_back(current_rows_.at(i)))) { + LOG_WARN("fail to push back row", K(i), K(count), K(limit_offset), K(limit_offset_size), K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + if (FALSE_IT(current_rows_.reuse())) { + // impossible + } else if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { + LOG_WARN("fail to packet resultset", K(op_name()), K(ret)); + } + } + + result = res; + } + } + + return ret; +} + +ObProxyMemMergeAggOp::~ObProxyMemMergeAggOp() +{ + GroupUnitHashMap::iterator it = group_unit_map_.begin(); + GroupUnitHashMap::iterator tmp_it; + GroupUnitHashMap::iterator end = group_unit_map_.end(); + for (; it != end;) { + tmp_it = it; + ++it; + ObProxyGroupUnit::destroy_group_unit(allocator_, &(*tmp_it)); + } +} + +int ObProxyMemMergeAggOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) +{ + int ret = OB_SUCCESS; + + ObProxyResultResp *opres = NULL; + ObProxyAggInput *input = NULL; + + if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input, data is NULL", K(ret)); + } else if (OB_ISNULL(opres = reinterpret_cast(data))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input, opres type is not match", K(ret)); + } else if (!opres->is_resultset_resp()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("resp is not resultset", K(opres), K(ret)); + } else if (OB_ISNULL(input = dynamic_cast(get_input()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input is invalid", K(ret)); + } else { + const ObSEArray& group_exprs = input->get_group_by_exprs(); + const ObSEArray& agg_exprs = input->get_agg_exprs(); + + if (OB_ISNULL(get_result_fields())) { + result_fields_ = opres->get_fields(); + if (OB_ISNULL(result_fields_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no result field, unexpected", K(ret)); + } + } + + ResultRow *row = NULL; + while (OB_SUCC(ret) && OB_SUCC(opres->next(row))) { + ObProxyGroupUnit group_unit(allocator_); + ObProxyGroupUnit *current_group_unit = NULL; + if (OB_FAIL(group_unit.init(row, group_exprs))) { + LOG_WARN("fail to init group unit", K(ret)); + } else if (OB_FAIL(group_unit_map_.get_refactored(group_unit, current_group_unit))) { + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(ObProxyGroupUnit::create_group_unit(allocator_, current_group_unit, group_unit))) { + LOG_WARN("fail to create group unit", K(ret)); + } else if (OB_FAIL(group_unit_map_.set_refactored(current_group_unit))) { + LOG_WARN("fail to set group unit", K(ret)); + } + } else { + LOG_WARN("fail to get group unit", K(ret)); + } + } else if (OB_FAIL(current_group_unit->aggregate(group_unit, agg_exprs))) { + LOG_WARN("fail to aggregate", K(ret)); + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + + if (OB_SUCC(ret) && is_final) { + ObProxyResultResp *res = NULL; + ResultRows *rows = NULL; + void *tmp_buf = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ResultRows), K(ret)); + } else if (FALSE_IT(rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { + // impossible + } else { + GroupUnitHashMap::iterator it = group_unit_map_.begin(); + GroupUnitHashMap::iterator tmp_it; + GroupUnitHashMap::iterator end = group_unit_map_.end(); + for (; OB_SUCC(ret) && it != end; ) { + if (OB_FAIL(it->set_agg_value())) { + LOG_WARN("fail to set agg value", K(ret)); + } else if (OB_FAIL(rows->push_back(it->get_row()))) { + LOG_WARN("fail to push back row", K(ret)); + } else { + tmp_it = it; + ++it; + ObProxyGroupUnit::destroy_group_unit(allocator_, &(*tmp_it)); + } + } + group_unit_map_.reset(); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { + LOG_WARN("fail to packet resultset", K(op_name()), K(ret)); + } + } + result = res; + } + } + + return ret; +} + +int ObProxyGroupUnit::create_group_unit(common::ObIAllocator &allocator, + ObProxyGroupUnit* ¤t_group_unit, + ObProxyGroupUnit &group_unit) +{ + int ret = OB_SUCCESS; + void *buf = NULL; + current_group_unit = NULL; + if (OB_ISNULL(buf = (char*)(allocator.alloc(sizeof(ObProxyGroupUnit))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc group unit buf", K(ret)); + } else if (OB_ISNULL(current_group_unit = new (buf)ObProxyGroupUnit(allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new group unit", K(ret)); + } else if (OB_FAIL(current_group_unit->assign(group_unit))) { + LOG_WARN("fail to assign group unit", K(ret)); + } + + return ret; +} + +void ObProxyGroupUnit::destroy_group_unit(common::ObIAllocator &allocator, + ObProxyGroupUnit* group_unit) +{ + group_unit->~ObProxyGroupUnit(); + allocator.free(group_unit); +} + +ObProxyGroupUnit::~ObProxyGroupUnit() +{ + for (int64_t i = 0; i < agg_units_.count(); i++) { + ObProxyAggUnit *agg_unit = agg_units_.at(i); + ObProxyAggUnit::destroy_agg_unit(allocator_, agg_unit); + } +} + +int ObProxyGroupUnit::init(ResultRow *row, const ObIArray& group_exprs) +{ + int ret = OB_SUCCESS; + + ObProxyExprCtx ctx(0, dbconfig::TESTLOAD_NON, false, &allocator_); + ObProxyExprCalcItem calc_item(row); + + for (int64_t i = 0; OB_SUCC(ret) && i < group_exprs.count(); i++) { + if (OB_FAIL(group_exprs.at(i)->calc(ctx, calc_item, group_values_))) { + LOG_WARN("fail to calc group exprs", K(ret)); + } + } + + if (OB_SUCC(ret)) { + row_ = row; + } + + return ret; +} + +uint64_t ObProxyGroupUnit::hash() const +{ + uint64_t hash = 0; + for (int64_t i = 0; i < group_values_.count(); i++) { + hash = group_values_.at(i).hash(hash); + } + + return hash; +} + +bool ObProxyGroupUnit::operator==(const ObProxyGroupUnit &group_unit) const +{ + bool bret = true; + const ObIArray &group_values = group_unit.get_group_values(); + if (group_values_.count() != group_values.count()) { + bret = false; + } else { + for (int64_t i = 0; bret && i < group_values_.count(); i++) { + bret = (group_values_.at(i) == group_values.at(i)); + } + } + + return bret; +} + +int ObProxyGroupUnit::assign(const ObProxyGroupUnit &group_unit) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(group_values_.assign(group_unit.get_group_values()))) { + LOG_WARN("fail to assign group value", K(ret)); + } else { + row_ = group_unit.get_row(); + } + + return ret; +} + +int ObProxyGroupUnit::do_aggregate(ResultRow *row) +{ + int ret = OB_SUCCESS; + + ObProxyExprCtx ctx(0, dbconfig::TESTLOAD_NON, false, &allocator_); + ObProxyExprCalcItem calc_item(row); + common::ObSEArray agg_values; + + for (int64_t i = 0; OB_SUCC(ret) && i < agg_units_.count(); i++) { + ObProxyAggUnit *agg_unit = agg_units_.at(i); + agg_values.reuse(); + if (OB_FAIL(agg_unit->get_agg_expr()->calc(ctx, calc_item, agg_values))) { + LOG_WARN("fail to calc agg expr", K(ret)); + } else if (OB_FAIL(agg_unit->merge(agg_values))) { + LOG_WARN("fail to merge agg value", K(ret)); + } + } + + return ret; +} + +int ObProxyGroupUnit::aggregate(const ObProxyGroupUnit &group_unit, const ObIArray& agg_exprs) +{ + int ret = OB_SUCCESS; + + if (agg_units_.empty() && !agg_exprs.empty()) { + for (int64_t i = 0; OB_SUCC(ret) && i < agg_exprs.count(); i++) { + ObProxyExpr *agg_expr = agg_exprs.at(i); + ObProxyAggUnit *agg_unit = NULL; + if (OB_FAIL(ObProxyAggUnit::create_agg_unit(allocator_, agg_expr, agg_unit))) { + LOG_WARN("fail to create agg unit", "agg type", agg_expr->get_expr_type(), K(ret)); + } else if (FALSE_IT(agg_unit->set_agg_expr(agg_expr))) { + LOG_WARN("fail to set agg expr", K(ret)); + } else if (agg_units_.push_back(agg_unit)) { + LOG_WARN("fail to push back agg unit", K(ret)); + } + } + + if (OB_SUCC(ret)) { + // Process the first row of data + if (OB_FAIL(do_aggregate(row_))) { + LOG_WARN("fail to do aggregate", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + // Aggregate subsequent data + if (OB_FAIL(do_aggregate(group_unit.get_row()))) { + LOG_WARN("fail to do aggregate", K(ret)); + } + } + + return ret; +} + +int ObProxyGroupUnit::set_agg_value() +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < agg_units_.count(); i++) { + ObProxyAggUnit *agg_unit = agg_units_.at(i); + int64_t index = agg_unit->get_agg_expr()->get_index(); + *(row_->at(index)) = agg_unit->get_result(); + } + + return ret; +} + +int ObProxyAggUnit::create_agg_unit(ObIAllocator &allocator, + ObProxyExpr *expr, + ObProxyAggUnit* &agg_unit) +{ + int ret = OB_SUCCESS; + void* buf = NULL; + ObProxyExprType expr_type = expr->get_expr_type(); +#define ALLOC_AGG_UNIT_BY_TYPE(ExprClass, args) \ + if (OB_ISNULL(buf = (allocator.alloc(sizeof(ExprClass))))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("fail to alloc mem", K(ret)); \ + } else if (OB_ISNULL(agg_unit = new (buf)ExprClass(allocator, args))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("fail to new expr", K(ret)); \ + } + + switch(expr_type) { + case OB_PROXY_EXPR_TYPE_FUNC_MAX: + ALLOC_AGG_UNIT_BY_TYPE(ObProxyComparableAggUnit, false); + break; + case OB_PROXY_EXPR_TYPE_FUNC_MIN: + ALLOC_AGG_UNIT_BY_TYPE(ObProxyComparableAggUnit, true); + break; + case OB_PROXY_EXPR_TYPE_FUNC_COUNT: + case OB_PROXY_EXPR_TYPE_FUNC_SUM: + ALLOC_AGG_UNIT_BY_TYPE(ObProxyAccumulationAggUnit, expr->get_accuracy().get_scale()); + break; + default: + ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; + LOG_WARN("unexpected type", K(expr_type)); + break; + } + + return ret; +} + +void ObProxyAggUnit::destroy_agg_unit(ObIAllocator &allocator, + ObProxyAggUnit *agg_unit) +{ + agg_unit->~ObProxyAggUnit(); + allocator.free(agg_unit); +} + +int ObProxyComparableAggUnit::merge(common::ObIArray &agg_values) +{ + int ret = OB_SUCCESS; + + if (!agg_values.empty()) { + ObObj obj = agg_values.at(0); + if (is_first_) { + obj_ = obj; + is_first_ = false; + } else if (!obj.is_null() && obj_.is_null()) { + obj_ = obj; + } else if (!obj.is_null()) { + int cmp = obj.compare(obj_); + if ((asc_ && cmp < 0) || (!asc_ && cmp > 0)) { + obj_ = obj; + } + } + } + + return ret; +} + +int ObProxyAccumulationAggUnit::merge(common::ObIArray &agg_values) +{ + int ret = OB_SUCCESS; + + if (!agg_values.empty()) { + ObObj obj = agg_values.at(0); + if (is_first_) { + obj_ = obj; + is_first_ = false; + } else if (!obj.is_null() && obj_.is_null()) { + obj_ = obj; + } else if (!obj.is_null()) { + ObProxyExprCtx ctx(0, dbconfig::TESTLOAD_NON, false, &allocator_); + ObProxyExprCalcItem calc_item; + ObProxyExprAdd add_expr; + ObProxyExprConst first_expr; + ObProxyExprConst second_expr; + ObSEArray result_obj; + + ctx.set_scale(scale_); + first_expr.set_object(obj_); + second_expr.set_object(agg_values.at(0)); + if (OB_FAIL(add_expr.add_param_expr(&first_expr))) { + LOG_WARN("fail to add first expr", K(ret)); + } else if (OB_FAIL(add_expr.add_param_expr(&second_expr))) { + LOG_WARN("fail to add second expr", K(ret)); + } else if (add_expr.calc(ctx, calc_item, result_obj)) { + LOG_WARN("fail to calc", K(ret)); + } else { + obj_ = result_obj.at(0); + } + } + } + + return ret; +} + ObAggregateFunction::ObAggregateFunction(common::ObIAllocator &allocator, common::ObSEArray &select_exprs) : //expr_ctx_(NULL, NULL, NULL, &allocator, NULL), @@ -594,8 +1067,8 @@ ObAggregateFunction::ObAggregateFunction(common::ObIAllocator &allocator, sort_columns_(NULL), agg_rows_(NULL), result_rows_(NULL) - //agg_rows_(*(new ResultRows(array_new_alloc_size, allocator))) - //result_rows(*(new ResultRows(array_new_alloc_size, allocator_))) + //agg_rows_(*(new ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator))) + //result_rows(*(new ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) {} ObAggregateFunction::~ObAggregateFunction() @@ -632,7 +1105,7 @@ int ObAggregateFunction::handle_all_result(ResultRow *&row) if (OB_ISNULL(buf)) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("handle_all_result not have enough memory", K(ret), K(sizeof(sizeof(ResultRow)))); - } else if (OB_ISNULL(row = new (buf) ResultRow(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(row = new (buf) ResultRow(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERROR; LOG_WARN("handle_all_result init ResultRow error", K(ret)); } else if (agg_rows_->count() > 0 @@ -721,7 +1194,7 @@ int ObAggregateFunction::handle_all_hash_result(ResultRows *rows) if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(sizeof(ResultRows))); - } else if (OB_ISNULL(new_rows = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(new_rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init ResultRows failed", K(ret), K(new_rows)); } else if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObMemorySort)))) { @@ -822,19 +1295,19 @@ int ObAggregateFunction::init(ObColInfoArray &group_col_idxs) // } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(ObColInfoArray)))) { // ret = common::OB_ALLOCATE_MEMORY_FAILED; // LOG_WARN("init not have enough memory", K(ret), K(sizeof(ObColInfoArray))); -// } else if (OB_ISNULL(group_col_idxs_ = new (tmp_buf) ObColInfoArray(array_new_alloc_size, allocator_))) { +// } else if (OB_ISNULL(group_col_idxs_ = new (tmp_buf) ObColInfoArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { // ret = common::OB_ERR_UNEXPECTED; // LOG_WARN("init construct error", K(ret), K(group_col_idxs_)); } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(SortColumnArray)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("init not have enough memory", K(ret), K(sizeof(SortColumnArray))); - } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init construct error", K(ret), K(sort_columns_)); } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(ResultRows)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("init not have enough memory", K(ret), K(sizeof(ResultRows))); - } else if (OB_ISNULL(agg_rows_ = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(agg_rows_ = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init construct error", K(ret), K(agg_rows_)); } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(HashTable)))) { @@ -1060,7 +1533,6 @@ int ObAggregateFunction::calc_aggr_cell(const ObProxyExprType aggr_fun, case OB_PROXY_EXPR_TYPE_NONE: case OB_PROXY_EXPR_TYPE_CONST: case OB_PROXY_EXPR_TYPE_SHARDING_CONST: - case OB_PROXY_EXPR_TYPE_SHARDING_ALIAS: case OB_PROXY_EXPR_TYPE_COLUMN: case OB_PROXY_EXPR_TYPE_FUNC_HASH: case OB_PROXY_EXPR_TYPE_FUNC_SUBSTR: diff --git a/src/obproxy/engine/ob_proxy_operator_agg.h b/src/obproxy/engine/ob_proxy_operator_agg.h index c3e8a56e655d1eab7da635b9567c5564033230d2..223d84968a6ed8cca1a1c5c2ee7c2f170d0439d3 100644 --- a/src/obproxy/engine/ob_proxy_operator_agg.h +++ b/src/obproxy/engine/ob_proxy_operator_agg.h @@ -25,6 +25,7 @@ namespace obproxy { namespace engine { typedef common::ObSEArray ObColInfoArray; +class ObProxyAggUnit; class ObAggregateFunction; class HashTable; class ObHashCols; @@ -52,7 +53,7 @@ class HashTable public: HashTable(common::ObIAllocator &allocator) : allocator_(allocator), - buckets_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), + buckets_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), nbuckets_(0), buf_cnt_(0), cur_(0) @@ -262,7 +263,7 @@ public: } virtual int init_group_by_columns(); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); virtual int process_exprs_in_agg(ResultRows *src_rows, ResultRows *obj_rows); protected: @@ -277,45 +278,30 @@ class ObProxyAggInput : public ObProxyOpInput public: ObProxyAggInput() : ObProxyOpInput(), - group_by_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - having_exprs_(NULL) {} - ObProxyAggInput( - const common::ObSEArray &select_exprs, - const common::ObSEArray &group_by_exprs, - ObProxyExpr *&having_exprs) - : ObProxyOpInput(select_exprs), - group_by_exprs_(group_by_exprs), - having_exprs_(having_exprs) {} + group_by_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + agg_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} ~ObProxyAggInput() {} - void init(const common::ObSEArray &select_exprs, - const common::ObSEArray &group_by_exprs, - ObProxyExpr *&having_exprs) { - select_exprs_ = select_exprs; - group_by_exprs_ = group_by_exprs; - having_exprs_ = having_exprs; + int set_group_by_exprs(const common::ObIArray &group_by_exprs) { + return group_by_exprs_.assign(group_by_exprs); } - void set_group_by_exprs(const common::ObSEArray &group_by_exprs) { - group_by_exprs_ = group_by_exprs; + common::ObSEArray& get_group_by_exprs() { + return group_by_exprs_; } - void set_having_exprs(ObProxyExpr* having_exprs) { - having_exprs_ = having_exprs; + int set_agg_exprs(const common::ObIArray &agg_exprs) { + return agg_exprs_.assign(agg_exprs); } - common::ObSEArray& get_group_by_exprs() { - return group_by_exprs_; - } - - ObProxyExpr* get_having_exprs() { - return having_exprs_; + common::ObSEArray& get_agg_exprs() { + return agg_exprs_; } protected: - common::ObSEArray group_by_exprs_; - ObProxyExpr* having_exprs_; + common::ObSEArray group_by_exprs_; + common::ObSEArray agg_exprs_; }; class ObProxyHashAggOp : public ObProxyAggOp @@ -329,7 +315,7 @@ public: ~ObProxyHashAggOp() {}; virtual int init(); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); private: HashTable *result_rows_; @@ -348,7 +334,7 @@ public: virtual int init(); virtual int init_result_rows_array(int64_t regions); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); int fetch_all_result(ResultRows *rows); protected: @@ -360,6 +346,152 @@ protected: ResultRespArray *regions_results_; }; +class ObProxyGroupUnit +{ +public: + ObProxyGroupUnit(common::ObIAllocator &allocator) + : allocator_(allocator), row_(NULL), + group_values_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + agg_units_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} + ~ObProxyGroupUnit(); + + int init(ResultRow *row, const common::ObIArray& group_by_exprs); + + uint64_t hash() const; + bool operator==(const ObProxyGroupUnit &group_unit) const; + int assign(const ObProxyGroupUnit &group_unit); + + int aggregate(const ObProxyGroupUnit &group_unit, + const common::ObIArray& agg_exprs); + int set_agg_value(); + + ResultRow *get_row() const { return row_; } + const common::ObIArray& get_group_values() const { return group_values_; } + const common::ObIArray& get_agg_units() { return agg_units_;; } + +public: + static int create_group_unit(common::ObIAllocator &allocator, + ObProxyGroupUnit* ¤t_group_unit, + ObProxyGroupUnit &group_unit); + static void destroy_group_unit(common::ObIAllocator &allocator, + ObProxyGroupUnit* group_unit); + +private: + int do_aggregate(ResultRow *row); + +public: + LINK(ObProxyGroupUnit, group_unit_link_); + +private: + common::ObIAllocator &allocator_; + ResultRow *row_; + common::ObSEArray group_values_; + common::ObSEArray agg_units_; +}; + +class ObProxyStreamAggOp : public ObProxyAggOp +{ +public: + ObProxyStreamAggOp(ObProxyOpInput *input, common::ObIAllocator &allocator) + : ObProxyAggOp(input, allocator), current_group_unit_(NULL), + current_rows_(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator) + { set_op_type(PHY_STREAM_AGG); } + + ~ObProxyStreamAggOp(); + + virtual int init() { return ObProxyOperator::init(); } + virtual int get_next_row() { return ObProxyOperator::get_next_row(); } + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); + +private: + ObProxyGroupUnit *current_group_unit_; + ResultRows current_rows_; +}; + +class ObProxyMemMergeAggOp : public ObProxyAggOp +{ +public: + ObProxyMemMergeAggOp(ObProxyOpInput *input, common::ObIAllocator &allocator) + : ObProxyAggOp(input, allocator), group_unit_map_() + { set_op_type(PHY_MEM_MERGE_AGG); } + + ~ObProxyMemMergeAggOp(); + + virtual int init() { return ObProxyOperator::init(); } + virtual int get_next_row() { return ObProxyOperator::get_next_row(); } + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); + +public: + struct ObGroupUnitHashing + { + typedef const ObProxyGroupUnit &Key; + typedef ObProxyGroupUnit Value; + typedef ObDLList(ObProxyGroupUnit, group_unit_link_) ListHead; + + static uint64_t hash(Key key) { return key.hash(); } + static Key key(Value *value) { return *value; } + static bool equal(Key lhs, Key rhs) { return lhs == rhs; } + }; + typedef common::hash::ObBuildInHashMap GroupUnitHashMap; + +private: + GroupUnitHashMap group_unit_map_; +}; + +class ObProxyAggUnit +{ +public: + ObProxyAggUnit(common::ObIAllocator &allocator) + : allocator_(allocator), agg_expr_(NULL), obj_(), is_first_(true) {} + ~ObProxyAggUnit() {} + + static int create_agg_unit(common::ObIAllocator &allocator, + ObProxyExpr *expr, + ObProxyAggUnit* &agg_unit); + static void destroy_agg_unit(common::ObIAllocator &allocator, + ObProxyAggUnit *agg_unit); + + virtual int merge(common::ObIArray &agg_values) = 0; + virtual ObObj &get_result() { return obj_; }; + + void set_agg_expr(ObProxyExpr* agg_expr) { agg_expr_ = agg_expr; } + ObProxyExpr* get_agg_expr() { return agg_expr_; } + + TO_STRING_KV(KP_(agg_expr), K_(obj), K_(is_first)); + +protected: + common::ObIAllocator &allocator_; + ObProxyExpr* agg_expr_; + ObObj obj_; + bool is_first_; +}; + +class ObProxyComparableAggUnit : public ObProxyAggUnit +{ +public: + ObProxyComparableAggUnit(common::ObIAllocator &allocator, bool asc) + : ObProxyAggUnit(allocator), asc_(asc) {} + ~ObProxyComparableAggUnit() {} + + virtual int merge(common::ObIArray &agg_values); + +private: + bool asc_; +}; + +class ObProxyAccumulationAggUnit : public ObProxyAggUnit +{ +public: + ObProxyAccumulationAggUnit(common::ObIAllocator &allocator, int64_t scale) + : ObProxyAggUnit(allocator), scale_(scale) {} + ~ObProxyAccumulationAggUnit() {} + + virtual int merge(common::ObIArray &agg_values); + +private: + int64_t scale_; +}; + } } } diff --git a/src/obproxy/engine/ob_proxy_operator_async_task.cpp b/src/obproxy/engine/ob_proxy_operator_async_task.cpp index 736bda0bd0d102d91341b7a9f072016ea35d09cd..554c46a9b1665f79d4a844075587cf2714a8a7b2 100644 --- a/src/obproxy/engine/ob_proxy_operator_async_task.cpp +++ b/src/obproxy/engine/ob_proxy_operator_async_task.cpp @@ -148,7 +148,6 @@ int ObOperatorAsyncCommonTask::main_handler(int event, void *data) if (!terminate_) { if (OB_SUCC(ret)) { if (event_ret == VC_EVENT_READ_READY || event_ret == VC_EVENT_READ_COMPLETE) { - LOG_DEBUG("ObOperatorAsyncCommonTask enter next OP", K(event_ret), KP_(ob_operator), "op_name", ob_operator_->op_name()); cb_cont_->handle_event(event_ret, ob_operator_->get_operator_result()); } diff --git a/src/obproxy/engine/ob_proxy_operator_projection.cpp b/src/obproxy/engine/ob_proxy_operator_projection.cpp index da6dd2968806e0200ae92002eed2929841171e2d..ffdd74c0f1abdd78117ad1004d9c9438b4cb10b0 100644 --- a/src/obproxy/engine/ob_proxy_operator_projection.cpp +++ b/src/obproxy/engine/ob_proxy_operator_projection.cpp @@ -23,16 +23,13 @@ int ObProxyProOp::get_next_row() return ObProxyOperator::get_next_row(); } -//int ObProxyProOp::handle_response_result(executor::ObProxyParallelResp *pres, bool is_final, ObProxyResultResp *&result) -int ObProxyProOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyProOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; - LOG_DEBUG("Enter ObProxyProOp::handle_response_result", K(op_name()), K(data)); ObProxyResultResp *opres = NULL; - ResultFields *origin_fields = NULL; ObProxyResultResp *res = NULL; - void *tmp_buf = NULL; + ObProxyProInput *input = NULL; if (OB_ISNULL(data)) { ret = common::OB_INVALID_ARGUMENT; @@ -43,129 +40,98 @@ int ObProxyProOp::handle_response_result(void *data, bool is_final, ObProxyResu } else if (!opres->is_resultset_resp()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObProxyProOp::handle_response_result not response result", K(data), KP(opres), K(opres->is_resultset_resp())); + } else if (OB_ISNULL(input = dynamic_cast(get_input()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input is invalid", K(ret)); } else { - LOG_DEBUG("ObProxyProOp::process_ready_data:resultset_resp", K(opres), KP(opres)); - expr_has_calced_ = opres->get_has_calc_exprs(); - LOG_DEBUG("ObProxyProOp::process_ready_data resultset", K(opres), K(opres->get_fields())); if (OB_ISNULL(get_result_fields())) { result_fields_ = opres->get_fields(); } + common::ObIArray &calc_exprs = input->get_calc_exprs(); + int64_t derived_column_count = input->get_derived_column_count(); + int64_t real_column_count = opres->get_fields()->count() - derived_column_count; + int64_t limit_offset = input->get_limit_offset(); + int64_t limit_offset_size = input->get_limit_size() + limit_offset; ResultRow *row = NULL; + while (OB_SUCC(ret) && OB_SUCC(opres->next(row))) { + ObProxyExprCtx ctx(0, dbconfig::TESTLOAD_NON, false, &allocator_); + ObProxyExprCalcItem calc_item(row); + ObSEArray result_obj_array; - common::ObSEArray& select_exprs = - get_input()->get_select_exprs(); - - int64_t limit_start = get_input()->get_op_limit_value(); - int64_t limit_offset = get_input()->get_op_offset_value(); - int64_t limit_topv = get_input()->get_op_top_value(); - - int64_t added_row_count = get_input()->get_added_row_count(); - LOG_DEBUG("get all recored from res", K(ret), K(opres), K(opres->get_result_rows().count()), - K(select_exprs.count()), K(added_row_count), K(limit_start), K(limit_offset), K(limit_topv)); - if (limit_topv != -1 && cur_result_rows_->count() >= limit_topv) { - //reach up limit in SELECT - LOG_DEBUG("not need to projection result any more, for reached the limit", K(ret), K(limit_topv)); - } else { - while (OB_SUCC(ret) && (OB_SUCC(opres->next(row)))) { - ResultRow *new_row = NULL; - if (OB_ISNULL(row)) { - LOG_DEBUG("ObProxyProOp::process_ready_data, handle all data", K(ret)); - break; - } else if (OB_FAIL(init_row(new_row))) { - LOG_WARN("ObProxyProOp::process_ready_data init row error", K(ret)); - ret = common::OB_ERROR; - break; + for (int64_t i = 0; OB_SUCC(ret) && i < calc_exprs.count(); i++) { + result_obj_array.reuse(); + ObProxyExpr *calc_expr = calc_exprs.at(i); + if (OB_FAIL(calc_expr->calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("fail to get next row", K(i), K(ret)); + } else { + *row->at(calc_expr->get_index()) = result_obj_array.at(0); } + } - LOG_DEBUG("get one recored from res", K(ret), K(opres), K(row)); - if (OB_FAIL(ObProxyOperator::calc_result(*row, *new_row, select_exprs, added_row_count))) { - LOG_WARN("ObProxyProOp::process_ready_data calc result error", K(ret)); - } else if (OB_FAIL(put_result_row(new_row))) { - LOG_WARN("ObProxyProOp::process_ready_data put row error", K(ret)); + if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < derived_column_count; i++) { + row->pop_back(); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(current_rows_.push_back(row))) { + LOG_WARN("fail to push back row", K(ret)); + } else if (limit_offset_size > 0 && current_rows_.count() == limit_offset_size) { + is_final = true; + break; } } } + if (ret == common::OB_ITER_END) { ret = common::OB_SUCCESS; - LOG_DEBUG("ObProxyProOp::process_ready_data, handle all data", K(ret)); - } - if (OB_SUCC(ret)) { - expr_has_calced_ = true; } + if (is_final) { - LOG_DEBUG("ObProxyOperator::process_complete_data get fields", K(ret), - K(get_input()->get_added_row_count())); - if (OB_ISNULL(origin_fields = get_result_fields())) { + void *tmp_buf_rows = NULL; + void *tmp_buf_fields = NULL; + if (OB_ISNULL(result_fields_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObProxyProOp::handle_response_result not result field", K(data), KP(result_fields_)); - } else if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultFields)))) { + } else if (OB_ISNULL(tmp_buf_rows = allocator_.alloc(sizeof(ResultRows)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ResultRows), K(ret)); + } else if (OB_ISNULL(tmp_buf_fields = allocator_.alloc(sizeof(ResultFields)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultFields))); + LOG_WARN("no have enough memory to init", K(op_name()), K(sizeof(ResultFields)), K(ret)); } else { - ResultFields &new_fields = *(new (tmp_buf) ResultFields(array_new_alloc_size, allocator_)); - for (int64_t i = 0; i < origin_fields->count() - get_input()->get_added_row_count(); i++) { - new_fields.push_back(origin_fields->at(i)); + ResultRows *rows = new (tmp_buf_rows) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); + ResultFields *new_fields = (new (tmp_buf_fields) ResultFields(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_)); + + for (int64_t i = 0; i < real_column_count; i++) { + new_fields->push_back(result_fields_->at(i)); } - - LOG_DEBUG("ObProxyOperator::process_complete_data packet final resultset", K(ret)); - if (OB_FAIL(get_limit_result(get_input()->get_op_limit_value(), get_input()->get_op_offset_value(), - cur_result_rows_))) { - LOG_WARN("packet resultset packet limit error", K(ret)); - } else if (OB_FAIL(packet_result_set(res, cur_result_rows_, &new_fields))) { - LOG_WARN("packet resultset packet error", K(ret)); + + int64_t count = current_rows_.count(); + if (limit_offset_size > 0) { + count = count > limit_offset_size ? limit_offset_size : count; } - res->set_column_count(new_fields.count()); - LOG_DEBUG("ObProxyOperator::process_complete_data packet final resultset over", K(ret), K(res), - K(res->get_column_count()), K(cur_result_rows_), KPC(cur_result_rows_), K(new_fields)); - } - } - result = res; - } - return ret; -} -int ObProxyProOp::get_limit_result(int64_t start, int64_t offset, ResultRows *rows) -{ - int ret = common::OB_SUCCESS; - if (OB_ISNULL(rows)) { - ret = common::OB_INVALID_ARGUMENT; - LOG_WARN("ObProxyProOp::get_limit_result invalid input", K(ret), K(rows)); - } else { - LOG_DEBUG("ObProxyProOp::get_limit_result remove rows by limit start", K(ret), K(start), K(offset), - K(rows->count()), K(*rows)); - if (start + offset < 0) { //no limit do nothing - LOG_DEBUG("ObProxyProOp::get_limit_result not value to limit", - K(ret), K(start), K(offset), K(rows->count())); - } else if (offset == -1) { // limt start, such as limit 10 - if (rows->count() > start) { - while (rows->count() > start) { - if (OB_FAIL(rows->remove(rows->count() - 1))) { //delete one record from backend and count_ will be minus - LOG_WARN("ObProxyProOp::get_limit_result remove rows failed", K(ret), K(start), K(offset)); - break; + for (int64_t i = limit_offset; OB_SUCC(ret) && i < count; i++) { + if (OB_FAIL(rows->push_back(current_rows_.at(i)))) { + LOG_WARN("fail to push back row", K(i), K(count), K(limit_offset), K(limit_offset_size), K(ret)); } } - } - } else { //limit start, offset, such as limit 10, 2 - int64_t i = 0; - for (i = 0; i < offset; i++) { - if (i + start >= rows->count()) { - break; - } - rows->at(i) = rows->at(i + start); - } - while (rows->count() > i) { - if (OB_FAIL(rows->remove(rows->count() - 1))) { //delete one record from backend and count_ will be minus - LOG_WARN("ObProxyProOp::get_limit_result remove rows failed", K(ret), K(start), K(offset)); - break; + + if (OB_SUCC(ret)) { + if (OB_FAIL(packet_result_set(res, rows, new_fields))) { + LOG_WARN("packet resultset packet error", K(ret)); + } else if (OB_NOT_NULL(res)) { + res->set_column_count(new_fields->count()); + } } } } - LOG_DEBUG("ObProxyProOp::get_limit_result remove rows by limit end", K(ret), K(start), K(offset), - K(rows->count()), K(*rows)); + result = res; } - return ret; } diff --git a/src/obproxy/engine/ob_proxy_operator_projection.h b/src/obproxy/engine/ob_proxy_operator_projection.h index edd6d3f2acf7068d11cafa4de0c9e808c3889b2c..60020ece57e7540d8eef0852e63c444dae00e568 100644 --- a/src/obproxy/engine/ob_proxy_operator_projection.h +++ b/src/obproxy/engine/ob_proxy_operator_projection.h @@ -24,26 +24,38 @@ class ObProxyProOp : public ObProxyOperator { public: ObProxyProOp(ObProxyOpInput *input, common::ObIAllocator &allocator) - : ObProxyOperator(input, allocator) { + : ObProxyOperator(input, allocator), current_rows_(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_) { set_op_type(PHY_PROJECTION); } - ~ObProxyProOp() {}; - virtual int get_limit_result(int64_t start, int64_t offset, ResultRows *rows); + ~ObProxyProOp() {} virtual int get_next_row(); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); -protected: - /* To temp preserve bulk records */ +private: + ResultRows current_rows_; }; class ObProxyProInput : public ObProxyOpInput { public: - ObProxyProInput() : ObProxyOpInput() {} - ObProxyProInput(const common::ObSEArray &select_exprs) - : ObProxyOpInput(select_exprs) {} + ObProxyProInput() : ObProxyOpInput(), derived_column_count_(0), + calc_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} ~ObProxyProInput() {} - // set method use ObProxyOpInput::set_xx_xx(..) + + void set_derived_column_count(int64_t derived_column_count) { + derived_column_count_ = derived_column_count; + } + + int64_t get_derived_column_count() { return derived_column_count_; } + + int set_calc_exprs(common::ObIArray &calc_exprs) { + return calc_exprs_.assign(calc_exprs); + } + common::ObIArray &get_calc_exprs() { return calc_exprs_; } + +private: + int64_t derived_column_count_; + common::ObSEArray calc_exprs_; }; } diff --git a/src/obproxy/engine/ob_proxy_operator_result.h b/src/obproxy/engine/ob_proxy_operator_result.h index 1da4cf7c01f560c6c1a604dfa02c21b32d831a56..575d7027becd5179f65700deeeb88e77d4c1bea2 100644 --- a/src/obproxy/engine/ob_proxy_operator_result.h +++ b/src/obproxy/engine/ob_proxy_operator_result.h @@ -14,6 +14,7 @@ #define OBPROXY_OB_PROXY_RESULT_RESP_H #include "common/ob_row.h" //#include "common/ob_field.h" +#include "common/obsm_utils.h" #include "rpc/obmysql/ob_mysql_field.h" #include "proxy/mysqllib/ob_resultset_fetcher.h" #include "lib/container/ob_se_array.h" @@ -26,7 +27,7 @@ namespace obproxy{ namespace engine{ -int64_t array_new_alloc_size = common::OB_MALLOC_NORMAL_BLOCK_SIZE; +const int64_t ENGINE_ARRAY_NEW_ALLOC_SIZE = 1; typedef common::ObSEArray ResultRow; typedef common::ObSEArray ResultRows; @@ -72,9 +73,9 @@ int change_sql_field(const ObMysqlField *src_field, obmysql::ObMySQLField *&dst_ MEMCPY(buf, src_field->table_.ptr(), src_field->table_.length()); dst_field->tname_.assign_ptr(buf, src_field->table_.length()); - buf = static_cast(allocator.alloc(src_field->table_.length())); - MEMCPY(buf, src_field->table_.ptr(), src_field->table_.length()); - dst_field->org_tname_.assign_ptr(src_field->table_.ptr(), src_field->table_.length()); + buf = static_cast(allocator.alloc(src_field->org_table_.length())); + MEMCPY(buf, src_field->org_table_.ptr(), src_field->org_table_.length()); + dst_field->org_tname_.assign_ptr(buf, src_field->org_table_.length()); buf = static_cast(allocator.alloc(src_field->name_.length())); @@ -83,9 +84,37 @@ int change_sql_field(const ObMysqlField *src_field, obmysql::ObMySQLField *&dst_ buf = static_cast(allocator.alloc(src_field->org_name_.length())); MEMCPY(buf, src_field->org_name_.ptr(), src_field->org_name_.length()); - dst_field->org_cname_.assign_ptr(buf, src_field->org_name_.length()); - dst_field->accuracy_.set_accuracy(static_cast(src_field->decimals_)); + + if (obmysql::OB_MYSQL_TYPE_FLOAT == src_field->type_ + || obmysql::OB_MYSQL_TYPE_DOUBLE == src_field->type_) { + if (0x1f == src_field->decimals_) { + ObObjType ob_type; + if (OB_SUCCESS != ObSMUtils::get_ob_type(ob_type, src_field->type_)) { + ob_type = ObDoubleType; + } + dst_field->accuracy_ = ObAccuracy::DML_DEFAULT_ACCURACY[ob_type]; + } else { + dst_field->accuracy_.set_scale(static_cast(src_field->decimals_)); + } + } else if(obmysql::OB_MYSQL_TYPE_NEWDECIMAL == src_field->type_ + || obmysql::OB_MYSQL_TYPE_DECIMAL == src_field->type_ + || obmysql::OB_MYSQL_TYPE_TIMESTAMP == src_field->type_ + || obmysql::OB_MYSQL_TYPE_DATETIME == src_field->type_ + || obmysql::OB_MYSQL_TYPE_TIME == src_field->type_) { + if (src_field->decimals_ > number::ObNumber::MAX_SCALE) { + ObObjType ob_type; + if (OB_SUCCESS != ObSMUtils::get_ob_type(ob_type, src_field->type_)) { + ob_type = ObNumberType; + } + dst_field->accuracy_ = ObAccuracy::DML_DEFAULT_ACCURACY[ob_type]; + } else { + dst_field->accuracy_.set_scale(static_cast(src_field->decimals_)); + } + } else { + dst_field->accuracy_.set_accuracy(static_cast(src_field->decimals_)); + } + dst_field->type_ = src_field->type_; dst_field->flags_ = static_cast(src_field->flags_); dst_field->set_charset_number(static_cast(src_field->charsetnr_)); diff --git a/src/obproxy/engine/ob_proxy_operator_sort.cpp b/src/obproxy/engine/ob_proxy_operator_sort.cpp index 69114adc8eed536e7c0e328f539f1826abe4a8c4..8f3122258877f24595b7f3f89f563f25391eb52f 100644 --- a/src/obproxy/engine/ob_proxy_operator_sort.cpp +++ b/src/obproxy/engine/ob_proxy_operator_sort.cpp @@ -15,6 +15,7 @@ #include "lib/oblog/ob_log_module.h" #include "common/ob_obj_compare.h" #include "ob_proxy_operator_sort.h" +#include "lib/container/ob_se_array_iterator.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::event; @@ -45,7 +46,7 @@ int ObProxySortOp::init() if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(sizeof(SortColumnArray)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("init error for no memory to alloc", K(ret), K(sizeof(SortColumnArray))); - } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(sort_columns_ = new (tmp_buf) SortColumnArray(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init error for no memory to construct", K(ret), K(sort_columns_)); } @@ -64,7 +65,7 @@ int ObProxySortOp::init_sort_columns() ret = common::OB_INVALID_ARGUMENT; LOG_WARN("input is invalid in ObProxySortOp", K(ret), K(input_)); } else { - const ObSEArray& order_by_expr = input->get_order_by_expr(); + const ObSEArray &order_by_expr = input->get_order_exprs(); ObProxyOrderItem* expr_ptr = NULL; void *tmp_ptr = NULL; @@ -72,7 +73,7 @@ int ObProxySortOp::init_sort_columns() LOG_DEBUG("ObProxySortOp::init_sort_columns", K(order_by_count), K(order_by_expr)); for (int64_t i = 0; OB_SUCC(ret) && i < order_by_count; i++) { - if (OB_ISNULL(expr_ptr = dynamic_cast(order_by_expr.at(i)))) { + if (OB_ISNULL(expr_ptr = order_by_expr.at(i))) { ret = common::OB_ERROR; LOG_WARN("internal error in ObProxySortOp::init_sort_columns", K(ret), K(expr_ptr)); } else { @@ -182,7 +183,7 @@ int ObProxyMemSortOp::get_next_row() if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); - } else if (OB_ISNULL(rows = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init ResultRows failed", K(ret), K(op_name()), K(rows)); } else if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObMemorySort)))) { @@ -201,7 +202,7 @@ int ObProxyMemSortOp::get_next_row() return ret; } -int ObProxyMemSortOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyMemSortOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; LOG_DEBUG("Enter ObProxyMemSortOp::handle_response_result", K(op_name()), K(data)); @@ -280,7 +281,7 @@ int ObProxyTopKOp::get_next_row() if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); - } else if (OB_ISNULL(rows = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_))) { + } else if (OB_ISNULL(rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; LOG_WARN("init ResultRows failed", K(ret), K(op_name()), K(rows)); } else if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObTopKSort)))) { @@ -295,8 +296,7 @@ int ObProxyTopKOp::get_next_row() return ObProxySortOp::get_next_row(); } -//int ObProxyTopKOp::handle_response_result(executor::ObProxyParallelResp *pres, bool is_final, ObProxyResultResp *&result) -int ObProxyTopKOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyTopKOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; LOG_DEBUG("Enter ObProxyMemSortOp::handle_response_result", K(op_name()), K(data)); @@ -362,6 +362,316 @@ int ObProxyTopKOp::handle_response_result(void *data, bool is_final, ObProxyResu return ret; } +ObProxyMemMergeSortOp::~ObProxyMemMergeSortOp() +{ + for (int64_t i = 0; i < sort_units_.count(); i++) { + ObProxyMemMergeSortUnit *sort_unit = sort_units_.at(i); + sort_unit->~ObProxyMemMergeSortUnit(); + allocator_.free(sort_unit); + } +} + +int ObProxyMemMergeSortOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) +{ + int ret = OB_SUCCESS; + + ObProxyResultResp *opres = NULL; + ObProxySortInput *input = NULL; + + if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input, data is NULL", K(ret)); + } else if (OB_ISNULL(opres = reinterpret_cast(data))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input, opres type is not match", K(ret)); + } else if (!opres->is_resultset_resp()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("resp is not resultset", K(opres), K(ret)); + } else if (OB_ISNULL(input = dynamic_cast(get_input()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input is invalid", K(ret)); + } else { + if (OB_ISNULL(get_result_fields())) { + result_fields_ = opres->get_fields(); + } + + ResultRow *row = NULL; + while (OB_SUCC(ret) && OB_SUCC(opres->next(row))) { + void *tmp_buf = NULL; + ObProxyMemMergeSortUnit *sort_unit = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObProxyMemMergeSortUnit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ObProxyMemMergeSortUnit), K(ret)); + } else if (OB_ISNULL(sort_unit = new (tmp_buf) ObProxyMemMergeSortUnit(allocator_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to new merge sort unit", K(ret)); + } else if (OB_FAIL(sort_unit->init(row, input->get_order_exprs()))) { + LOG_WARN("fail to init sort unit", K(ret)); + } else if (OB_FAIL(sort_units_.push_back(sort_unit))) { + LOG_WARN("fail to push back sort unit", K(ret)); + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + } + + if (OB_SUCC(ret) && is_final) { + int64_t limit_offset = get_input()->get_limit_offset(); + int64_t limit_offset_size = get_input()->get_limit_size() + limit_offset; + ObProxyResultResp *res = NULL; + ResultRows *rows = NULL; + void *tmp_buf = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ResultRows), K(ret)); + } else { + rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); + std::sort(sort_units_.begin(), sort_units_.end(), ObProxySortUnitCompare()); + } + + int64_t count = sort_units_.count(); + if (limit_offset_size > 0) { + count = count > limit_offset_size ? limit_offset_size : count; + } + + for (int64_t i = limit_offset; OB_SUCC(ret) && i < count; i++) { + ObProxyMemMergeSortUnit *&sort_unit = sort_units_.at(i); + + ResultRow *row = sort_unit->get_row(); + if (OB_FAIL(rows->push_back(row))) { + LOG_WARN("fail to push back row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { + LOG_WARN("fail to packet resultset", K(op_name()), K(ret)); + } + } + + if (OB_FAIL(ret) && OB_NOT_NULL(res)) { + res->set_packet_flag(PCK_ERR_RESPONSE); + } + result = res; + } + + return ret; +} + +int ObProxyMemMergeSortUnit::init(ResultRow *row, ObIArray &order_exprs) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(order_exprs_.assign(order_exprs))) { + LOG_WARN("fail to assign order exprs", K(ret)); + } else { + row_ = row; + if (OB_FAIL(calc_order_values())) { + LOG_WARN("fail to get order value", K(ret)); + } + } + + return ret; +} + +int ObProxyMemMergeSortUnit::calc_order_values() +{ + int ret = OB_SUCCESS; + + ObProxyExprCtx ctx(0, dbconfig::TESTLOAD_NON, false, &allocator_); + ObProxyExprCalcItem calc_item(row_); + order_values_.reuse(); + + for (int64_t i = 0; OB_SUCC(ret) && i < order_exprs_.count(); i++) { + if (OB_FAIL(order_exprs_.at(i)->calc(ctx, calc_item, order_values_))) { + LOG_WARN("fail to calc order exprs", K(ret)); + } + } + + return ret; +} + +// Return true to come first +bool ObProxyMemMergeSortUnit::compare(const ObProxyMemMergeSortUnit* sort_unit) const +{ + bool bret = true; + const ObIArray &order_values = sort_unit->get_order_values(); + int i = 0; + int64_t count_this = order_values_.count(); + int64_t count_other = order_values.count(); + int64_t count = count_this <= count_other ? count_this : count_other; + + for (; i < count; i++) { + int cmp = order_values_.at(i).compare(order_values.at(i)); + ObProxyOrderItem *order_expr = order_exprs_.at(i); + if (0 != cmp) { + if (order_expr->order_direction_ == NULLS_FIRST_ASC) { + bret = cmp < 0; + } else { + bret = cmp > 0; + } + break; + } + } + + if (i == count) { + bret = count_this <= count_other; + } + + return bret; +} + +ObProxyStreamSortOp::~ObProxyStreamSortOp() +{ + for (int64_t i = 0; i < sort_units_.count(); i++) { + ObProxyStreamSortUnit *sort_unit = sort_units_.at(i); + sort_unit->~ObProxyStreamSortUnit(); + allocator_.free(sort_unit); + } +} + +int ObProxyStreamSortOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) +{ + int ret = OB_SUCCESS; + + ObProxyResultResp *opres = NULL; + ObProxySortInput *input = NULL; + + if (OB_ISNULL(data)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input, data is NULL", K(ret)); + } else if (OB_ISNULL(opres = reinterpret_cast(data))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input, opres type is not match", K(ret)); + } else if (!opres->is_resultset_resp()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("resp is not resultset", K(opres), K(ret)); + } else if (OB_ISNULL(input = dynamic_cast(get_input()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("input is invalid", K(ret)); + } else { + if (OB_ISNULL(get_result_fields())) { + result_fields_ = opres->get_fields(); + } + + ResultRows &result_rows = opres->get_result_rows(); + if (!result_rows.empty()) { + void *tmp_buf = NULL; + ObProxyStreamSortUnit *sort_unit = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ObProxyStreamSortUnit)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ObProxyStreamSortUnit), K(ret)); + } else if (OB_ISNULL(sort_unit = new (tmp_buf) ObProxyStreamSortUnit(allocator_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to new merge sort unit", K(ret)); + } else if (OB_FAIL(sort_unit->init(opres, input->get_order_exprs()))) { + LOG_WARN("fail to init sort unit", K(ret)); + } else if (OB_FAIL(sort_units_.push_back(sort_unit))) { + LOG_WARN("fail to push back sort unit", K(ret)); + } + } + } + + if (OB_SUCC(ret) && is_final) { + int64_t limit_offset = input->get_limit_offset(); + int64_t limit_size = input->get_limit_size(); + ObProxyResultResp *res = NULL; + ResultRows *rows = NULL; + void *tmp_buf = NULL; + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", "size", sizeof(ResultRows), K(ret)); + } else { + rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); + } + + while (OB_SUCC(ret) && !sort_units_.empty()) { + // sort + // Flashback according to expectations, such as positive order, + // put the minimum value at the end, which is convenient for pop_back + std::sort(sort_units_.begin(), sort_units_.end(), ObProxySortUnitCompare()); + ObProxyStreamSortUnit *sort_unit = sort_units_.at(sort_units_.count() - 1); + + ResultRow *row = sort_unit->get_row(); + + if (limit_offset > 0) { + limit_offset--; + } else { + if (OB_FAIL(rows->push_back(row))) { + LOG_WARN("failed to push back row", K(ret)); + } else if (limit_size > 0 && rows->count() == limit_size) { + break; + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(sort_unit->next())) { + if (OB_ITER_END == ret) { + if (OB_FAIL(sort_units_.pop_back(sort_unit))) { + LOG_WARN("fail to pop back sort unit", K(ret)); + } else { + sort_unit->~ObProxyStreamSortUnit(); + allocator_.free(sort_unit); + } + } else { + LOG_WARN("fail to exec next", K(ret)); + } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { + LOG_WARN("fail to packet resultset", K(op_name()), K(ret)); + } + } + + if (OB_FAIL(ret) && OB_NOT_NULL(res)) { + res->set_packet_flag(PCK_ERR_RESPONSE); + } + + result = res; + } + + return ret; +} + +int ObProxyStreamSortUnit::init(ObProxyResultResp* result_set, common::ObIArray &order_exprs) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(order_exprs_.assign((order_exprs)))) { + LOG_WARN("fail to assign order expr", K(ret)); + } else if (OB_ISNULL(result_set_ = result_set)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result set should not NULL", K(ret)); + } else if (OB_FAIL(next())) { + // The first time, there must be a result, so it must not be OB_ITER_END + LOG_WARN("fail to exec next", K(ret)); + } + return ret; +} + +bool ObProxyStreamSortUnit::compare(const ObProxyStreamSortUnit* sort_unit) const +{ + bool bret = ObProxyMemMergeSortUnit::compare(sort_unit); + return !bret; +} + +int ObProxyStreamSortUnit::next() +{ + int ret = OB_SUCCESS; + + if (OB_SUCC(result_set_->next(row_))) { + if (OB_FAIL(calc_order_values())) { + LOG_WARN("fail to get order value", K(ret)); + } + } + + return ret; +} + ObBaseSort::ObBaseSort(SortColumnArray &sort_columns, common::ObIAllocator &allocator, ResultRows &sort_rows) : allocator_(allocator), sort_columns_(sort_columns), sort_rows_(sort_rows), topn_cnt_(0), row_count_(0), diff --git a/src/obproxy/engine/ob_proxy_operator_sort.h b/src/obproxy/engine/ob_proxy_operator_sort.h index a358bbfeba7d07374a017d6ab01ef2113c48a8cc..033c73d4e52db4938b984881e9d8946f9b24b5a9 100644 --- a/src/obproxy/engine/ob_proxy_operator_sort.h +++ b/src/obproxy/engine/ob_proxy_operator_sort.h @@ -24,6 +24,8 @@ namespace engine { class ObSortColumn; class ObBaseSort; class ObRowTrunk; +class ObProxyStreamSortUnit; +class ObProxyMemMergeSortUnit; typedef common::ObSEArray SortColumnArray; class ObProxySortOp : public ObProxyOperator { @@ -67,23 +69,17 @@ class ObProxySortInput : public ObProxyOpInput public: ObProxySortInput() : ObProxyOpInput(), - order_by_expr_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size) {} - ObProxySortInput(const common::ObSEArray &select_exprs, - const common::ObSEArray &order_by_expr) - : ObProxyOpInput(select_exprs), order_by_expr_(order_by_expr) - {} + order_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} - void set_order_by_expr(const common::ObSEArray &select_exprs) - { - select_exprs_ = select_exprs; + int set_order_by_exprs(const common::ObIArray &order_exprs) { + return order_exprs_.assign(order_exprs); } - common::ObSEArray& get_order_by_expr() - { - return order_by_expr_; + common::ObSEArray &get_order_exprs() { + return order_exprs_; } protected: - common::ObSEArray order_by_expr_; + common::ObSEArray order_exprs_; }; class ObProxyMemSortOp : public ObProxySortOp @@ -97,7 +93,7 @@ public: ~ObProxyMemSortOp() {}; virtual int get_next_row(); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); }; @@ -111,7 +107,93 @@ public: ~ObProxyTopKOp() {}; virtual int get_next_row(); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); +}; + +class ObProxyMemMergeSortOp : public ObProxySortOp +{ +public: + + ObProxyMemMergeSortOp(ObProxyOpInput *input, common::ObIAllocator &allocator) + : ObProxySortOp(input, allocator), sort_units_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) { + set_op_type(PHY_MEM_MERGE_SORT); + } + + ~ObProxyMemMergeSortOp(); + + virtual int init() { return ObProxyOperator::init(); } + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); + +private: + common::ObSEArray sort_units_; +}; + +template +class ObProxySortUnitCompare +{ +public: + bool operator()(const T *l, const T *r) const + { + return l->compare(r); + } +}; + +class ObProxyMemMergeSortUnit +{ +public: + ObProxyMemMergeSortUnit(common::ObIAllocator &allocator) + : allocator_(allocator), row_(NULL), + order_values_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + order_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} + ~ObProxyMemMergeSortUnit() {} + + int init(ResultRow *row, common::ObIArray &order_exprs); + virtual bool compare(const ObProxyMemMergeSortUnit* sort_unit) const; + int calc_order_values(); + + ResultRow* get_row() { return row_; } + const common::ObIArray &get_order_values() const { return order_values_; } + + TO_STRING_KV(K(order_values_), K(order_exprs_)); + +protected: + common::ObIAllocator &allocator_; + ResultRow *row_; + common::ObSEArray order_values_; + common::ObSEArray order_exprs_; +}; + +class ObProxyStreamSortOp : public ObProxySortOp +{ +public: + + ObProxyStreamSortOp(ObProxyOpInput *input, common::ObIAllocator &allocator) + : ObProxySortOp(input, allocator), sort_units_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) { + set_op_type(PHY_STREAM_SORT); + } + + ~ObProxyStreamSortOp(); + + virtual int init() { return ObProxyOperator::init(); } + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); + +private: + common::ObSEArray sort_units_; +}; + +class ObProxyStreamSortUnit : public ObProxyMemMergeSortUnit +{ +public: + ObProxyStreamSortUnit(common::ObIAllocator &allocator) + : ObProxyMemMergeSortUnit(allocator), result_set_(NULL) {} + ~ObProxyStreamSortUnit() {} + + int init(ObProxyResultResp* result_set, common::ObIArray &order_exprs); + virtual bool compare(const ObProxyStreamSortUnit* sort_unit) const; + int next(); + +private: + ObProxyResultResp* result_set_; }; class ObSortColumn : public common::ObColumnInfo diff --git a/src/obproxy/engine/ob_proxy_operator_table_scan.cpp b/src/obproxy/engine/ob_proxy_operator_table_scan.cpp index f8345ce48917c6c9ca771035e5dfa726d53ddc89..f9c16c5ab5bcfd420973a1dd6861620e0457062f 100644 --- a/src/obproxy/engine/ob_proxy_operator_table_scan.cpp +++ b/src/obproxy/engine/ob_proxy_operator_table_scan.cpp @@ -19,8 +19,10 @@ #include "iocore/eventsystem/ob_ethread.h" #include "executor/ob_proxy_parallel_cont.h" #include "executor/ob_proxy_parallel_execute_cont.h" +#include "proxy/shard/obproxy_shard_utils.h" using namespace oceanbase::obproxy::executor; +using namespace oceanbase::common; namespace oceanbase { namespace obproxy { @@ -44,301 +46,85 @@ ObProxyOperator* ObProxyTableScanOp::get_child(const uint32_t idx) return NULL; } -int ObProxyTableScanOp::rewrite_hint_table(const common::ObString &hint_string, - common::ObSqlString &obj_hint_string, - const common::ObString &table_name, const common::ObString &database_name, - const common::ObString &real_table_name, const common::ObString &real_database_name, - bool is_oracle_mode) { - char* hint_buf = NULL; - char* tmp_buf = NULL; - int32_t db_table_len = table_name.length() > database_name.length() - ? table_name.length() : database_name.length(); - int32_t hint_len = hint_string.length(); - db_table_len++; - int ret = common::OB_SUCCESS; - if (OB_ISNULL(hint_string.ptr())) { - //ret = common::OB_INVALID_ARGUMENT; - LOG_DEBUG("not have any hint_stirng, not need to rewrite_hint_table."); - //LOG_WARN("invalid input", K(hint_string)); - } else if (OB_ISNULL(hint_buf - = reinterpret_cast(allocator_.alloc(hint_len + 1))) - ||(OB_ISNULL(tmp_buf - = reinterpret_cast(allocator_.alloc(db_table_len))))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory alloc error", K(ret), K(hint_string)); - } else { - char* table_pos = NULL; - char* db_pos = NULL; - memcpy(hint_buf, hint_string.ptr(), hint_len); - hint_buf[hint_string.length()] = '\0'; - string_to_upper_case(hint_buf, hint_len); - memcpy(tmp_buf, table_name.ptr(), table_name.length()); - tmp_buf[table_name.length()] = '\0'; - table_pos = strstr(hint_buf, tmp_buf); - if (OB_NOT_NULL(table_pos)) { - LOG_DEBUG("found the table_name in hint_string", K(hint_string), K(table_name)); - int64_t pos = table_pos - hint_buf; - // has db_name.`table_name` or db_name.table_name - if (pos > 2 && (hint_buf[pos-1] == '.' || hint_buf[pos-2] == '.')) { - memcpy(tmp_buf, database_name.ptr(), database_name.length()); - tmp_buf[database_name.length()] = '\0'; - db_pos = strstr(hint_buf, tmp_buf); - } - if (OB_NOT_NULL(db_pos)) { - obj_hint_string.append(hint_string.ptr(), db_pos - hint_buf); - }else { - obj_hint_string.append(hint_string.ptr(), *(table_pos - 1) == '"' ? table_pos - 1 - hint_buf - : table_pos - hint_buf); - } - if (is_oracle_mode) { - obj_hint_string.append("\"", 1); - obj_hint_string.append(real_database_name.ptr(), real_database_name.length()); - obj_hint_string.append("\"", 1); - } else { - obj_hint_string.append(real_database_name.ptr(), real_database_name.length()); - } - - obj_hint_string.append(".", 1); - if (is_oracle_mode) { - obj_hint_string.append("\"", 1); - obj_hint_string.append(real_table_name.ptr(), real_table_name.length()); - obj_hint_string.append("\"", 1); - } else { - obj_hint_string.append(real_table_name.ptr(), real_table_name.length()); - } - if (*(table_pos + table_name.length()) == '\"') { - obj_hint_string.append(hint_string.ptr() + (table_pos + 1 - hint_buf), - hint_string.length() - (table_pos + 1 - hint_buf)); - } else { - obj_hint_string.append(hint_string.ptr() + (table_pos - hint_buf), - hint_string.length() - (table_pos - hint_buf)); - } - } - } - return ret; -} - -int ObProxyTableScanOp::format_sql_header(ObSqlString &obj_sql_head) -{ - int ret = common::OB_SUCCESS; - ObProxyTableScanInput* input = NULL; - - OB_ASSERT(OB_NOT_NULL(ObProxyOperator::get_input())); - - if (OB_ISNULL(input - = dynamic_cast(ObProxyOperator::get_input()))) { - ret = common::OB_INVALID_ARGUMENT; - LOG_WARN("invalid input for ObProxyTableScanOp"); - } - - if (OB_SUCC(ret)) { - LOG_DEBUG("SELECT EXPR count", K(ret), K(input->get_select_exprs().count())); - for (int64_t i = 0; i < input->get_select_exprs().count(); i++) { - if (OB_ISNULL(input->get_select_exprs()[i]) - || OB_FAIL((int)(input->get_select_exprs()[i]->to_sql_string(obj_sql_head)))) { - LOG_WARN("inner error when select exprs to sql_string", K(ret), - K(input->get_select_exprs()[i])); - } - if (i + 1 != input->get_select_exprs().count()) { - obj_sql_head.append(", "); - } - } - - obj_sql_head.append(" FROM "); - } - return ret; -} - -int ObProxyTableScanOp::format_sql_tailer(ObSqlString &obj_sql_tail, bool is_oracle_mode) -{ - int ret = common::OB_SUCCESS; - ObProxyTableScanInput* input = NULL; - bool has_group_by = false; - - OB_ASSERT(OB_NOT_NULL(ObProxyOperator::get_input())); - - if (OB_ISNULL(input - = dynamic_cast(ObProxyOperator::get_input()))) { - ret = common::OB_INVALID_ARGUMENT; - LOG_WARN("invalid input for ObProxyTableScanOp"); - } - - if(OB_SUCC(ret)) { - if (OB_SUCC(ret) && input->get_condition_exprs().count() > 0) { - obj_sql_tail.append(" WHERE "); - } - for (int64_t i = 0; OB_SUCC(ret) && i < input->get_condition_exprs().count(); i++) { - if (OB_ISNULL(input->get_condition_exprs()[i]) - || OB_FAIL((int)(input->get_condition_exprs()[i]->to_sql_string(obj_sql_tail)))) { - LOG_WARN("inner error when condition exprs to sql_string", - K(input->get_condition_exprs()[i])); - break; - } - } - - /* GROUP BY col_a, col_b ORDER BY col_c, col_d */ - if (OB_SUCC(ret) && input->get_group_by_exprs().count() > 0) { - obj_sql_tail.append(" GROUP BY "); - has_group_by = true; - for (int64_t i = 0; OB_SUCC(ret) && i < input->get_group_by_exprs().count(); i++) { - if (OB_ISNULL(input->get_group_by_exprs()[i]) - || OB_FAIL((int)(input->get_group_by_exprs()[i]->to_sql_string(obj_sql_tail)))) { - LOG_WARN("inner error when groupby exprs to sql_string", - K(input->get_group_by_exprs()[i])); - } - if (i + 1 != input->get_group_by_exprs().count()) { - obj_sql_tail.append(", "); - } - } - if (!is_oracle_mode) { - /* Add order by for GROUP BY columns for ObServer not sort by group by columns. */ - obj_sql_tail.append(" ORDER BY "); - for (int64_t i = 0; OB_SUCC(ret) && i < input->get_group_by_exprs().count(); i++) { - if (OB_ISNULL(input->get_group_by_exprs()[i]) - || OB_FAIL((int)(input->get_group_by_exprs()[i]->to_sql_string(obj_sql_tail)))) { - LOG_WARN("inner error when groupby exprs to sql_string", - K(input->get_group_by_exprs()[i])); - } - if (i + 1 != input->get_group_by_exprs().count()) { - obj_sql_tail.append(", "); - } - } - } - } else if (OB_SUCC(ret) && input->get_order_by_exprs().count() > 0) { - obj_sql_tail.append(" ORDER BY "); - for (int64_t i = 0; OB_SUCC(ret) && i < input->get_order_by_exprs().count(); i++) { - if (OB_ISNULL(input->get_order_by_exprs()[i]) - || OB_FAIL((int)(input->get_order_by_exprs()[i]->to_sql_string(obj_sql_tail)))) { - LOG_WARN("inner error when order by exprs to sql_string", K(ret), - K(input->get_order_by_exprs()[i])); - } - if (i + 1 != input->get_order_by_exprs().count()) { - obj_sql_tail.append(", "); - } - LOG_DEBUG("ORDER BY add expr", K(ret), K(obj_sql_tail)); - } - } - - int64_t value = -1; - if (OB_SUCC(ret) && !has_group_by && -1 != (value = input->get_op_top_value())) { - obj_sql_tail.append_fmt(" LIMIT %ld", value); - int64_t start = input->get_op_limit_value(); - int64_t offset = input->get_op_limit_value(); - LOG_DEBUG("add LIMIT base on limit ", K(start), K(offset), K(value)); - } - - } - - return ret; -} - int ObProxyTableScanOp::get_next_row() { - int ret = common::OB_SUCCESS; + int ret = OB_SUCCESS; ObProxyTableScanInput* input = NULL; + common::ObSEArray parallel_param; if (OB_ISNULL(ObProxyOperator::get_input())) { - ret = common::OB_INVALID_ARGUMENT; + ret = OB_INVALID_ARGUMENT; LOG_WARN("not have any input for table_scan", K(ret), KP(ObProxyOperator::get_input())); - } else if (OB_ISNULL(input - = dynamic_cast(ObProxyOperator::get_input()))) { - ret = common::OB_INVALID_ARGUMENT; + } else if (OB_ISNULL(input = dynamic_cast(ObProxyOperator::get_input()))) { + ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid input for ObProxyTableScanOp", K(ret)); } else if (OB_ISNULL(operator_async_task_)) { - ret = common::OB_INVALID_ARGUMENT; + ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid input for ObProxyTableScanOp", K(ret)); - } - - LOG_DEBUG("input limit value:", K(input->get_op_limit_value()), K(input->get_op_offset_value())); - - if (OB_SUCC(ret)) { - /* init object sql for select */ - ObSqlString obj_sql_head; - ObSqlString obj_sql_tail; - - if (OB_FAIL(format_sql_header(obj_sql_head))) { - LOG_WARN("inner error to format sql base on exprs"); - } else if (input->get_phy_db_table_names().count() - != input->get_db_key_names().count()) { - ret = common::OB_ERROR; - LOG_WARN("inner error for sharding info", K(input->get_phy_db_table_names().count()), - K(input->get_db_key_names().count())); - } else { - common::ObSEArray parallel_param; - //void *tmp_buf = NULL; + } else { + ObIArray > &table_name_maps = input->get_table_name_maps(); + ObIArray &db_key_names = input->get_db_key_names(); + ObIArray &shard_props = input->get_shard_props(); + ObString &request_sql = input->get_request_sql(); + obutils::ObProxySqlParser sql_parser; + obutils::ObSqlParseResult parse_result; + + if (table_name_maps.count() != db_key_names.count() + || shard_props.count() != db_key_names.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner error for sharding info", + "phy table count", table_name_maps.count(), + "shard prop count", shard_props.count(), + "dbkey count", db_key_names.count(), K(ret)); + } else if (OB_FAIL(sql_parser.parse_sql_by_obparser(request_sql, NORMAL_PARSE_MODE, parse_result, true))) { + LOG_WARN("parse_sql_by_obparser failed", K(request_sql), K(ret)); + } - int64_t count = input->get_db_key_names().count(); + for (int64_t i = 0; OB_SUCC(ret) && i < db_key_names.count(); i++) { + dbconfig::ObShardConnector *db_key_name = db_key_names.at(i); + dbconfig::ObShardProp *shard_prop = shard_props.at(i); + hash::ObHashMapWrapper &table_name_map_warraper = table_name_maps.at(i); + ObSqlString new_sql; char *tmp_buf = NULL; - ObSqlString obj_sql; // a temp ObSqlString object to use - for (int64_t i = 0; i < count; i++) { - bool is_oracle_mode = input->get_db_key_names().at(i)->server_type_ == common::DB_OB_ORACLE; - obj_sql.reset(); - obj_sql.append(input->get_normal_annotation_string()); - /* Rewrite and add hint string */ - obj_sql.append("SELECT "); - obj_sql.append(obj_sql_head.ptr()); - obj_sql_tail.reuse(); - if (OB_FAIL(format_sql_tailer(obj_sql_tail, is_oracle_mode))) { - LOG_WARN("inner error to format sql base on exprs"); - } else if (OB_FAIL(rewrite_hint_table(input->get_hint_string(), obj_sql, - //TODO get origin table_name and db_name - input->get_logical_table_name(), input->get_logical_database_name(), - input->get_phy_db_table_names().at(i), input->get_db_key_names().at(i)->database_name_, - is_oracle_mode))) { - LOG_WARN("internal error in rewrite_hint_table", K(ret), - K(input->get_hint_string())); - } else if (input->get_db_key_names().at(i)->database_name_.length() <= 0 - || input->get_phy_db_table_names().at(i).length() <= 0) { - ret = common::OB_INVALID_ARGUMENT; - LOG_WARN("invilid table name or db_name", K(ret), - K(input->get_db_key_names().at(i)->database_name_), - K(input->get_phy_db_table_names().at(i))); - } else { - obj_sql.append(input->get_db_key_names().at(i)->database_name_.ptr(), - input->get_db_key_names().at(i)->database_name_.length()); - obj_sql.append("."); - obj_sql.append(input->get_phy_db_table_names().at(i).ptr(), - input->get_phy_db_table_names().at(i).length()); - LOG_DEBUG("sub_sql table info", K(input->get_db_key_names().at(i)->database_name_.ptr()), - K(input->get_phy_db_table_names().at(i).ptr())); - obj_sql.append(obj_sql_tail.ptr(), obj_sql_tail.length()); - - if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(obj_sql.length() + 1))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(obj_sql.length() + 1)); - } - - MEMCPY(tmp_buf, obj_sql.ptr(), obj_sql.length()); - tmp_buf[obj_sql.length()] = '\0'; + bool is_oracle_mode = db_key_name->server_type_ == common::DB_OB_ORACLE; + if (OB_FAIL(proxy::ObProxyShardUtils::do_rewrite_shard_select_request(request_sql, parse_result, is_oracle_mode, + table_name_map_warraper.get_hash_map(), + db_key_name->database_name_, false, + new_sql))) { + LOG_WARN("fail to rewrite shard request", K(request_sql), K(is_oracle_mode), K(ret)); + } else if (OB_ISNULL(tmp_buf = (char *)allocator_.alloc(new_sql.length()))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", K(op_name()), "sql len", new_sql.length(), K(ret)); + } else { + MEMCPY(tmp_buf, new_sql.ptr(), new_sql.length()); + ObProxyParallelParam param; - ObProxyParallelParam param; - param.shard_conn_ = input->get_db_key_names().at(i); - //char* tmp_p = obj_sql.ptr(); - param.request_sql_.assign(tmp_buf, static_cast(obj_sql.length())); - LOG_DEBUG("sub_sql and len:", K(param.request_sql_), K(obj_sql.length())); - parallel_param.push_back(param); - } + param.shard_conn_ = db_key_name; + param.shard_prop_ = shard_prop; + param.request_sql_.assign(tmp_buf, static_cast(new_sql.length())); + parallel_param.push_back(param); } + } + } - if (OB_SUCC(ret)) { - for (int64_t i = 0; i < count; i++) { - LOG_DEBUG("sub_sql before send", K(i), K(¶llel_param.at(i).request_sql_), - K(parallel_param.at(i).request_sql_), K(parallel_param.at(i).shard_conn_->database_name_)); - } - if (OB_FAIL(get_global_parallel_processor().open(*operator_async_task_, operator_async_task_->parallel_action_array_[0], - parallel_param, &allocator_, timeout_ms_))) { - LOG_WARN("fail to op parallel processor", K(parallel_param)); - } else { - set_sub_sql_count(count); - } - } + if (OB_SUCC(ret)) { + for (int64_t i = 0; i < parallel_param.count(); i++) { + LOG_DEBUG("sub_sql before send", K(i), "sql", parallel_param.at(i).request_sql_, + "database", parallel_param.at(i).shard_conn_->database_name_); + } + if (OB_FAIL(get_global_parallel_processor().open(*operator_async_task_, operator_async_task_->parallel_action_array_[0], + parallel_param, &allocator_, timeout_ms_))) { + LOG_WARN("fail to op parallel processor", K(parallel_param)); + } else { + set_sub_sql_count(parallel_param.count()); } } + return ret; } -int ObProxyTableScanOp::handle_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyTableScanOp::handle_result(void *data, bool &is_final, ObProxyResultResp *&result) { int ret = OB_SUCCESS; @@ -377,7 +163,7 @@ int ObProxyTableScanOp::handle_result(void *data, bool is_final, ObProxyResultRe return ret; } -int ObProxyTableScanOp::handle_response_result(void *data, bool is_final, ObProxyResultResp *&result) +int ObProxyTableScanOp::handle_response_result(void *data, bool &is_final, ObProxyResultResp *&result) { UNUSED(is_final); int ret = OB_SUCCESS; @@ -387,8 +173,10 @@ int ObProxyTableScanOp::handle_response_result(void *data, bool is_final, ObProx executor::ObProxyParallelResp *pres = NULL; ObMysqlField *fields = NULL; //_array = NULL; - UNUSED(data); - UNUSED(is_final); + ResultRows *rows = NULL; + int64_t result_sum = 0; + ResultRow *row = NULL; + common::ObObj *row_ptr = NULL; if (OB_ISNULL(data)) { ret = common::OB_INVALID_ARGUMENT; @@ -399,149 +187,252 @@ int ObProxyTableScanOp::handle_response_result(void *data, bool is_final, ObProx } else if (!pres->is_resultset_resp()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObProxyTableScanOp::handle_response_result not response result", K(pres), KP(pres), K(pres->is_resultset_resp())); - } else { - LOG_DEBUG("ObProxyTableScanOp::process_ready_data:resultset_resp", K(pres), KP(pres)); + } else if (OB_UNLIKELY((columns_length = pres->get_column_count()) <= 0)) { + ret = common::OB_ERR_UNEXPECTED; + LOG_WARN("columns_length less than 0", K(columns_length), K(pres), K(ret)); + } else if (OB_ISNULL(fields = pres->get_field())) { + ret = common::OB_ERR_UNEXPECTED; + LOG_WARN("fields is null", K(pres), K(ret)); + } - common::ObObj *row_ptr = NULL; - ResultRows *rows = NULL; - if (OB_UNLIKELY((columns_length = pres->get_column_count()) <= 0)) { - ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("columns_length less than 0", K(columns_length), K(pres), K(ret)); - } else if (OB_ISNULL(fields = pres->get_field())) { + if (OB_SUCC(ret) && OB_ISNULL(result_fields_)) { + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultFields)))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultFields))); + } else if (OB_ISNULL(result_fields_ = new (tmp_buf) ResultFields(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_))) { ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("fields is null", K(pres), K(ret)); - } - - if (OB_SUCC(ret) && OB_ISNULL(result_fields_)) { - if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultFields)))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultFields))); - } else if (OB_ISNULL(result_fields_ = new (tmp_buf) ResultFields(array_new_alloc_size, allocator_))) { - ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultFields))); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < columns_length; i++) { - LOG_DEBUG("change field info", K(i), K(result_fields_)); - obmysql::ObMySQLField *obj_field = NULL; - if (OB_FAIL(change_sql_field(fields, obj_field, allocator_))) { - LOG_WARN("change field info failed", K(ret)); - } else { - result_fields_->push_back(*obj_field); - fields++; - } + LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultFields))); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < columns_length; i++) { + LOG_DEBUG("change field info", K(i), K(result_fields_)); + obmysql::ObMySQLField *obj_field = NULL; + if (OB_FAIL(change_sql_field(fields, obj_field, allocator_))) { + LOG_WARN("change field info failed", K(ret)); + } else { + result_fields_->push_back(*obj_field); + fields++; } } } if (OB_SUCC(ret)) { - if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); - } else { - rows = new (tmp_buf) ResultRows(array_new_alloc_size, allocator_); + if (OB_FAIL(set_index())) { + LOG_WARN("fail to set index", K(ret)); } } + } - int64_t result_sum = 0; - ResultRow *row = NULL; + if (OB_SUCC(ret)) { + if (OB_ISNULL(tmp_buf = allocator_.alloc(sizeof(ResultRows)))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(sizeof(ResultRows))); + } else { + rows = new (tmp_buf) ResultRows(ENGINE_ARRAY_NEW_ALLOC_SIZE, allocator_); + } + } - while(OB_SUCC(ret) && OB_SUCC(pres->next(row_ptr))) { - if (NULL == row_ptr) { - ret = common::OB_ERR_UNEXPECTED; - LOG_WARN("row prt is NULL", K(ret)); - } else if (OB_FAIL(init_row(row))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(row)); - } else { - result_sum++; + while(OB_SUCC(ret) && OB_SUCC(pres->next(row_ptr))) { + if (NULL == row_ptr) { + ret = common::OB_ERR_UNEXPECTED; + LOG_WARN("row prt is NULL", K(ret)); + } else if (OB_FAIL(init_row(row))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("no have enough memory to init", K(ret), K(op_name()), K(row)); + } else { + result_sum++; - row->reserve(columns_length); - for (int64_t i = 0; i < columns_length; i++) { - LOG_DEBUG("row object info--------------->", K(i), KPC(row_ptr), K(pres->get_cont_index())); - row->push_back(row_ptr); - row_ptr++; - } - rows->push_back(row); - LOG_DEBUG("process_ready_data: get one result from server", K(op_name()), K(row), KPC(row)); + row->reserve(columns_length); + for (int64_t i = 0; i < columns_length; i++) { + LOG_DEBUG("row object info--------------->", K(i), KPC(row_ptr), K(pres->get_cont_index())); + row->push_back(row_ptr); + row_ptr++; } + rows->push_back(row); + LOG_DEBUG("process_ready_data: get one result from server", K(op_name()), K(row), KPC(row)); } + } - if (common::OB_ITER_END == ret) { - ret = common::OB_SUCCESS; - } + if (common::OB_ITER_END == ret) { + ret = common::OB_SUCCESS; + } - LOG_DEBUG("ObProxyTableScanOp::process_ready_data get all rows", K(ret), K(result_sum), - K(pres->get_cont_index())); + LOG_DEBUG("ObProxyTableScanOp::process_ready_data get all rows", K(ret), K(result_sum), + K(pres->get_cont_index())); - ObProxyResultResp *res = NULL; - if (OB_SUCC(ret)) { - if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { - LOG_WARN("process_ready_data:failed to packet resultset", K(op_name()), K(ret)); - } else if (OB_ISNULL(res)) { - LOG_WARN("process_ready_data::packet_result_set success but res is NULL", K(ret), K(res)); - ret = common::OB_ERR_UNEXPECTED; - } else { - res->set_result_sum(get_sub_sql_count()); - res->set_result_idx(pres->get_cont_index()); - LOG_DEBUG("ObProxyTableScanOp::process_ready_data sub_sql_count", K(ret), K(res->get_result_sum()), K(res)); - } - result = res; + ObProxyResultResp *res = NULL; + if (OB_SUCC(ret)) { + if (OB_FAIL(packet_result_set(res, rows, get_result_fields()))) { + LOG_WARN("process_ready_data:failed to packet resultset", K(op_name()), K(ret)); + } else if (OB_ISNULL(res)) { + ret = common::OB_ERR_UNEXPECTED; + LOG_WARN("process_ready_data::packet_result_set success but res is NULL", K(ret), K(res)); + } else { + res->set_result_sum(get_sub_sql_count()); + res->set_result_idx(pres->get_cont_index()); + LOG_DEBUG("ObProxyTableScanOp::process_ready_data sub_sql_count", K(ret), K(res->get_result_sum()), K(res)); } + result = res; } - LOG_DEBUG("Exit ObProxyTableScanOp::handle_result", K(ret), K(pres)); + return ret; } -int ObProxyTableScanOp::process_ready_data(void *data, int &event) +int ObProxyTableScanOp::set_index() { int ret = OB_SUCCESS; - ObProxyResultResp *result = NULL; - executor::ObProxyParallelResp *res = NULL; - if (OB_ISNULL(data)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid param, data is null", "op_name", op_name(), K(ret)); - } else if (OB_ISNULL(res = reinterpret_cast(data))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid input, pres type is not match", K(ret), KP(data)); - } else if (OB_FAIL(handle_result(res, false, result))) { - LOG_WARN("fail to handle result", "op_name", op_name(), K(ret)); - } else if (OB_ISNULL(result)) { - event = VC_EVENT_CONT; - } else if (result->is_error_resp()) { - result_ = result; - event = VC_EVENT_READ_COMPLETE; - } else { - result_ = result; - event = VC_EVENT_READ_READY; + + ObProxyTableScanInput* input = dynamic_cast(get_input()); + if (OB_FAIL(set_index_for_exprs(input->get_calc_exprs()))) { + LOG_WARN("fail to set index for calc exprs", K(ret)); + } else if (OB_FAIL(set_index_for_exprs(input->get_agg_exprs()))) { + LOG_WARN("fail to set index for agg exprs", K(ret)); + } else if (OB_FAIL(set_index_for_exprs(input->get_group_exprs()))) { + LOG_WARN("fail to set index for group exprs", K(ret)); + } else if (OB_FAIL(set_index_for_exprs(input->get_order_exprs()))) { + LOG_WARN("fail to set index for order exprs", K(ret)); } - LOG_DEBUG("finish process_ready_data", K(event), K(ret)); + return ret; } -int ObProxyTableScanOp::process_complete_data(void *data) +template +int ObProxyTableScanOp::set_index_for_exprs(ObIArray &expr_array) { int ret = OB_SUCCESS; - ObProxyResultResp *result = NULL; - executor::ObProxyParallelResp *res = NULL; - if (OB_ISNULL(data)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid param, data is null", K(ret)); - } else if (OB_ISNULL(res = reinterpret_cast(data))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid input, pres type is not match", K(ret)); - } else if (OB_FAIL(handle_result(res, true, result))) { - LOG_WARN("fail to handle result", K(ret)); - } else if (OB_ISNULL(result)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("reulst is NULL, something error", K(ret)); - } else { - result_ = result; + + for (int64_t i = 0; OB_SUCC(ret) && i < expr_array.count(); i++) { + T *expr = expr_array.at(i); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected expr is null", K(ret)); + } else if (OB_FAIL(set_index_for_expr(expr))) { + LOG_WARN("fail to set index", K(ret)); + } + } + + return ret; +} + +int ObProxyTableScanOp::set_index_for_expr(ObProxyExpr *expr) +{ + int ret = OB_SUCCESS; + + ObProxyExprType expr_type = expr->get_expr_type(); + + if (OB_PROXY_EXPR_TYPE_FUNC_GROUP == expr_type || OB_PROXY_EXPR_TYPE_FUNC_ORDER == expr_type) { + ObProxyGroupItem *group_expr = dynamic_cast(expr); + if (OB_ISNULL(group_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else if (OB_ISNULL(expr = group_expr->get_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group expr or order expr do not have child expr", KPC(group_expr), K(ret)); + } else { + expr_type = expr->get_expr_type(); + } + } + + if (OB_SUCC(ret) && -1 == expr->get_index() + && (OB_PROXY_EXPR_TYPE_COLUMN == expr_type + || OB_PROXY_EXPR_TYPE_SHARDING_CONST == expr_type + || expr->is_func_expr())) { + int64_t i = 0; + for (; OB_SUCC(ret) && i < result_fields_->count(); ++i) { + obmysql::ObMySQLField &field = result_fields_->at(i); + ObString &alias_name = expr->get_alias_name(); + if (!alias_name.empty()) { + if (0 == alias_name.case_compare(field.cname_)) { + expr->set_index(i); + expr->set_accuracy(field.accuracy_); + break; + } + } else if (OB_PROXY_EXPR_TYPE_SHARDING_CONST == expr_type || expr->is_func_expr()) { + ObString &expr_name = expr->get_expr_name(); + if (0 == expr_name.case_compare(field.cname_)) { + expr->set_index(i); + expr->set_accuracy(field.accuracy_); + break; + } + } else if (OB_PROXY_EXPR_TYPE_COLUMN == expr_type) { + ObProxyExprColumn *expr_column = dynamic_cast(expr); + if (OB_ISNULL(expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else { + ObString &table_name = expr_column->get_table_name(); + ObString &column_name = expr_column->get_column_name(); + if ((table_name.empty() || field.tname_.prefix_match(table_name)) + && 0 == column_name.case_compare(field.cname_)) { + expr->set_index(i); + expr->set_accuracy(field.accuracy_); + break; + } + } + } + } + + if (i == result_fields_->count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column from field, mayby something error", K(expr), K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_PROXY_EXPR_TYPE_FUNC_AVG == expr_type) { + // Avg-dependent count and sum expressions also set index + ObProxyExprAvg *avg_expr = dynamic_cast(expr); + ObProxyExprSum *sum_expr = NULL; + ObProxyExprCount *count_expr = NULL; + if (OB_ISNULL(avg_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else if (OB_ISNULL(sum_expr = avg_expr->get_sum_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("avg expr don't have sum expr", K(avg_expr), K(ret)); + } else if (OB_ISNULL(count_expr = avg_expr->get_count_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("avg expr don't have count expr", K(avg_expr), K(ret)); + } else if (OB_FAIL(set_index_for_expr(sum_expr))) { + LOG_WARN("fail to set index for sum expr", K(ret)); + } else if (OB_FAIL(set_index_for_expr(count_expr))) { + LOG_WARN("fail to set index for count expr", K(ret)); + } + } else if (expr->has_agg() && !expr->is_agg()) { + // If the expression contains an aggregate, but it is not an aggregate function, + // set an index for its parameter, which is used to calculate + ObProxyFuncExpr *func_expr = dynamic_cast(expr); + if (OB_ISNULL(func_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else { + ObSEArray& param_array = func_expr->get_param_array(); + for (int64_t i = 0; OB_SUCC(ret) && i < param_array.count(); i++) { + ObProxyExpr* param_expr = param_array.at(i); + if (OB_FAIL(set_index_for_expr(param_expr))) { + LOG_WARN("fail to set index for expr", K(ret)); + } + } + } + } } - LOG_DEBUG("finish process_complete_data", K(ret)); return ret; } +ObProxyTableScanInput::~ObProxyTableScanInput() +{ + // free reference count + for (int64_t i = 0; i < db_key_names_.count(); i++) { + dbconfig::ObShardConnector *db_key_name = db_key_names_.at(i); + db_key_name->dec_ref(); + } + + for (int64_t i = 0; i < shard_props_.count(); i++) { + dbconfig::ObShardProp *shard_prop = shard_props_.at(i); + shard_prop->dec_ref(); + } +} + } } } diff --git a/src/obproxy/engine/ob_proxy_operator_table_scan.h b/src/obproxy/engine/ob_proxy_operator_table_scan.h index 5e06ccb2912c08ace1c0b63780eb27e420cae34d..1c14c926727edf36856e7f3bf90c390435ba99df 100644 --- a/src/obproxy/engine/ob_proxy_operator_table_scan.h +++ b/src/obproxy/engine/ob_proxy_operator_table_scan.h @@ -23,6 +23,7 @@ namespace oceanbase { namespace obproxy { namespace engine { + class ObProxyTableScanOp : public ObProxyOperator { public: @@ -38,21 +39,18 @@ public: virtual ObProxyOperator* get_child(const uint32_t idx);// { return NULL; } - int rewrite_hint_table(const common::ObString &hint_string, common::ObSqlString &obj_hint_string, - const common::ObString &table_name, const common::ObString &database_name, - const common::ObString &real_table_name, const common::ObString &real_database_name, - bool is_oracle_mode); - - virtual int format_sql_header(ObSqlString &sql_header); /* sql: SELECT ... FROM */ - virtual int format_sql_tailer(ObSqlString &sql_header, bool is_oracle_mode); /* sql: WHERE ... GROUP BY ORDER BY LIMIT */ - - virtual int handle_result(void *src, bool is_final, ObProxyResultResp *&result); - virtual int handle_response_result(void *src, bool is_final, ObProxyResultResp *&result); - virtual int process_ready_data(void *data, int &event); - virtual int process_complete_data(void *data); + virtual int handle_result(void *src, bool &is_final, ObProxyResultResp *&result); + virtual int handle_response_result(void *src, bool &is_final, ObProxyResultResp *&result); int64_t get_sub_sql_count() { return sub_sql_count_; } void set_sub_sql_count(int64_t count) { sub_sql_count_ = count; } + +private: + int set_index(); + template + int set_index_for_exprs(common::ObIArray &expr_array); + int set_index_for_expr(opsql::ObProxyExpr *expr); + protected: int64_t sub_sql_count_; }; @@ -60,127 +58,78 @@ protected: class ObProxyTableScanInput : public ObProxyOpInput { public: - ObProxyTableScanInput() - : ObProxyOpInput(), // { } - logical_table_name_(), - logical_database_name_(), - phy_db_table_names_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - db_key_names_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - condition_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - group_by_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - having_exprs_(NULL), - order_by_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, array_new_alloc_size), - hint_string_(), - normal_annotation_string_() - {} - - ObProxyTableScanInput(const common::ObString &logical_table_name, - const common::ObString &logical_database_name, - //const common::ObSEArray, 1> - // &phy_db_table_names, - const common::ObSEArray &phy_db_table_names, - const common::ObSEArray &db_key_names, - const common::ObSEArray &select_exprs, - const common::ObSEArray &condition_exprs, - const common::ObSEArray &group_by_exprs, - ObProxyExpr* having_exprs, - const common::ObSEArray &order_by_exprs, - const common::ObString &hint_string, - const common::ObString &normal_annotation_string) - : ObProxyOpInput(select_exprs), - logical_table_name_(logical_table_name), - logical_database_name_(logical_database_name), - phy_db_table_names_(phy_db_table_names), - db_key_names_(db_key_names), - condition_exprs_(condition_exprs), - group_by_exprs_(group_by_exprs), - having_exprs_(having_exprs), - order_by_exprs_(order_by_exprs), - hint_string_(hint_string), - normal_annotation_string_(normal_annotation_string) - { - } + ObProxyTableScanInput() : ObProxyOpInput(), request_sql_(), + table_name_maps_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + db_key_names_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + shard_props_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + calc_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + agg_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + group_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE), + order_exprs_(ObModIds::OB_SE_ARRAY_ENGINE, ENGINE_ARRAY_NEW_ALLOC_SIZE) {} - ~ObProxyTableScanInput() {} + ~ObProxyTableScanInput(); - void set_logical_table_name(const common::ObString &logical_table_name) { - logical_table_name_ = logical_table_name; - } - common::ObString& get_logical_table_name() { - return logical_table_name_; - } - - void set_logical_database_name(const common::ObString &logical_database_name) { - logical_database_name_ = logical_database_name; - } - common::ObString& get_logical_database_name() { - return logical_database_name_; - } + void set_request_sql(const ObString &request_sql) { request_sql_ = request_sql; } + ObString &get_request_sql() { return request_sql_; } - void set_db_key_names(const common::ObIArray &db_key_names) { - db_key_names_.assign(db_key_names); + int set_db_key_names(const common::ObIArray &db_key_names) { + return db_key_names_.assign(db_key_names); } - const common::ObSEArray& get_db_key_names() { + common::ObIArray &get_db_key_names() { return db_key_names_; } - void set_phy_db_table_names( - //const common::ObSEArray, 1> - const common::ObIArray &phy_db_table_names) { - phy_db_table_names_.assign(phy_db_table_names); + int set_shard_props(const common::ObIArray &shard_props) { + return shard_props_.assign(shard_props); } - //common::ObSEArray, 1>& - common::ObSEArray& get_phy_db_table_names() { - return phy_db_table_names_; + common::ObIArray &get_shard_props() { + return shard_props_; } - void set_condition_exprs(const common::ObSEArray &condition_exprs) { - condition_exprs_ = condition_exprs; + int set_table_name_maps(common::ObIArray > &table_name_map_array) { + return table_name_maps_.assign(table_name_map_array); } - common::ObSEArray& get_condition_exprs() { - return condition_exprs_; + common::ObIArray > &get_table_name_maps() { + return table_name_maps_; } - void set_group_by_exprs(const common::ObSEArray &group_by_exprs) { - group_by_exprs_ = group_by_exprs; + int set_calc_exprs(common::ObIArray &calc_exprs) { + return calc_exprs_.assign(calc_exprs); } - common::ObSEArray& get_group_by_exprs() { - return group_by_exprs_; + common::ObIArray &get_calc_exprs() { + return calc_exprs_; } - void set_order_by_exprs(const common::ObSEArray &order_by_exprs) { - order_by_exprs_ = order_by_exprs; + int set_agg_exprs(common::ObIArray &agg_exprs) { + return agg_exprs_.assign(agg_exprs); } - common::ObSEArray& get_order_by_exprs() { - return order_by_exprs_; + common::ObIArray &get_agg_exprs() { + return agg_exprs_; } - void set_hint_string(const common::ObString &hint_string) { - hint_string_ = hint_string; + int set_group_exprs(common::ObIArray &group_exprs) { + return group_exprs_.assign(group_exprs); } - common::ObString& get_hint_string() { - return hint_string_; + common::ObIArray &get_group_exprs() { + return group_exprs_; } - void set_normal_annotation_string(const common::ObString &normal_annotation_string) { - normal_annotation_string_ = normal_annotation_string; + int set_order_exprs(common::ObIArray &order_exprs) { + return order_exprs_.assign(order_exprs); } - common::ObString& get_normal_annotation_string() { - return normal_annotation_string_; + common::ObIArray &get_order_exprs() { + return order_exprs_; } -protected: - common::ObString logical_table_name_; - common::ObString logical_database_name_; - common::ObSEArray phy_db_table_names_; - common::ObSEArray db_key_names_; - //common::ObSEArray select_exprs_; - common::ObSEArray condition_exprs_; - common::ObSEArray group_by_exprs_; - ObProxyExpr* having_exprs_; - common::ObSEArray order_by_exprs_; - common::ObString hint_string_; - common::ObString normal_annotation_string_; +private: + ObString request_sql_; + ObSEArray, 4> table_name_maps_; + common::ObSEArray db_key_names_; + common::ObSEArray shard_props_; + common::ObSEArray calc_exprs_; + common::ObSEArray agg_exprs_; + common::ObSEArray group_exprs_; + common::ObSEArray order_exprs_; }; } diff --git a/src/obproxy/executor/ob_proxy_parallel_execute_cont.cpp b/src/obproxy/executor/ob_proxy_parallel_execute_cont.cpp index 1ed1f4d607a52c81058c5271f4e5b8487fd75fc7..51994759c90426d5ed939ee7b6d94863be998eec 100644 --- a/src/obproxy/executor/ob_proxy_parallel_execute_cont.cpp +++ b/src/obproxy/executor/ob_proxy_parallel_execute_cont.cpp @@ -83,7 +83,19 @@ int ObProxyParallelResp::next(ObObj *&rows) for (int64_t i = 0; OB_SUCC(ret) && i < column_count_; i++) { if (OB_FAIL(rs_fetcher_->get_obj(i, rows[i]))) { LOG_WARN("fail to get varchar", K(i), K(ret)); - } else if (rows[i].is_varchar()) { + } else if (rows[i].need_deep_copy()) { + int64_t copy_size = rows[i].get_deep_copy_size(); + int64_t pos = 0; + buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator_->alloc(copy_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem", K(copy_size), K(ret)); + } else if (OB_FAIL(rows[i].deep_copy(rows[i], buf, copy_size, pos))) { + LOG_WARN("fail to deep coy", "obj", rows[i], K(copy_size), K(ret)); + } + } + + if (OB_SUCC(ret) && rows[i].is_varchar()) { ObCollationType cs_type = static_cast(fields[i].charsetnr_); // utf8_general_ci => CS_TYPE_UTF8MB4_GENERAL_CI if (33 == fields[i].charsetnr_) { @@ -100,6 +112,9 @@ int ObProxyParallelResp::next(ObObj *&rows) if (OB_FAIL(ObSMUtils::get_ob_type(ob_type, fields[i].type_))) { LOG_INFO("cast ob type from mysql type failed", K(ob_type), "elem_type", fields[i].type_, K(ret)); ret = OB_SUCCESS; + } else if (ObTimestampType == ob_type || ObTimeType == ob_type + || ObDateType == ob_type || ObDateTimeType == ob_type) { + //do nothing } else { ObCastCtx cast_ctx(allocator_, NULL, CM_NULL_ON_WARN, cs_type); // use src_obj as buf_obj @@ -137,7 +152,8 @@ int ObProxyParallelExecuteCont::init(const ObProxyParallelParam ¶llel_param, LOG_WARN("fail to alloc ObMysqlProxy"); } else if (OB_FAIL(mysql_proxy_->init(timeout_ms, username, passwd_string, database_name))) { LOG_WARN("fail to init proxy", K(username), K(database_name)); - } else if (OB_FAIL(mysql_proxy_->rebuild_client_pool(shard_conn_, false, username, passwd_string, database_name))) { + } else if (OB_FAIL(mysql_proxy_->rebuild_client_pool(shard_conn_, parallel_param.shard_prop_, + false, username, passwd_string, database_name))) { LOG_WARN("fail to create mysql client pool", K(username), K(database_name), K(ret)); } else if (OB_FAIL(deep_copy_sql(parallel_param.request_sql_))) { LOG_WARN("fail to deep_copy_sql", K(parallel_param.request_sql_), K(ret)); diff --git a/src/obproxy/executor/ob_proxy_parallel_processor.h b/src/obproxy/executor/ob_proxy_parallel_processor.h index 2042ba33a563cdfa38d7cb4ca30c3d8c67626d08..a32ff1eeddaee9cf034fc59e5135e1fc4739626d 100644 --- a/src/obproxy/executor/ob_proxy_parallel_processor.h +++ b/src/obproxy/executor/ob_proxy_parallel_processor.h @@ -27,13 +27,14 @@ namespace executor class ObProxyParallelParam { public: - ObProxyParallelParam() : shard_conn_(NULL), request_sql_() {} + ObProxyParallelParam() : shard_conn_(NULL), shard_prop_(NULL), request_sql_() {} ~ObProxyParallelParam() {} TO_STRING_KV(KPC_(shard_conn), K_(request_sql)); public: dbconfig::ObShardConnector* shard_conn_; + dbconfig::ObShardProp* shard_prop_; common::ObString request_sql_; }; diff --git a/src/obproxy/iocore/eventsystem/ob_ethread.cpp b/src/obproxy/iocore/eventsystem/ob_ethread.cpp index 95d764bac29a1eab428eb5fdca158868134d4ed8..a71e6ff888ec1cd615a035b4569914bcba937958 100644 --- a/src/obproxy/iocore/eventsystem/ob_ethread.cpp +++ b/src/obproxy/iocore/eventsystem/ob_ethread.cpp @@ -51,6 +51,7 @@ ObEThread::ObEThread() id_(NO_ETHREAD_ID), event_types_(0), stack_start_(0), + dedicate_thread_type_(DEDICATE_THREAD_NONE), signal_hook_(NULL), ep_(NULL), net_handler_(NULL), @@ -63,11 +64,13 @@ ObEThread::ObEThread() congestion_map_(NULL), cache_cleaner_(NULL), sql_table_map_(NULL), + ps_entry_cache_(NULL), random_seed_(NULL), warn_log_buf_(NULL), warn_log_buf_start_(NULL), tt_(REGULAR), - pending_event_(NULL) + pending_event_(NULL), + thread_prometheus_(NULL) { #if OB_HAVE_EVENTFD evfd_ = -1; @@ -84,6 +87,7 @@ ObEThread::ObEThread(const ObThreadType att, const int64_t anid) id_(anid), event_types_(0), stack_start_(0), + dedicate_thread_type_(DEDICATE_THREAD_NONE), signal_hook_(NULL), ep_(NULL), net_handler_(NULL), @@ -96,11 +100,13 @@ ObEThread::ObEThread(const ObThreadType att, const int64_t anid) congestion_map_(NULL), cache_cleaner_(NULL), sql_table_map_(NULL), + ps_entry_cache_(NULL), random_seed_(NULL), warn_log_buf_(NULL), warn_log_buf_start_(NULL), tt_(att), - pending_event_(NULL) + pending_event_(NULL), + thread_prometheus_(NULL) { #if OB_HAVE_EVENTFD evfd_ = -1; @@ -117,6 +123,7 @@ ObEThread::ObEThread(const ObThreadType att, ObEvent *e) id_(NO_ETHREAD_ID), event_types_(0), stack_start_(0), + dedicate_thread_type_(DEDICATE_THREAD_NONE), signal_hook_(NULL), ep_(NULL), net_handler_(NULL), @@ -129,11 +136,13 @@ ObEThread::ObEThread(const ObThreadType att, ObEvent *e) congestion_map_(NULL), cache_cleaner_(NULL), sql_table_map_(NULL), + ps_entry_cache_(NULL), random_seed_(NULL), warn_log_buf_(NULL), warn_log_buf_start_(NULL), tt_(att), - pending_event_(e) + pending_event_(e), + thread_prometheus_(NULL) { #if OB_HAVE_EVENTFD evfd_ = -1; diff --git a/src/obproxy/iocore/eventsystem/ob_ethread.h b/src/obproxy/iocore/eventsystem/ob_ethread.h index 527cf56bba9f8de87bda0942408c4a3c529a5a9d..2b7eecf90c0654c94987e6268dfdc8fc515d5af9 100644 --- a/src/obproxy/iocore/eventsystem/ob_ethread.h +++ b/src/obproxy/iocore/eventsystem/ob_ethread.h @@ -56,6 +56,7 @@ class ObPartitionRefHashMap; class ObRoutineRefHashMap; class ObSqlTableRefHashMap; class ObCacheCleaner; +class ObBasePsEntryCache; } namespace net { @@ -65,6 +66,11 @@ class ObNetPoll; class ObInactivityCop; } +namespace prometheus +{ +class ObThreadPrometheus; +} + namespace event { class ObEvent; @@ -79,6 +85,12 @@ enum ObThreadType DEDICATED }; +enum ObDedicateThreadType +{ + DEDICATE_THREAD_NONE = 0, + DEDICATE_THREAD_ACCEPT, +}; + /** * ObEvent System specific type of thread. * The ObEThread class is the type of thread created and managed by @@ -301,6 +313,8 @@ public: bool is_event_thread_type(const ObEventThreadType et) { return !!(event_types_ & (1 << et)); } void set_event_thread_type(const ObEventThreadType et) { event_types_ |= (1 << et); } + ObDedicateThreadType get_dedicate_type() { return dedicate_thread_type_; } + void set_dedicate_type(const ObDedicateThreadType dedicate_thread_type) { dedicate_thread_type_ = dedicate_thread_type; } net::ObNetHandler &get_net_handler() { return *net_handler_; } net::ObNetPoll &get_net_poll() { return *net_poll_; } @@ -310,6 +324,7 @@ public: proxy::ObSqlTableRefHashMap &get_sql_table_map() { return *sql_table_map_; } proxy::ObPartitionRefHashMap &get_partition_map() { return *partition_map_; } proxy::ObRoutineRefHashMap &get_routine_map() { return *routine_map_; } + proxy::ObBasePsEntryCache &get_ps_entry_cache() { return *ps_entry_cache_; } obutils::ObCongestionRefHashMap &get_cgt_map() { return *congestion_map_; } common::ObMysqlRandom &get_random_seed() { return *random_seed_; } @@ -343,6 +358,7 @@ public: int64_t id_; int64_t event_types_; int64_t stack_start_; // statck start pos, used to minitor stack size + ObDedicateThreadType dedicate_thread_type_; int (*signal_hook_)(ObEThread &); @@ -363,6 +379,7 @@ public: obutils::ObCongestionRefHashMap *congestion_map_; proxy::ObCacheCleaner *cache_cleaner_; proxy::ObSqlTableRefHashMap *sql_table_map_; + proxy::ObBasePsEntryCache *ps_entry_cache_; common::ObMysqlRandom *random_seed_; char *warn_log_buf_; @@ -370,6 +387,7 @@ public: ObThreadType tt_; ObEvent *pending_event_; // For dedicated event thread + prometheus::ObThreadPrometheus *thread_prometheus_; private: // prevent unauthorized copies (Not implemented) diff --git a/src/obproxy/iocore/eventsystem/ob_event_processor.cpp b/src/obproxy/iocore/eventsystem/ob_event_processor.cpp index b0d40711485efdee528a62958a070c1d1c874b58..91b9c946ebc4cdaaf30ea86fa2fbb1bb6be1185a 100644 --- a/src/obproxy/iocore/eventsystem/ob_event_processor.cpp +++ b/src/obproxy/iocore/eventsystem/ob_event_processor.cpp @@ -280,7 +280,8 @@ int ObEventProcessor::start(const int64_t net_thread_count, const int64_t stacks } ObEvent *ObEventProcessor::spawn_thread( - ObContinuation *cont, const char *thr_name, const int64_t stacksize) + ObContinuation *cont, const char *thr_name, const int64_t stacksize, + ObDedicateThreadType dedicate_thread_type) { int ret = OB_SUCCESS; ObEvent *event = NULL; @@ -310,6 +311,7 @@ ObEvent *ObEventProcessor::spawn_thread( delete all_dedicate_threads_[dedicate_thread_count_]; all_dedicate_threads_[dedicate_thread_count_] = NULL; } else { + all_dedicate_threads_[dedicate_thread_count_]->set_dedicate_type(dedicate_thread_type); all_dedicate_threads_[dedicate_thread_count_]->id_ = dedicate_thread_count_; event->ethread_ = all_dedicate_threads_[dedicate_thread_count_]; event->continuation_->mutex_ = all_dedicate_threads_[dedicate_thread_count_]->mutex_; diff --git a/src/obproxy/iocore/eventsystem/ob_event_processor.h b/src/obproxy/iocore/eventsystem/ob_event_processor.h index 81d825a8395da57f2cf190bc750ca3fd7cc43883..5c7acac3e768dca5ad01317275a9b5c39a9c1790 100644 --- a/src/obproxy/iocore/eventsystem/ob_event_processor.h +++ b/src/obproxy/iocore/eventsystem/ob_event_processor.h @@ -131,7 +131,8 @@ public: * @return event object representing the start of the thread. */ ObEvent *spawn_thread(ObContinuation *cont, const char *thr_name, - const int64_t stacksize = 0); + const int64_t stacksize = 0, + ObDedicateThreadType dedicate_thread_type = DEDICATE_THREAD_NONE); /** * Spawns a group of threads for an event type. Spawns the number of diff --git a/src/obproxy/iocore/eventsystem/ob_io_buffer.h b/src/obproxy/iocore/eventsystem/ob_io_buffer.h index 1bbb7c234de028bec61cc33d2f977508b943f0ae..4679c1d2310c4a6a4525111e6bd4b453d108eba0 100644 --- a/src/obproxy/iocore/eventsystem/ob_io_buffer.h +++ b/src/obproxy/iocore/eventsystem/ob_io_buffer.h @@ -206,7 +206,7 @@ public: * * @return pointer to the underlying data. */ - char *buf() { return data_->data_; } + inline char *buf() { return data_->data_; } /** * Beginning of the inuse section. Returns the position in the buffer @@ -214,7 +214,7 @@ public: * * @return pointer to the start of the inuse section. */ - char *start() { return start_; } + inline char *start() { return start_; } /** * End of the used space. Returns a pointer to end of the used space @@ -222,7 +222,7 @@ public: * * @return pointer to the end of the inuse portion of the block. */ - char *end() { return end_; } + inline char *end() { return end_; } /** * End of the data buffer. Returns a pointer to end of the data buffer @@ -230,14 +230,14 @@ public: * * @return */ - char *buf_end() { return buf_end_; } + inline char *buf_end() { return buf_end_; } /** * Size of the inuse area. Returns the size of the current inuse area. * * @return bytes occupied by the inuse area. */ - int64_t size() const { return static_cast(end_ - start_); } + inline int64_t size() const { return static_cast(end_ - start_); } /** * Size of the data available for reading. Returns the size of the data @@ -245,7 +245,7 @@ public: * * @return bytes available for reading from the inuse area. */ - int64_t read_avail() const { return static_cast(end_ - start_); } + inline int64_t read_avail() const { return static_cast(end_ - start_); } /** * Space available in the buffer. Returns the number of bytes that can @@ -253,7 +253,7 @@ public: * * @return space available for writing in this ObIOBufferBlock. */ - int64_t write_avail() const { return static_cast(buf_end_ - end_); } + inline int64_t write_avail() const { return static_cast(buf_end_ - end_); } /** * Size of the memory allocated by the underlying ObIOBufferData. @@ -799,16 +799,14 @@ public: { ObIOBufferBlock *ret = NULL; - if (NULL != writer_) { + if (OB_LIKELY(NULL != writer_)) { if (NULL != writer_->next_ && 0 == writer_->write_avail()) { ret = writer_->next_; + } else if (OB_UNLIKELY(NULL != writer_->next_) && OB_UNLIKELY(0 != writer_->next_->read_avail())) { + PROXY_EVENT_LOG(ERROR, "next block read avail must be 0", + "next_block_read_avail", writer_->next_->read_avail()); } else { - if (OB_UNLIKELY(NULL != writer_->next_) && OB_UNLIKELY(0 != writer_->next_->read_avail())) { - PROXY_EVENT_LOG(ERROR, "next block read avail must be 0", - "next_block_read_avail", writer_->next_->read_avail()); - } else { - ret = writer_; - } + ret = writer_; } } @@ -2006,7 +2004,7 @@ inline int ObMIOBuffer::add_block(const int64_t block_count) inline int ObMIOBuffer::check_add_block() { int ret = common::OB_SUCCESS; - if (!is_high_water() && is_current_low_water()) { + if (OB_UNLIKELY(!is_high_water() && is_current_low_water())) { ret = add_block(); } return ret; diff --git a/src/obproxy/iocore/eventsystem/ob_thread_impl.h b/src/obproxy/iocore/eventsystem/ob_thread_impl.h index d55f230d17d6647d32f39a0c3f4625bd62ca662a..1cbcf2bb56664523b095a70572fa9cb32bce6d56 100644 --- a/src/obproxy/iocore/eventsystem/ob_thread_impl.h +++ b/src/obproxy/iocore/eventsystem/ob_thread_impl.h @@ -142,6 +142,17 @@ static inline int thread_join(ObThreadId t) return ret; } +static inline int thread_kill(ObThreadId t, int sig) +{ + int ret = common::OB_SUCCESS; + int err_code = 0; + if (OB_UNLIKELY(0 != (err_code = pthread_kill(t, sig)))) { + ret = ob_get_sys_errno(err_code); + PROXY_EVENT_LOG(ERROR, "failed to pthread_kill", KERRNOMSGS(err_code), K(ret)); + } + return ret; +} + static inline ObThreadId thread_self() { return (pthread_self()); diff --git a/src/obproxy/iocore/net/ob_connection.cpp b/src/obproxy/iocore/net/ob_connection.cpp index 0717d593ce6e57ac5a05e24388894b5f77fcaa91..66dee9a9b45c79a444e12a12a0d1cab201926348 100644 --- a/src/obproxy/iocore/net/ob_connection.cpp +++ b/src/obproxy/iocore/net/ob_connection.cpp @@ -350,7 +350,7 @@ void ObConnection::cleanup() } } -int ObServerConnection::accept(ObConnection *c) +int ObServerConnection::accept(ObConnection *c, bool need_return_eintr /* false */) { int ret = OB_SUCCESS; int res = -1; @@ -359,8 +359,8 @@ int ObServerConnection::accept(ObConnection *c) PROXY_SOCK_LOG(WARN, "invalid argument conn", K(c), K(ret)); } else { int64_t sz = sizeof(c->addr_.sa_); - if (OB_FAIL(ObSocketManager::accept(fd_, &c->addr_.sa_, &sz, c->fd_))) { - if (OB_SYS_EAGAIN != ret) { + if (OB_FAIL(ObSocketManager::accept(fd_, &c->addr_.sa_, &sz, c->fd_, need_return_eintr))) { + if (OB_SYS_EAGAIN != ret && (!need_return_eintr || OB_SYS_EINTR != ret)) { PROXY_SOCK_LOG(WARN, "fail to accept", K(fd_), K(ret)); } } else { diff --git a/src/obproxy/iocore/net/ob_connection.h b/src/obproxy/iocore/net/ob_connection.h index 360ef344b3417adfd623ff6da45b5f796c0b98b8..31cf184e77ca12b20813dbfe620e21ae755c7add 100644 --- a/src/obproxy/iocore/net/ob_connection.h +++ b/src/obproxy/iocore/net/ob_connection.h @@ -133,7 +133,7 @@ public: } virtual ~ObServerConnection() {}; - int accept(ObConnection *c); + int accept(ObConnection *c, bool need_return_eintr = false); int setup_fd_for_listen( const bool non_blocking = false, diff --git a/src/obproxy/iocore/net/ob_inet.h b/src/obproxy/iocore/net/ob_inet.h index c7b31477aac0a0db78612d7abe6f229e3ed64e2c..369d8ce39fdd0bd6883a1a2d596321e01715fda8 100644 --- a/src/obproxy/iocore/net/ob_inet.h +++ b/src/obproxy/iocore/net/ob_inet.h @@ -88,7 +88,10 @@ union ObIpEndpoint ~ObIpEndpoint() {} void reset() { memset(this, 0, sizeof(ObIpEndpoint)); } - explicit ObIpEndpoint(const sockaddr &addr) { assign(addr); } + explicit ObIpEndpoint(const sockaddr &addr) { + memset(this, 0, sizeof(ObIpEndpoint)); + assign(addr); + } ObIpEndpoint (const ObIpEndpoint &point) { MEMCPY(this, &point, sizeof(ObIpEndpoint)); } void operator= (const ObIpEndpoint &point) { MEMCPY(this, &point, sizeof(ObIpEndpoint)); } diff --git a/src/obproxy/iocore/net/ob_net_accept.cpp b/src/obproxy/iocore/net/ob_net_accept.cpp index 9e7c4f14de1f905bd4e9b7fe3574f80b9af93ee5..1a4d12ca569235b4b2ddcdbedd8136ece5118664 100644 --- a/src/obproxy/iocore/net/ob_net_accept.cpp +++ b/src/obproxy/iocore/net/ob_net_accept.cpp @@ -156,7 +156,7 @@ int ObNetAccept::init_accept_loop(const char *thread_name, const int64_t stacksi } else { NET_SUM_GLOBAL_DYN_STAT(NET_GLOBAL_ACCEPTS_CURRENTLY_OPEN, 1); SET_CONTINUATION_HANDLER(this, &ObNetAccept::accept_loop_event); - if (OB_ISNULL(g_event_processor.spawn_thread(this, thread_name, stacksize))) { + if (OB_ISNULL(g_event_processor.spawn_thread(this, thread_name, stacksize, DEDICATE_THREAD_ACCEPT))) { ret = OB_ERR_UNEXPECTED; PROXY_NET_LOG(ERROR, "g_event_processor fail to spawn_thread", K(ret)); } @@ -327,8 +327,12 @@ int ObNetAccept::do_blocking_accept() continue; } - if (OB_FAIL(server_.accept(&con))) { - PROXY_NET_LOG(ERROR, "failed to accept con", K(ret)); + if (OB_FAIL(server_.accept(&con, true))) { + if (OB_LIKELY(OB_SYS_EINTR == ret && !info.need_conn_accept_)) { + PROXY_NET_LOG(DEBUG, "accept interrupted when need_conn_accept is false", K(ret), "need_conn_accept", info.need_conn_accept_); + } else { + PROXY_NET_LOG(ERROR, "failed to accept con", K(ret)); + } } else { NET_SUM_GLOBAL_DYN_STAT(NET_GLOBAL_CONNECTIONS_CURRENTLY_OPEN, 1); NET_SUM_GLOBAL_DYN_STAT(NET_GLOBAL_CLIENT_CONNECTIONS_CURRENTLY_OPEN, 1); diff --git a/src/obproxy/iocore/net/ob_socket_manager.h b/src/obproxy/iocore/net/ob_socket_manager.h index e667caeaba77ea9d4a519139a411ed16da014ef0..223b62e3d5ad2ee454f9d33c07d154ff7f3af5e8 100644 --- a/src/obproxy/iocore/net/ob_socket_manager.h +++ b/src/obproxy/iocore/net/ob_socket_manager.h @@ -59,7 +59,7 @@ class ObSocketManager const int maxevents, const int timeout, int64_t &count); static int socket(const int domain, const int type, const int protocol, int &sockfd); - static int accept(int sockfd, struct sockaddr *addr, int64_t *addrlen, int &fd); + static int accept(int sockfd, struct sockaddr *addr, int64_t *addrlen, int &fd, bool need_return_eintr = false); static int bind(int sockfd, const struct sockaddr *name, const int64_t namelen); static int listen(int sockfd, const int backlog); static int connect(int sockfd, const struct sockaddr *addr, const int64_t addrlen); @@ -100,7 +100,7 @@ class ObSocketManager static int ssl_connect(SSL *ssl, bool &is_connected, int &tmp_code); static int ssl_read(SSL *ssl, void *buf, const size_t size, int64_t &count, int &tmp_code); static int ssl_write(SSL *ssl, const void *buf, const size_t size, int64_t &count, int &tmp_code); - static void print_ssl_error(); + static int handle_ssl_error(int tmp_code); private: DISALLOW_COPY_AND_ASSIGN(ObSocketManager); @@ -178,7 +178,7 @@ inline int ObSocketManager::socket(const int domain, const int type, const int p return ret; } -inline int ObSocketManager::accept(int sockfd, struct sockaddr *addr, int64_t *addrlen, int &fd) +inline int ObSocketManager::accept(int sockfd, struct sockaddr *addr, int64_t *addrlen, int &fd, bool need_return_eintr /* false */) { int ret = common::OB_SUCCESS; if (OB_UNLIKELY(sockfd < 3) || OB_ISNULL(addr) || OB_ISNULL(addrlen)) { @@ -186,7 +186,7 @@ inline int ObSocketManager::accept(int sockfd, struct sockaddr *addr, int64_t *a } else { do { fd = ::accept(sockfd, addr, reinterpret_cast(addrlen)); - } while (fd < 0 && EINTR == errno); + } while (fd < 0 && EINTR == errno && !need_return_eintr); if (OB_UNLIKELY(fd < 0)) { ret = ob_get_sys_errno(); } @@ -575,20 +575,11 @@ int ObSocketManager::ssl_accept(SSL *ssl, bool &is_connected, int &tmp_code) int tmp_ret = SSL_accept(ssl); if (1 != tmp_ret) { tmp_code = SSL_get_error(ssl, tmp_ret); - switch(tmp_code) { - case SSL_ERROR_NONE: - PROXY_NET_LOG(DEBUG, "ssl server accept succ", K(ret)); - is_connected = true; - break; - // SSL_accept() will yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - break; - default: - ret = common::OB_SSL_ERROR; - print_ssl_error(); - PROXY_NET_LOG(WARN, "unexpected error", K(ret), K(tmp_code)); - break; + if (SSL_ERROR_NONE == tmp_code) { + PROXY_NET_LOG(DEBUG, "ssl server accept succ", K(ret)); + is_connected = true; + } else { + ret = handle_ssl_error(tmp_code); } } else { is_connected = true; @@ -610,20 +601,11 @@ int ObSocketManager::ssl_connect(SSL *ssl, bool &is_connected, int &tmp_code) int tmp_ret = SSL_connect(ssl); if (1 != tmp_ret) { tmp_code = SSL_get_error(ssl, tmp_ret); - switch(tmp_code) { - case SSL_ERROR_NONE: - PROXY_NET_LOG(DEBUG, "ssl client connect succ", K(ret)); - is_connected = true; - break; - // SSL_connect() will yield SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE - case SSL_ERROR_WANT_WRITE: - case SSL_ERROR_WANT_READ: - break; - default: - ret = common::OB_SSL_ERROR; - print_ssl_error(); - PROXY_NET_LOG(WARN, "unexpected error", K(ret), K(tmp_code)); - break; + if (SSL_ERROR_NONE == tmp_code) { + PROXY_NET_LOG(DEBUG, "ssl client connect succ", K(ret)); + is_connected = true; + } else { + ret = handle_ssl_error(tmp_code); } } else { is_connected = true; @@ -647,17 +629,8 @@ int ObSocketManager::ssl_read(SSL *ssl, void *buf, const size_t size, count = SSL_read(ssl, buf, static_cast(size)); if (count <= 0 && !last_succ) { tmp_code = SSL_get_error(ssl, static_cast(count)); - switch(tmp_code) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - count = 0; - break; - default: - ret = common::OB_SSL_ERROR; - print_ssl_error(); - PROXY_NET_LOG(WARN, "ssl unexpected error", K(ret), K(tmp_code), K(count)); - break; - } + ret = handle_ssl_error(tmp_code); + count = 0; } else { PROXY_NET_LOG(DEBUG, "ssl read succ", K(count)); } @@ -678,17 +651,8 @@ int ObSocketManager::ssl_write(SSL *ssl, const void *buf, const size_t size, count = SSL_write(ssl, buf, static_cast(size)); if (count <= 0 && !last_succ) { tmp_code = SSL_get_error(ssl, static_cast(count)); - switch(tmp_code) { - case SSL_ERROR_WANT_WRITE: - case SSL_ERROR_WANT_READ: - count = 0; - break; - default: - ret = common::OB_SSL_ERROR; - print_ssl_error(); - PROXY_NET_LOG(WARN, "ssl unexpecte error", K(ret), K(tmp_code), K(count)); - break; - } + ret = handle_ssl_error(tmp_code); + count = 0; } else { PROXY_NET_LOG(DEBUG, "ssl write succ", K(count)); } @@ -696,12 +660,27 @@ int ObSocketManager::ssl_write(SSL *ssl, const void *buf, const size_t size, return ret; } -void ObSocketManager::print_ssl_error() +inline int ObSocketManager::handle_ssl_error(int tmp_code) { - char temp_buf[512]; - unsigned long e = ERR_peek_last_error(); - ERR_error_string_n(e, temp_buf, sizeof(temp_buf)); - PROXY_NET_LOG(WARN, "SSL error", K(e), K(temp_buf), K(errno), K(strerror(errno))); + int ret = common::OB_SUCCESS; + switch(tmp_code) { + case SSL_ERROR_WANT_WRITE: + case SSL_ERROR_WANT_READ: + break; + case SSL_ERROR_ZERO_RETURN: + ret = common::OB_SSL_ERROR; + PROXY_NET_LOG(WARN, "the ssl peer has closed", K(ret)); + break; + default: + ret = common::OB_SSL_ERROR; + char temp_buf[512]; + unsigned long e = ERR_peek_last_error(); + ERR_error_string_n(e, temp_buf, sizeof(temp_buf)); + PROXY_NET_LOG(WARN, "SSL error", K(e), K(temp_buf), K(errno), K(strerror(errno))); + break; + } + + return ret; } } // end of namespace net diff --git a/src/obproxy/iocore/net/ob_ssl_processor.cpp b/src/obproxy/iocore/net/ob_ssl_processor.cpp index 1a7a313d327b92334f446e5259aa8b7092de1539..ac06b0d2edc941da94a402d91dc36a1bac895206 100644 --- a/src/obproxy/iocore/net/ob_ssl_processor.cpp +++ b/src/obproxy/iocore/net/ob_ssl_processor.cpp @@ -19,8 +19,10 @@ #include "obutils/ob_proxy_config.h" #include "lib/atomic/ob_atomic.h" #include "lib/alloc/malloc_hook.h" +#include "utils/ob_proxy_utils.h" using namespace oceanbase::common; +using namespace oceanbase::common::hash; using namespace oceanbase::obproxy::obutils; using namespace oceanbase::obproxy::event; @@ -44,14 +46,8 @@ int ObSSLProcessor::init() OpenSSL_add_all_algorithms(); SSL_load_error_strings(); - - SSL_CTX *ssl_ctx = SSL_CTX_new(SSLv23_method()); - if (OB_ISNULL(ssl_ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ssl ctx is null", K(ret)); - } else { - ssl_ctx_ = ssl_ctx; - SSL_CTX_set_verify(ssl_ctx_, SSL_VERIFY_PEER, NULL); + if (OB_FAIL(ssl_ctx_map_.create(32, ObModIds::OB_PROXY_SSL_RELATED))) { + LOG_WARN("create ssl ctx map failed", K(ret)); } return ret; @@ -85,66 +81,121 @@ void ObSSLProcessor::free_for_ssl(void *ptr) lib::glibc_hook_opt = lib::GHO_NOHOOK; } -int ObSSLProcessor::update_key(const ObString &source_type, +int ObSSLProcessor::update_key(const ObString &cluster_name, + const ObString &tenant_name, + const ObString &source_type, const ObString &ca, const ObString &public_key, const ObString &private_key) { int ret = OB_SUCCESS; - if (source_type == "DBMESH") { - if (OB_FAIL(update_key_from_dbmesh(ca, public_key, private_key))) { - LOG_WARN("update key from dbmesh failed", K(ret), K(ca), K(public_key), K(private_key)); + ObFixedLengthString key_string; + SSL_CTX *ssl_ctx = NULL; + SSL_CTX *old_ssl_ctx = NULL; + if (OB_UNLIKELY(cluster_name.empty() || tenant_name.empty() || source_type.empty() + || ca.empty() || public_key.empty() || private_key.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("argument is invalid", K(ret), K(ca), K(public_key), K(private_key)); + } else if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluser name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + DRWLock::WRLockGuard guard(ssl_ctx_lock_); + if (OB_FAIL(ssl_ctx_map_.get_refactored(key_string, old_ssl_ctx))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } else { + ret = OB_SUCCESS; + } + } + + if (OB_SUCC(ret)) { + if (OB_ISNULL(ssl_ctx = SSL_CTX_new(SSLv23_method()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ssl ctx new failed", K(ret)); + } else { + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + } + } + + if (OB_SUCC(ret)) { + if (source_type == "DBMESH" || source_type == "KEY") { + if (OB_FAIL(update_key_from_string(ssl_ctx, ca, public_key, private_key))) { + LOG_WARN("update key from dbmesh failed", K(ret), K(cluster_name), K(tenant_name), + K(ca), K(public_key), K(private_key)); + } else { + LOG_INFO("update key from string succ", K(tenant_name), K(cluster_name)); + } + } else if (source_type == "FILE") { + if (OB_FAIL(update_key_from_file(ssl_ctx, ca, public_key, private_key))) { + LOG_WARN("update key from file failed", K(ret), K(ca), K(public_key), K(private_key)); + } else { + LOG_INFO("update key from file succ", K(tenant_name), K(cluster_name), K(ca), K(public_key), K(private_key)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown source type", K(source_type), K(ret)); + } } - } else if (source_type == "FILE") { - if (OB_FAIL(update_key_from_file(ca, public_key, private_key))) { - LOG_WARN("update key from file failed", K(ret), K(ca), K(public_key), K(private_key)); + + if (OB_SUCC(ret)) { + if (OB_FAIL(ssl_ctx_map_.set_refactored(key_string, ssl_ctx, 1))) { + LOG_WARN("ssl ctx map set refactored failed", K(ret)); + } else { + LOG_INFO("ssl inited succ", K(cluster_name), K(tenant_name), K(ca), K(public_key), K(private_key)); + if (OB_LIKELY(NULL != old_ssl_ctx)) { + SSL_CTX_free(old_ssl_ctx); + } + } + } + + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(ssl_ctx)) { + SSL_CTX_free(ssl_ctx); + } } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unknown source type", K(source_type), K(ret)); } return ret; } -int ObSSLProcessor::update_key_from_file(const ObString &ca, +int ObSSLProcessor::update_key_from_file(SSL_CTX *ssl_ctx, + const ObString &ca, const ObString &public_key, const ObString &private_key) { int ret = OB_SUCCESS; - if (ca.empty() || public_key.empty() || private_key.empty()) { + if (NULL == ssl_ctx || ca.empty() || public_key.empty() + || private_key.empty()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("argument is invalid", K(ret), K(ca), K(public_key), K(private_key)); + LOG_WARN("invalid argument", K(ret)); } else { - if (OB_SSL_SUCC_RET != SSL_CTX_load_verify_locations(ssl_ctx_, ca.ptr(), NULL)) { + if (OB_SSL_SUCC_RET != SSL_CTX_load_verify_locations(ssl_ctx, ca.ptr(), NULL)) { ret = OB_SSL_ERROR; LOG_WARN("load verify location failed", K(ca), K(ret)); - } else if (OB_SSL_SUCC_RET != SSL_CTX_use_certificate_chain_file(ssl_ctx_, public_key.ptr())) { + } else if (OB_SSL_SUCC_RET != SSL_CTX_use_certificate_chain_file(ssl_ctx, public_key.ptr())) { ret = OB_SSL_ERROR; LOG_WARN("use certificate file failed", K(ret), K(public_key)); - } else if (OB_SSL_SUCC_RET != SSL_CTX_use_PrivateKey_file(ssl_ctx_, private_key.ptr(), SSL_FILETYPE_PEM)) { + } else if (OB_SSL_SUCC_RET != SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key.ptr(), SSL_FILETYPE_PEM)) { ret = OB_SSL_ERROR; LOG_WARN("use private key file failed", K(ret), K(private_key)); - } else if (OB_SSL_SUCC_RET != SSL_CTX_check_private_key(ssl_ctx_)) { + } else if (OB_SSL_SUCC_RET != SSL_CTX_check_private_key(ssl_ctx)) { ret = OB_SSL_ERROR; LOG_WARN("check private key failed", K(ret), K(ca), K(public_key), K(private_key)); - } else { - ATOMIC_STORE(&ssl_inited_, true); - LOG_INFO("ssl inited using file succ"); } } return ret; } -int ObSSLProcessor::update_key_from_dbmesh(const ObString &ca, +int ObSSLProcessor::update_key_from_string(SSL_CTX *ssl_ctx, + const ObString &ca, const ObString &public_key, const ObString &private_key) { int ret = OB_SUCCESS; - if (ca.empty() || public_key.empty() || private_key.empty()) { + if (NULL == ssl_ctx || ca.empty() || public_key.empty() || private_key.empty()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("argument is invalid", K(ret), K(ca), K(public_key), K(private_key)); + LOG_WARN("invalid argument", K(ret)); } else { //load ca cert BIO *cbio = BIO_new_mem_buf((void*)ca.ptr(), -1); @@ -157,7 +208,7 @@ int ObSSLProcessor::update_key_from_dbmesh(const ObString &ca, ret = OB_SSL_ERROR; LOG_WARN("x509 store add cert failed", K(ret)); } else { - SSL_CTX_set_cert_store(ssl_ctx_, x509_store); + SSL_CTX_set_cert_store(ssl_ctx, x509_store); } // load app cert chain @@ -170,13 +221,13 @@ int ObSSLProcessor::update_key_from_dbmesh(const ObString &ca, if (itmp->x509) { if (is_first) { is_first = 0; - if (OB_SSL_SUCC_RET != SSL_CTX_use_certificate(ssl_ctx_, itmp->x509)) { + if (OB_SSL_SUCC_RET != SSL_CTX_use_certificate(ssl_ctx, itmp->x509)) { ret = OB_SSL_ERROR; sk_X509_INFO_pop_free(inf, X509_INFO_free); //cleanup LOG_WARN("ssl ctx use cerjtificate failed", K(ret)); } } else { - if (OB_SSL_SUCC_RET != SSL_CTX_add_extra_chain_cert(ssl_ctx_, itmp->x509)) { + if (OB_SSL_SUCC_RET != SSL_CTX_add_extra_chain_cert(ssl_ctx, itmp->x509)) { ret = OB_SSL_ERROR; sk_X509_INFO_pop_free(inf, X509_INFO_free); //cleanup LOG_WARN("ssl ctx add extra chain cert failed", K(ret)); @@ -200,27 +251,73 @@ int ObSSLProcessor::update_key_from_dbmesh(const ObString &ca, if (NULL == (rsa = PEM_read_bio_RSAPrivateKey(cbio, NULL, 0, NULL))) { ret = OB_SSL_ERROR; LOG_WARN("pem read bio rsaprivatekey failed", K(ret)); - } else if (OB_SSL_SUCC_RET != SSL_CTX_use_RSAPrivateKey(ssl_ctx_, rsa)) { + } else if (OB_SSL_SUCC_RET != SSL_CTX_use_RSAPrivateKey(ssl_ctx, rsa)) { ret = OB_SSL_ERROR; LOG_WARN("ssl ctx use rsaprivatekey failed", K(ret)); } else { LOG_DEBUG("update ssl key from dbmesh"); } } - - if (OB_SUCC(ret)) { - ATOMIC_STORE(&ssl_inited_, true); - LOG_INFO("ssl inited using dbmesh succ"); - } } return ret; } -SSL* ObSSLProcessor::create_new_ssl() +SSL* ObSSLProcessor::create_new_ssl(const common::ObString &cluster_name, + const common::ObString &tenant_name) { + int ret = OB_SUCCESS; SSL *new_ssl = NULL; - new_ssl = SSL_new(ssl_ctx_); + SSL_CTX *ssl_ctx = NULL; + ObFixedLengthString key_string; + // Take the tenant-level configuration first + DRWLock::RDLockGuard guard(ssl_ctx_lock_); + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + if (OB_FAIL(ssl_ctx_map_.get_refactored(key_string, ssl_ctx))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } + } + + // If you do not get the tenant configuration, get the cluster configuration + if (OB_HASH_NOT_EXIST == ret) { + key_string.reset(); + if (OB_FAIL(paste_tenant_and_cluster_name("*", cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + if (OB_FAIL(ssl_ctx_map_.get_refactored(key_string, ssl_ctx))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } + } + } + + // If you don't get the cluster configuration, get the global configuration + if (OB_HASH_NOT_EXIST == ret) { + key_string.reset(); + if (OB_FAIL(paste_tenant_and_cluster_name("*", "*", key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + if (OB_FAIL(ssl_ctx_map_.get_refactored(key_string, ssl_ctx))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_ISNULL(ssl_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ssl ctx is null unexpected", K(ret)); + } else { + new_ssl = SSL_new(ssl_ctx); + } + } return new_ssl; } @@ -235,14 +332,18 @@ void ObSSLProcessor::release_ssl(SSL* ssl, const bool can_shutdown_ssl) } } -bool ObSSLProcessor::is_client_ssl_supported() -{ - return get_global_proxy_config().enable_client_ssl && ATOMIC_LOAD(&ssl_inited_); -} - -bool ObSSLProcessor::is_server_ssl_supported() +void ObSSLProcessor::release_ssl_ctx(common::ObFixedLengthString &delete_info) { - return get_global_proxy_config().enable_server_ssl && ATOMIC_LOAD(&ssl_inited_); + int ret = OB_SUCCESS; + SSL_CTX *ssl_ctx = NULL; + DRWLock::RDLockGuard guard(ssl_ctx_lock_); + if (OB_FAIL(ssl_ctx_map_.erase_refactored(delete_info, &ssl_ctx))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("get ssl ctx failed", K(ret), K(delete_info)); + } + } else if (NULL != ssl_ctx) { + SSL_CTX_free(ssl_ctx); + } } } // end net diff --git a/src/obproxy/iocore/net/ob_ssl_processor.h b/src/obproxy/iocore/net/ob_ssl_processor.h index 51a01179ec681f07007de37da5409e29c3a3fdd0..359bfb0de9a3370f5b5dc673b856fc27e5782351 100644 --- a/src/obproxy/iocore/net/ob_ssl_processor.h +++ b/src/obproxy/iocore/net/ob_ssl_processor.h @@ -16,8 +16,9 @@ #include #include -#include "iocore/eventsystem/ob_lock.h" #include "lib/lock/ob_drw_lock.h" +#include "lib/hash/ob_hashmap.h" +#include "utils/ob_proxy_lib.h" #define OB_SSL_SUCC_RET 1 @@ -31,26 +32,30 @@ namespace net class ObSSLProcessor { public: - ObSSLProcessor() : ssl_ctx_(NULL), ssl_inited_(false) {} + ObSSLProcessor() : ssl_ctx_map_(), ssl_ctx_lock_(obsys::WRITE_PRIORITY) {} ~ObSSLProcessor() {} int init(); - SSL* create_new_ssl(); + SSL* create_new_ssl(const common::ObString &cluster_name, + const common::ObString &tenant_name); void release_ssl(SSL* ssl, const bool can_shutdown_ssl); + void release_ssl_ctx(common::ObFixedLengthString &delete_info); void openssl_lock(int n); void openssl_unlock(int n); - bool is_client_ssl_supported(); - bool is_server_ssl_supported(); - int update_key(const common::ObString &source_type, + int update_key(const common::ObString &cluster_name, + const common::ObString &tenant_name, + const common::ObString &source_type, const common::ObString &ca, const common::ObString &public_key, const common::ObString &private_key); private: - int update_key_from_file(const common::ObString &ca, + int update_key_from_file(SSL_CTX *ssl_ctx, + const common::ObString &ca, const common::ObString &public_key, const common::ObString &private_key); - int update_key_from_dbmesh(const common::ObString &ca, + int update_key_from_string(SSL_CTX *ssl_ctx, + const common::ObString &ca, const common::ObString &public_key, const common::ObString &private_key); private: @@ -59,8 +64,9 @@ private: static void free_for_ssl(void *str); private: - SSL_CTX *ssl_ctx_; - bool ssl_inited_; + typedef common::hash::ObHashMap, SSL_CTX*> SSLCtxHashMap; + SSLCtxHashMap ssl_ctx_map_; + common::DRWLock ssl_ctx_lock_; }; extern ObSSLProcessor g_ssl_processor; @@ -69,5 +75,4 @@ extern ObSSLProcessor g_ssl_processor; } // end obproxy } // end oceanbase -#endif - +#endif \ No newline at end of file diff --git a/src/obproxy/iocore/net/ob_unix_net.cpp b/src/obproxy/iocore/net/ob_unix_net.cpp index 2fb3564d44ed7609c14825c3b5218ff3d8436455..64a7e42ed6bf04b544641b5e6756d28460f5870c 100644 --- a/src/obproxy/iocore/net/ob_unix_net.cpp +++ b/src/obproxy/iocore/net/ob_unix_net.cpp @@ -291,10 +291,10 @@ int ObInactivityCop::check_inactivity(int event, ObEvent *e) ret = OB_ERR_UNEXPECTED; PROXY_NET_LOG(WARN, "ObEvent is NULL", K(e), K(ret)); } else { - if (OB_UNLIKELY(!info.need_conn_accept_) && OB_LIKELY(info.graceful_exit_end_time_ > 0)) { + if (OB_UNLIKELY(!info.need_conn_accept_ || OB_SUCCESS != g_proxy_fatal_errcode) && OB_LIKELY(info.graceful_exit_end_time_ > 0)) { int64_t global_connections = 0; + NET_READ_GLOBAL_DYN_SUM(NET_GLOBAL_CLIENT_CONNECTIONS_CURRENTLY_OPEN, global_connections); if (0 == ethread->id_ && -1 == info.active_client_vc_count_) { - NET_READ_GLOBAL_DYN_SUM(NET_GLOBAL_CLIENT_CONNECTIONS_CURRENTLY_OPEN, global_connections); info.active_client_vc_count_ = global_connections; const int64_t remain_time = hrtime_to_sec(info.graceful_exit_end_time_ - now); PROXY_NET_LOG(INFO, "begin orderly close", "active_client_vc_count", global_connections, @@ -304,17 +304,23 @@ int ObInactivityCop::check_inactivity(int event, ObEvent *e) PROXY_NET_LOG(INFO, "receive signal", K(info.received_sig_)); } } - if (info.graceful_exit_end_time_ < now) {//it is time now - NET_READ_GLOBAL_DYN_SUM(NET_GLOBAL_CONNECTIONS_CURRENTLY_OPEN, global_connections); - int64_t thread_local_client_connections = 0; - NET_THREAD_READ_DYN_SUM(ethread, NET_CLIENT_CONNECTIONS_CURRENTLY_OPEN, thread_local_client_connections); - if (global_connections > 0) { + + if (global_connections > 0) { + if (info.graceful_exit_end_time_ >= info.graceful_exit_start_time_ && info.graceful_exit_end_time_ <= now) { + pthread_once(&g_exit_once, proxy_exit_once); + } else { + int64_t thread_local_client_connections = 0; + NET_THREAD_READ_DYN_SUM(ethread, NET_CLIENT_CONNECTIONS_CURRENTLY_OPEN, thread_local_client_connections); PROXY_NET_LOG(INFO, "wait for net_global_connections_currently_open_stat down to zero", K(global_connections), K(thread_local_client_connections), K(g_proxy_fatal_errcode)); - } else { - pthread_once(&g_exit_once, proxy_exit_once); } + } else { + pthread_once(&g_exit_once, proxy_exit_once); } + } else { + info.active_client_vc_count_ = 0; + info.graceful_exit_start_time_ = 0; + info.graceful_exit_end_time_ = 0; } // Copy the list and use pop() to catch any closes caused by callbacks. @@ -343,7 +349,9 @@ int ObInactivityCop::check_inactivity(int event, ObEvent *e) } else { if (vc->get_is_force_timeout() - || (info.graceful_exit_end_time_ > 0 && info.graceful_exit_end_time_ < now)) { + || (info.graceful_exit_end_time_ >= info.graceful_exit_start_time_ + && info.graceful_exit_end_time_ > 0 + && info.graceful_exit_end_time_ < now)) { vc->next_inactivity_timeout_at_ = now; // force the connection timeout } @@ -501,7 +509,7 @@ inline void ObNetHandler::process_enabled_list() while (NULL != (vc = rq.pop())) { vc->ep_->modify(EVENTIO_READ); vc->read_.in_enabled_list_ = false; - if ((vc->reenable_read_time_at_ > 0) && vc->reenable_read_time_at_ < get_hrtime()) { + if ((vc->reenable_read_time_at_ > 0) && vc->reenable_read_time_at_ > get_hrtime()) { vc->read_.in_enabled_list_ = true; read_enable_list_.push(vc); } else if ((vc->read_.enabled_ && vc->read_.triggered_) || vc->closed_) { diff --git a/src/obproxy/iocore/net/ob_unix_net_vconnection.cpp b/src/obproxy/iocore/net/ob_unix_net_vconnection.cpp index 1897fb218620b73ddffd740a615fcf85f3b0ec60..e2e7a833f6df89e0c2a1c6860431e5ca777c55be 100644 --- a/src/obproxy/iocore/net/ob_unix_net_vconnection.cpp +++ b/src/obproxy/iocore/net/ob_unix_net_vconnection.cpp @@ -39,6 +39,7 @@ using namespace oceanbase::common; using namespace oceanbase::obproxy::event; using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy::proxy; namespace oceanbase { @@ -47,7 +48,8 @@ namespace obproxy namespace net { -static const int64_t NET_MAX_IOV = 16; +// A block size is 8k, and 2 blocks can transmit 16k data, which meets the requirements +static const int64_t NET_MAX_IOV = 2; static inline ObNetState &get_net_state_by_vio(ObVIO &vio) { @@ -213,16 +215,13 @@ inline void ObUnixNetVConnection::net_activity() inline bool ObUnixNetVConnection::check_read_state() { bool ret = true; - if (!read_.enabled_ || ObVIO::READ != read_.vio_.op_) { - // if it is not enabled. - read_disable(); - ret = false; - } else if ((read_.vio_.ntodo()) <= 0) { - // if there is nothing to do, disable connection + if (OB_UNLIKELY(!read_.enabled_ + || ObVIO::READ != read_.vio_.op_ + || read_.vio_.ntodo() <= 0)) { read_disable(); ret = false; } else if (OB_ISNULL(read_.vio_.buffer_.writer())) { - PROXY_NET_LOG(ERROR, "fail to get writer from buf", "vc: ", this); + PROXY_NET_LOG(ERROR, "fail to get writer from buf", "vc", this); read_disable(); ret = false; } @@ -232,17 +231,14 @@ inline bool ObUnixNetVConnection::check_read_state() inline bool ObUnixNetVConnection::check_write_state() { bool ret = true; - if (!write_.enabled_ || ObVIO::WRITE != write_.vio_.op_) { - // If it is not enabled,add to WaitList. - write_disable(); - ret = false; - } else if ((write_.vio_.ntodo()) <= 0) { - // If there is nothing to do, disable + if (OB_UNLIKELY(!write_.enabled_ + || ObVIO::WRITE != write_.vio_.op_ + || write_.vio_.ntodo() <= 0)) { write_disable(); ret = false; } else if (OB_ISNULL(write_.vio_.buffer_.reader()) || OB_ISNULL(write_.vio_.buffer_.writer())) { - PROXY_NET_LOG(ERROR, "fail to get reader or writer from buf", K(this)); + PROXY_NET_LOG(ERROR, "fail to get reader or writer from buf", "vc: ", this); write_disable(); ret = false; } @@ -261,7 +257,7 @@ inline bool ObUnixNetVConnection::calculate_towrite_size(int64_t &towrite, bool towrite = ntodo; } - if (towrite <= 0 && towrite < ntodo && writer->write_avail() > 0) { + if (OB_UNLIKELY(towrite <= 0 && towrite < ntodo && writer->write_avail() > 0)) { if (EVENT_CONT == write_signal_and_update(VC_EVENT_WRITE_READY)) { if ((ntodo = write_.vio_.ntodo()) <= 0) { towrite = ntodo; @@ -396,7 +392,7 @@ inline bool ObUnixNetVConnection::handle_read_from_net_success( } else { if (EVENT_CONT != read_signal_and_update(VC_EVENT_READ_READY)) { // finish read, needn't reschedule - } else if (mutex != read_.vio_.mutex_.ptr_) { + } else if (OB_UNLIKELY(mutex != read_.vio_.mutex_.ptr_)) { // change of lock... don't look at shared variables! read_reschedule(); } else { @@ -408,7 +404,7 @@ inline bool ObUnixNetVConnection::handle_read_from_net_success( } inline bool ObUnixNetVConnection::handle_write_to_net_success( - ObEThread &thread, const ObProxyMutex *mutex, const int64_t total_write, const bool signalled) + ObEThread &thread, const ObProxyMutex *mutex, const int64_t total_write, const bool signalled) { bool is_done = true; ObProxyMutex *mutex_ = thread.mutex_; @@ -442,7 +438,7 @@ inline bool ObUnixNetVConnection::handle_write_to_net_success( } } else if (!signalled) { if (EVENT_CONT != write_signal_and_update(VC_EVENT_WRITE_READY)) { - } else if (mutex != write_.vio_.mutex_.ptr_) { + } else if (OB_UNLIKELY(mutex != write_.vio_.mutex_.ptr_)) { // change of lock... don't look at shared variables! write_reschedule(); } else { @@ -578,13 +574,12 @@ inline int ObUnixNetVConnection::read_from_net_internal( total_read += count; } } - NET_INCREMENT_DYN_STAT(NET_CALLS_TO_READ); } while (OB_SUCCESS == ret && (count == rattempted) && (total_read < toread) && (NULL != block)); } // if once read data form fd successfully, return OB_SUCCESS. - if (total_read > 0) { + if (OB_LIKELY(total_read > 0)) { ret = OB_SUCCESS; } @@ -597,7 +592,6 @@ inline int ObUnixNetVConnection::read_from_net_internal( inline void ObUnixNetVConnection::read_from_net(ObEThread &thread) { int ret = OB_SUCCESS; - ObProxyMutex *mutex_ = thread.mutex_; NET_INCREMENT_DYN_STAT(NET_CALLS_TO_READFROMNET); @@ -607,11 +601,10 @@ inline void ObUnixNetVConnection::read_from_net(ObEThread &thread) bool is_done = true; MUTEX_TRY_LOCK(lock, read_.vio_.mutex_, &thread); - - if (!lock.is_locked()) { + if (OB_UNLIKELY(!lock.is_locked())) { read_reschedule(); - } else if (!check_read_state()) { - PROXY_NET_LOG(DEBUG, "fail to check_read_state", K(this)); + } else if (OB_UNLIKELY(!check_read_state())) { + PROXY_NET_LOG(WARN, "fail to check_read_state", K(this)); } else { reenable_read_time_at_ = 0; ntodo = read_.vio_.ntodo(); @@ -631,12 +624,12 @@ inline void ObUnixNetVConnection::read_from_net(ObEThread &thread) // read data if (toread > 0) { int tmp_code = 0; - if (OB_FAIL(read_from_net_internal(writer, toread, total_read, tmp_code)) || (0 == total_read)) { + if (OB_FAIL(read_from_net_internal(writer, toread, total_read, tmp_code)) || OB_UNLIKELY((0 == total_read))) { is_done = handle_read_from_net_error(thread, total_read, ret, tmp_code); } else { ++read_.active_count_; // just check it - if (ntodo < 0) { + if (OB_UNLIKELY(ntodo < 0)) { PROXY_NET_LOG(ERROR, "occur fatal error", K(ntodo), K(this)); } is_done = handle_read_from_net_success(thread, lock.get_mutex(), total_read); @@ -662,7 +655,6 @@ inline int ObUnixNetVConnection::write_to_net_internal(ObIOBufferReader &reader, const int64_t towrite, int64_t &total_write, int &tmp_code) { int ret = OB_SUCCESS; - ObProxyMutex *mutex_ = &self_ethread().get_mutex(); int64_t count = 0; int64_t wattempted = 0; @@ -693,7 +685,7 @@ inline int ObUnixNetVConnection::write_to_net_internal(ObIOBufferReader &reader, } } - if (len > 0) { + if (OB_LIKELY(len > 0)) { do { niov = 0; wattempted = 0; @@ -737,14 +729,13 @@ inline int ObUnixNetVConnection::write_to_net_internal(ObIOBufferReader &reader, total_write += count; } } - NET_INCREMENT_DYN_STAT(NET_CALLS_TO_WRITE); } while (OB_SUCC(ret) && (count == wattempted) && (total_write < towrite) && (NULL != block)); } } // if once write data to fd successfully, return OB_SUCCESS - if (total_write > 0) { + if (OB_LIKELY(total_write > 0)) { ret = OB_SUCCESS; } @@ -758,27 +749,25 @@ inline void ObUnixNetVConnection::write_to_net(ObEThread &thread) int ret = OB_SUCCESS; ObProxyMutex *mutex_ = thread.mutex_; NET_INCREMENT_DYN_STAT(NET_CALLS_TO_WRITETONET); - int64_t total_write = 0; int64_t towrite = 0; bool is_done = true; bool signalled = false; MUTEX_TRY_LOCK(lock, write_.vio_.mutex_, &thread); - - if (!lock.is_locked() || lock.get_mutex() != write_.vio_.mutex_.ptr_) { + if (OB_UNLIKELY(!lock.is_locked() || lock.get_mutex() != write_.vio_.mutex_.ptr_)) { write_reschedule(); - } else if (!check_write_state()) { + } else if (OB_UNLIKELY(!check_write_state())) { PROXY_NET_LOG(DEBUG, "fail to check_write_state", K(this)); } else { ObIOBufferReader &reader = *(write_.vio_.buffer_.reader()); ObIOBufferReader *old_reader = write_.vio_.buffer_.reader(); is_done = calculate_towrite_size(towrite, signalled); - if (!is_done) { - if (towrite > 0) { + if (OB_LIKELY(!is_done)) { + if (OB_LIKELY(towrite > 0)) { int tmp_code = 0; - if (OB_FAIL(write_to_net_internal(reader, towrite, total_write, tmp_code)) || (0 == total_write)) { + if (OB_FAIL(write_to_net_internal(reader, towrite, total_write, tmp_code)) || OB_UNLIKELY((0 == total_write))) { is_done = handle_write_to_net_error(thread, total_write, ret, tmp_code); } else { is_done = handle_write_to_net_success(thread, lock.get_mutex(), total_write, signalled); @@ -795,7 +784,7 @@ inline void ObUnixNetVConnection::write_to_net(ObEThread &thread) * if not same, get read avail from new reader */ ObIOBufferReader *new_reader = write_.vio_.buffer_.reader(); - if (NULL != new_reader && new_reader != old_reader) { + if (OB_UNLIKELY(NULL != new_reader && new_reader != old_reader)) { read_avail = new_reader->read_avail(); } } @@ -803,7 +792,7 @@ inline void ObUnixNetVConnection::write_to_net(ObEThread &thread) if (0 == read_avail) { write_disable(); } else { - if (read_avail < 0) { + if (OB_UNLIKELY(read_avail < 0)) { PROXY_NET_LOG(ERROR, "write vio buffer's reader read_avail < 0", K(read_avail), K(towrite), K(total_write), K(signalled)); } @@ -1631,7 +1620,9 @@ int ObUnixNetVConnection::start_event(int event, ObEvent *e) return event_ret; } -int ObUnixNetVConnection::ssl_init(const SSLType ssl_type) +int ObUnixNetVConnection::ssl_init(const SSLType ssl_type, + const ObString &cluster_name, + const ObString &tenant_name) { int ret = OB_SUCCESS; @@ -1640,7 +1631,7 @@ int ObUnixNetVConnection::ssl_init(const SSLType ssl_type) PROXY_NET_LOG(WARN, "ssl is not null, init before", K(ret), K(ssl_connected_), K(using_ssl_), K(ssl_)); } else { - ssl_ = g_ssl_processor.create_new_ssl(); + ssl_ = g_ssl_processor.create_new_ssl(cluster_name, tenant_name); if (NULL == ssl_) { ret = OB_SSL_ERROR; PROXY_NET_LOG(WARN, "ssl new failed", K(ret)); diff --git a/src/obproxy/iocore/net/ob_unix_net_vconnection.h b/src/obproxy/iocore/net/ob_unix_net_vconnection.h index b717f02c56de06cef64f5585067bec0b4efe64ec..33459f4753978db5c21ce7b56e9eb3ef399370f2 100644 --- a/src/obproxy/iocore/net/ob_unix_net_vconnection.h +++ b/src/obproxy/iocore/net/ob_unix_net_vconnection.h @@ -36,6 +36,7 @@ #include "iocore/net/ob_net_vconnection.h" #include "iocore/net/ob_net_state.h" #include "iocore/net/ob_connection.h" +#include "lib/string/ob_string.h" namespace oceanbase { @@ -233,7 +234,7 @@ public: IO_WRITE, }; - int ssl_init(const SSLType ssL_type); + int ssl_init(const SSLType ssL_type, const common::ObString &cluster_name, const common::ObString &tenant_name); inline bool using_ssl() const { return using_ssl_; } inline bool ssl_connected() const { return ssl_connected_; } void do_ssl_io(event::ObEThread &thread); diff --git a/src/obproxy/ob_proxy.cpp b/src/obproxy/ob_proxy.cpp index 86e7df1e63eab2f0f18a8a293b8039683e9e5894..36003e15a3b1bd4ef7195ce66f7652fdeae51ef8 100644 --- a/src/obproxy/ob_proxy.cpp +++ b/src/obproxy/ob_proxy.cpp @@ -33,6 +33,7 @@ #include "obutils/ob_log_file_processor.h" #include "obutils/ob_proxy_config_utils.h" #include "obutils/ob_proxy_table_processor_utils.h" +#include "obutils/ob_hot_upgrade_processor.h" #include "obutils/ob_task_flow_controller.h" #include "obutils/ob_metadb_create_cont.h" #include "obutils/ob_tenant_stat_manager.h" @@ -72,6 +73,8 @@ #include "cmd/ob_show_global_session_handler.h" #include "cmd/ob_kill_global_session_handler.h" #include "obutils/ob_session_pool_processor.h" +#include "obutils/ob_config_processor.h" +#include "iocore/net/ob_ssl_processor.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -121,6 +124,7 @@ ObProxy::ObProxy() config_(&get_global_proxy_config()), reload_config_(this), proxy_table_processor_(get_global_proxy_table_processor()), + hot_upgrade_processor_(get_global_hot_upgrade_processor()), mysql_config_params_(NULL), proxy_opts_(NULL), proxy_version_(NULL) { } @@ -165,8 +169,12 @@ int ObProxy::init(ObProxyOptions &opts, ObAppVersionInfo &proxy_version) LOG_ERROR("fail to init sql table cache", K(ret)); } else if (OB_FAIL(table_processor.init(&table_cache))) { LOG_ERROR("fail to init table processor", K(ret)); + } else if (OB_FAIL(g_ssl_processor.init())) { + LOG_ERROR("fail to init ssl processor", K(ret)); } else if (OB_FAIL(init_config())) { LOG_ERROR("fail to init config", K(ret)); + } else if (get_global_config_processor().init()) { + LOG_ERROR("fail to init config processor", K(ret)); } else if (OB_FAIL(config_->enable_sharding && dbconfig_processor.init(config_->grpc_client_num, ObProxyMain::get_instance()->get_startup_time()))) { LOG_ERROR("fail to init dbconfig processor", K(ret)); @@ -319,6 +327,8 @@ int ObProxy::start() LOG_WARN("fail to alloc and schedule cache cleaner", K(ret)); } else if (config_->is_metadb_used() && OB_FAIL(proxy_table_processor_.start_check_table_task())) { LOG_WARN("fail to start check table check", K(ret)); + } else if (OB_FAIL(hot_upgrade_processor_.start_hot_upgrade_task())) { + LOG_WARN("fail to start hot upgrade task", K(ret)); } else if (OB_FAIL(log_file_processor_->start_cleanup_log_file())) { LOG_WARN("fail to start cleanup log file task", K(ret)); } else if (config_->with_config_server_ && OB_FAIL(cs_processor_->start_refresh_task())) { @@ -342,8 +352,8 @@ int ObProxy::start() LOG_WARN("fail to start_session_pool_task", K(ret)); } else { // if fail to init prometheus, do not stop the startup of obproxy - if (g_ob_prometheus_processor.start_prometheus_task()) { - LOG_WARN("fail to start prometheus task"); + if (g_ob_prometheus_processor.start_prometheus()) { + LOG_WARN("fail to start prometheus"); } mysql_config_params_ = NULL; @@ -358,6 +368,15 @@ int ObProxy::start() LOG_INFO("obproxy start up successful and can provide normal service", "cost time(us)", ObTimeUtility::current_time() - start_time_, K_(is_service_started)); + ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); + if (info.is_inherited_) { + pid_t parent_pid= getppid(); + if (parent_pid != 1) { + LOG_INFO("obproxy child will send SIGTERM to parent", K(parent_pid)); + kill(parent_pid, SIGUSR1); + } + info.is_parent_ = true; + } this_ethread()->execute(); } } @@ -446,14 +465,6 @@ int ObProxy::init_user_specified_config() } } - //6. update from_local_restart - if (OB_SUCC(ret)) { - if (config_->is_local_restart() && !info.is_parent()) { - config_->reset_local_cmd(); - info.cmd_ = HUC_LOCAL_RESTART; - LOG_INFO("proxy was restarted", K(info)); - } - } return ret; } @@ -571,6 +582,9 @@ int ObProxy::init_conn_pool(const bool load_local_config_succ) } else if (config_->is_metadb_used() && OB_FAIL(proxy_table_processor_.init(meta_client_proxy_, *proxy_version_, reload_config_))) { LOG_ERROR("fail to init proxy processor", K(ret)); + } else if (!config_->is_metadb_used() + && OB_FAIL(hot_upgrade_processor_.init(meta_client_proxy_))) { + LOG_ERROR("fail to init hot upgrade processor", K(ret)); } else { // if we need init tables on metadb, or load local config failed, we need do force start refresh table is_force_remote_start_ = (proxy_opts_->execute_cfg_sql_ || !load_local_config_succ); @@ -912,6 +926,7 @@ int ObProxy::do_reload_config(obutils::ObProxyConfig &config) get_pl_task_flow_controller().set_normal_threshold(config_->normal_pl_update_threshold); get_pl_task_flow_controller().set_limited_threshold(config_->limited_pl_update_threshold); } + return ret; } diff --git a/src/obproxy/ob_proxy.h b/src/obproxy/ob_proxy.h index 379f527292d323218d0b7aea57ef9129571b8e85..7b92747be842626e3cbd478818cace876cbf61af 100644 --- a/src/obproxy/ob_proxy.h +++ b/src/obproxy/ob_proxy.h @@ -137,6 +137,7 @@ private: obutils::ObProxyConfig *config_; obutils::ObProxyReloadConfig reload_config_; obutils::ObProxyTableProcessor &proxy_table_processor_; + obutils::ObHotUpgradeProcessor &hot_upgrade_processor_; // only used at start up proxy::ObMysqlConfigParams *mysql_config_params_; diff --git a/src/obproxy/ob_proxy_main.cpp b/src/obproxy/ob_proxy_main.cpp index 811ffb4052c2e35599792151229d30eb564a1f35..ceb87398cea9c864ef84c98032d3572ddfdce6ff 100644 --- a/src/obproxy/ob_proxy_main.cpp +++ b/src/obproxy/ob_proxy_main.cpp @@ -505,7 +505,6 @@ int ObProxyMain::handle_inherited_sockets(const int argc, char *const argv[]) // and set it HU_STATE_WAIT_HU_CMD state info.update_state(HU_STATE_WAIT_HU_CMD); info.is_parent_ = true; - info.update_both_status(HU_STATUS_NONE, HU_STATUS_NONE); } else if (info.is_inherited_) { ret = OB_ERR_UNEXPECTED; MPRINT("hot upgrade info is_inherited can't be true, ret=%d", ret); @@ -514,9 +513,8 @@ int ObProxyMain::handle_inherited_sockets(const int argc, char *const argv[]) // we will set it HU_STATE_WAIT_CR_CMD state and HU_STATUS_NEW_PROXY_CREATED_SUCC status info.is_inherited_ = true; info.fd_ = atoi(inherited); - info.update_state(HU_STATE_WAIT_CR_CMD); + info.update_state(HU_STATE_WAIT_HU_CMD); info.is_parent_ = false; - info.update_both_status(HU_STATUS_CREATE_NEW_PROXY_SUCC, HU_STATUS_NEW_PROXY_CREATED_SUCC); if (OB_FAIL(unsetenv(OBPROXY_INHERITED_FD))) { MPRINT("fail to unsetenv OBPROXY_INHERITED_FD, ret=%d", ret); } @@ -576,6 +574,10 @@ int ObProxyMain::init_signal() LOG_WARN("fail to add_sig_direct_catched", K(ret)); } else if (OB_FAIL(add_sig_direct_catched(action, SIGTERM))) { LOG_WARN("fail to add_sig_direct_catched", K(ret)); + } else if (OB_FAIL(add_sig_direct_catched(action, SIGUSR1))) { + LOG_WARN("fail to add_sig_direct_catched", K(ret)); + } else if (OB_FAIL(add_sig_direct_catched(action, 43))) { + LOG_WARN("fail to add_sig_direct_catched", K(ret)); // when a process terminates, the SIGHUP signal can be catch by its sub process } else if (OB_FAIL(prctl(PR_SET_PDEATHSIG, SIGHUP))) { @@ -724,7 +726,6 @@ int ObProxyMain::do_detect_sig() if (OB_INVALID_INDEX != sig) { info.received_sig_ = OB_INVALID_INDEX; - LOG_INFO("receive signal", K(sig)); switch (sig) { case SIGCHLD: { pid_t pid = OB_INVALID_INDEX; @@ -736,8 +737,13 @@ int ObProxyMain::do_detect_sig() while((pid = waitpid(-1, &stat, WNOHANG)) > 0) { LOG_INFO("sub process exit", K(info), K(pid), "status", stat, KERRMSGS); if (info.sub_pid_ == pid) { + info.reset_sub_pid(); // after sub was exited, we need passing this status - info.update_sub_status(HU_STATUS_EXITED); + if (OB_LIKELY(common::OB_SUCCESS == lib::mutex_acquire(&info.hot_upgrade_mutex_))) { + info.update_sub_status(HU_STATUS_EXITED); + info.parent_hot_upgrade_flag_ = false; + lib::mutex_release(&info.hot_upgrade_mutex_); + } } else { LOG_WARN("sub process exit, but recv it late"); } @@ -794,6 +800,30 @@ extern "C" { void ObProxyMain::sig_direct_handler(const int sig) { switch (sig) { + case SIGUSR1: { + ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); + if (OB_LIKELY(common::OB_SUCCESS == lib::mutex_acquire(&info.hot_upgrade_mutex_))) { + // If an exit signal has been sent to the child process, + // the parent process ignores the signal sent by the child process + if (OB_LIKELY(!info.parent_hot_upgrade_flag_)) { +#ifdef TEST_COVER + LOG_INFO("gcov flush now"); + __gcov_flush(); +#endif + info.received_sig_ = sig; + LOG_INFO("recv SIGUSR1 signal, will graceful exit", K(info)); + if (info.need_conn_accept_) { + info.disable_net_accept(); // disable accecpt new connection + } + info.graceful_exit_start_time_ = get_hrtime_internal(); + info.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_exit_timeout) + + info.graceful_exit_start_time_; + info.parent_hot_upgrade_flag_ = true; + } + lib::mutex_release(&info.hot_upgrade_mutex_); + } + break; + } case SIGTERM: case SIGINT: { ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); @@ -801,14 +831,14 @@ void ObProxyMain::sig_direct_handler(const int sig) LOG_INFO("gcov flush now"); __gcov_flush(); #endif + info.received_sig_ = sig; if (info.need_conn_accept_) { - info.received_sig_ = sig; info.disable_net_accept(); // disable accecpt new connection - info.graceful_exit_start_time_ = get_hrtime_internal(); - info.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().delay_exit_time) - + info.graceful_exit_start_time_; - g_proxy_fatal_errcode = OB_GOT_SIGNAL_ABORTING; } + info.graceful_exit_start_time_ = get_hrtime_internal(); + info.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().delay_exit_time) + + info.graceful_exit_start_time_; + g_proxy_fatal_errcode = OB_GOT_SIGNAL_ABORTING; break; } default: { diff --git a/src/obproxy/obutils/Makemodule.am b/src/obproxy/obutils/Makemodule.am index 8dd168908745460d934a7b33a488af4814cd8421..8de2e2a970d71ab195595166856ea59d17b44fb4 100644 --- a/src/obproxy/obutils/Makemodule.am +++ b/src/obproxy/obutils/Makemodule.am @@ -70,4 +70,6 @@ obproxy/obutils/ob_cached_variables.cpp\ obproxy/obutils/ob_task_flow_controller.h\ obproxy/obutils/ob_task_flow_controller.cpp\ obproxy/obutils/ob_proxy_config_processor.h\ -obproxy/obutils/ob_proxy_config_processor.cpp +obproxy/obutils/ob_proxy_config_processor.cpp\ +obproxy/obutils/ob_config_processor.h\ +obproxy/obutils/ob_config_processor.cpp diff --git a/src/obproxy/obutils/ob_async_common_task.cpp b/src/obproxy/obutils/ob_async_common_task.cpp index 87100c1d08ab4976d60fa886acc25c5b0d46c5fd..501541f0a945aca4a1460b8a98ad6554fdf85fc6 100644 --- a/src/obproxy/obutils/ob_async_common_task.cpp +++ b/src/obproxy/obutils/ob_async_common_task.cpp @@ -305,7 +305,7 @@ inline int ObAsyncCommonTask::handle_repeat_task() inline void ObAsyncCommonTask::check_stop_repeat_task() { const ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); - if (OB_UNLIKELY(!info.need_conn_accept_)) { + if (OB_UNLIKELY(!info.need_conn_accept_ && OB_SUCCESS != g_proxy_fatal_errcode)) { // no need do repeat task again is_stop_ = true;; // for schedule_in task int ret = OB_SUCCESS; diff --git a/src/obproxy/obutils/ob_config_processor.cpp b/src/obproxy/obutils/ob_config_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ede75b11827af9be05a5f09cc486a1de3c0011c8 --- /dev/null +++ b/src/obproxy/obutils/ob_config_processor.cpp @@ -0,0 +1,676 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include "obutils/ob_config_processor.h" +#include "opsql/parser/ob_proxy_parser.h" +#include "iocore/eventsystem/ob_ethread.h" +#include "obutils/ob_proxy_sql_parser.h" +#include "cmd/ob_config_v2_handler.h" +#include "omt/ob_white_list_table_processor.h" +#include "omt/ob_resource_unit_table_processor.h" +#include "omt/ob_ssl_config_table_processor.h" + +using namespace obsys; +using namespace oceanbase::common; +using namespace oceanbase::obproxy::opsql; +using namespace oceanbase::obproxy::event; +using namespace oceanbase::obproxy::omt; +using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy; + +namespace oceanbase +{ +namespace obproxy +{ +namespace obutils +{ + +#define MAX_INIT_SQL_LEN 1024 +static const char* sqlite3_db_name = "etc/proxyconfig.db"; +static const char* all_table_version_table_name = "all_table_version"; +static const char* white_list_table_name = "white_list"; +static const char* resource_unit_table_name = "resource_unit"; +static const char *ssl_config_table_name = "ssl_config"; +static const char *table_name_array[] = {white_list_table_name, resource_unit_table_name, ssl_config_table_name}; + +static const char* create_all_table_version_table = + "create table if not exists " + "all_table_version(gmt_created date default (datetime('now', 'localtime')), " + "gmt_modified date, table_name varchar(1000), " + "version int default 0, primary key(table_name))"; +static const char* create_white_list_table = + "create table if not exists " + "white_list(gmt_created date default (datetime('now', 'localtime')), gmt_modified date, " + "cluster_name varchar(256), tenant_name varchar(256), name varchar(256), " + "value text, primary key(cluster_name, tenant_name))"; +static const char* create_resource_unit_table = + "create table if not exists " + "resource_unit(gmt_created date default (datetime('now', 'localtime')), " + "gmt_modified date, cluster_name varchar(256), tenant_name varchar(256), " + "name varchar(256), value text, primary key(cluster_name, tenant_name, name))"; +static const char* create_ssl_config_table = + "create table if not exists " + "ssl_config(gmt_created date default (datetime('now', 'localtime')), gmt_modified date, " + "cluster_name varchar(256), tenant_name varchar(256), name varchar(256), value text, " + "primary key(cluster_name, tenant_name, name))"; + +ObConfigProcessor &get_global_config_processor() +{ + static ObConfigProcessor g_config_processor; + return g_config_processor; +} + +ObFnParams::ObFnParams() : config_type_(OBPROXY_CONFIG_INVALID), + stmt_type_(OBPROXY_T_INVALID), + table_name_(), fields_(NULL) +{} + +ObFnParams::~ObFnParams() {} + +ObCloudFnParams::ObCloudFnParams() : ObFnParams(), cluster_name_(), tenant_name_() {} + +ObCloudFnParams::~ObCloudFnParams() {} + +ObConfigProcessor::ObConfigProcessor() : proxy_config_db_(NULL), table_handler_map_() +{} + +ObConfigProcessor::~ObConfigProcessor() +{ +} + +int ObConfigProcessor::init() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(table_handler_map_.create(32, ObModIds::OB_HASH_BUCKET))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (SQLITE_OK != sqlite3_open_v2( + sqlite3_db_name, &proxy_config_db_, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sqlite3 open failed", K(ret), "err_msg", sqlite3_errmsg(proxy_config_db_)); + } else if (OB_FAIL(check_and_create_table())) { + LOG_WARN("check create table failed", K(ret)); + } else { + // TODO: Compatibility Check + if (OB_FAIL(get_global_white_list_table_processor().init())) { + LOG_WARN("white list table processor init failed", K(ret)); + } else if (OB_FAIL(get_global_resource_unit_table_processor().init())) { + LOG_WARN("resource unit table processor init failed", K(ret)); + } else if (OB_FAIL(get_global_ssl_config_table_processor().init())) { + LOG_WARN("ssl config table processor init failed", K(ret)); + } else if (OB_FAIL(init_config_from_disk())) { + LOG_WARN("init config from disk failed", K(ret)); + } + } + + // Do compatibility processing, currently it is related to ssl, + // if the -o startup parameter specifies that synchronization is required here + if (OB_SUCC(ret)) { + if (OB_FAIL(store_cloud_config(ssl_config_table_name, "*", "*", "enable_client_ssl", + get_global_proxy_config().enable_client_ssl.str()))) { + LOG_WARN("store client ssl failed", K(ret)); + } else if (OB_FAIL(store_cloud_config(ssl_config_table_name, "*", "*", "enable_server_ssl", + get_global_proxy_config().enable_server_ssl.str()))) { + LOG_WARN("store server ssl failed", K(ret)); + } + } + return ret; +} + +int ObConfigProcessor::init_callback(void *data, int argc, char **argv, char **column_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == data || NULL == argv || NULL == column_name)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("argument is null unexpected", K(ret)); + } else { + ObConfigHandler *handler = reinterpret_cast(data); + ObCloudFnParams params; + params.config_type_ = OBPROXY_CONFIG_CLOUD; + params.stmt_type_ = OBPROXY_T_REPLACE; + SqlFieldResult field_result; + for (int64_t i = 0; OB_SUCC(ret) && i < argc; i++) { + if (NULL == argv[i]) { + continue; + } else { + SqlField sql_field; + sql_field.column_name_.set_string(column_name[i], static_cast(strlen(column_name[i]))); + sql_field.value_type_ = TOKEN_STR_VAL; + sql_field.column_value_.set_value(static_cast(strlen(argv[i])), argv[i]); + if (OB_FAIL(field_result.fields_.push_back(sql_field))) { + LOG_WARN("push back failed", K(ret)); + } else { + field_result.field_num_++; + } + } + } + + if (OB_SUCC(ret)) { + params.fields_ = &field_result; + const char* cluster_name_str = "cluster_name"; + const char* tenant_name_str = "tenant_name"; + for (int64_t i = 0; i < field_result.field_num_; i++) { + SqlField &field = field_result.fields_.at(i); + if (field.column_name_.string_ == cluster_name_str) { + params.cluster_name_ = field.column_value_.config_string_; + } else if (field.column_name_.string_ == tenant_name_str) { + params.tenant_name_ = field.column_value_.config_string_; + } + } + + bool is_success = true; + if (OB_FAIL(handler->execute_func_(¶ms))) { + is_success = false; + LOG_WARN("execute fn failed", K(ret)); + } + + if (OB_FAIL(handler->commit_func_(is_success))) { + LOG_WARN("commit func failed", K(ret), K(is_success)); + } + } + } + + return ret; +} + +int ObConfigProcessor::init_config_from_disk() +{ + int ret = OB_SUCCESS; + + const char* select_sql = "select * from %.*s"; + for(int64_t i = 0; OB_SUCC(ret) && i < (sizeof(table_name_array) / sizeof(const char*)); i++) { + const char *table_name = table_name_array[i]; + ObConfigHandler handler; + char sql_buf[MAX_INIT_SQL_LEN]; + char *err_msg = NULL; + int64_t len = snprintf(sql_buf, MAX_INIT_SQL_LEN, select_sql, strlen(table_name), table_name); + if (OB_UNLIKELY(len <= 0 || len >= MAX_INIT_SQL_LEN)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("generate sql faield", K(ret), K(len)); + } else if (OB_FAIL(table_handler_map_.get_refactored(table_name, handler))) { + LOG_WARN("get handler failed", K(ret), "table_name", table_name); + } else if (SQLITE_OK != sqlite3_exec(proxy_config_db_, sql_buf, ObConfigProcessor::init_callback, &handler, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec sql failed", K(ret), "err_msg", err_msg); + sqlite3_free(err_msg); + } + } + return ret; +} + +int ObConfigProcessor::execute(ObString &sql, const ObProxyBasicStmtType stmt_type, obproxy::ObConfigV2Handler *v2_handler) +{ + int ret = OB_SUCCESS; + LOG_DEBUG("begin to execute", K(sql), K(stmt_type)); + DRWLock::WRLockGuard guard(config_lock_); + if (OB_UNLIKELY(sql.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid sql", K(ret), K(sql.length())); + } else if (stmt_type == OBPROXY_T_SELECT) { + if (OB_FAIL(handle_select_stmt(sql, v2_handler))) { + LOG_WARN("handle select stmt failed", K(ret)); + } + } else { + ObArenaAllocator allocator; + ParseResult parse_result; + const int64_t EXTRA_NUM = 2; + ObString parse_sql(sql.length() + EXTRA_NUM, sql.ptr()); + ObProxyParser obproxy_parser(allocator, NORMAL_PARSE_MODE); + if (OB_FAIL(obproxy_parser.obparse(parse_sql, parse_result))) { + LOG_WARN("fail to parse sql", K(sql), K(ret)); + } else if (OB_FAIL(handle_dml_stmt(sql, parse_result))) { + LOG_WARN("handle stmt failed", K(ret), K(sql)); + } + } + + return ret; +} + +int ObConfigProcessor::check_and_create_table() +{ + int ret = OB_SUCCESS; + char *err_msg = NULL; + if (SQLITE_OK != sqlite3_exec(proxy_config_db_, create_all_table_version_table, NULL, 0, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec create all table version table sql failed", K(ret), "err_msg", err_msg); + } else if (SQLITE_OK != sqlite3_exec(proxy_config_db_, create_white_list_table, NULL, 0, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec create white list table sql failed", K(ret), "err_msg", err_msg); + } else if (SQLITE_OK != sqlite3_exec(proxy_config_db_, create_resource_unit_table, NULL, 0, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec create resource unit table sql failed", K(ret), "err_msg", err_msg); + } else if (SQLITE_OK != sqlite3_exec(proxy_config_db_, create_ssl_config_table, NULL, 0, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("exec create ssl config table sql failed", K(ret), "err_msg", err_msg); + } + + if (NULL != err_msg) { + sqlite3_free(err_msg); + } + + return ret; +} + +int ObConfigProcessor::resolve_insert(const ParseNode* node, ResolveContext &ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || T_INSERT != node->type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is null or wrong type unexpected", K(ret)); + } else { + if (node->children_[1] != NULL) { + if (node->children_[1]->type_ == T_INSERT) { + ctx.stmt_type_ = OBPROXY_T_INSERT; + } else if (node->children_[1]->type_ == T_REPLACE) { + ctx.stmt_type_ = OBPROXY_T_REPLACE; + } + } + + if (node->children_[0] != NULL && node->children_[0]->type_ == T_SINGLE_TABLE_INSERT) { + ParseNode* tmp_node = node->children_[0]; + if (tmp_node->children_[0] != NULL && tmp_node->children_[0]->type_ == T_INSERT_INTO_CLAUSE) { + ParseNode* tmp_node1 = tmp_node->children_[0]; + if (tmp_node1->children_[0] != NULL && tmp_node1->children_[0]->type_ == T_ORG) { + ParseNode* tmp_node2 = tmp_node1->children_[0]; + if (tmp_node2->children_[0] != NULL && tmp_node2->children_[0]->type_ == T_RELATION_FACTOR) { + ParseNode* tmp_node3 = tmp_node2->children_[0]; + ctx.table_name_.assign_ptr(tmp_node3->str_value_, static_cast(tmp_node3->str_len_)); + if (OB_FAIL(table_handler_map_.get_refactored(ctx.table_name_, ctx.handler_))) { + LOG_DEBUG("fail to get handler by table name", K(ret), K(ctx.table_name_)); + ret = OB_SUCCESS; + } else { + ctx.config_type_ = ctx.handler_.config_type_; + } + } + } + + if (!ctx.table_name_.empty() && tmp_node1->children_[1] != NULL && tmp_node1->children_[1]->type_ == T_COLUMN_LIST) { + ParseNode *tmp_node2 = tmp_node1->children_[1]; + for (int64_t i = 0; OB_SUCC(ret) && i < tmp_node2->num_child_; i++) { + if (NULL != tmp_node2->children_[i]) { + ObString column_name; + column_name.assign_ptr(tmp_node2->children_[i]->str_value_, static_cast(tmp_node2->children_[i]->str_len_)); + if (OB_FAIL(ctx.column_name_array_.push_back(column_name))) { + LOG_WARN("column name push back failed", K(ret), K(column_name)); + } + } + } + } + } + + if (!ctx.table_name_.empty() && tmp_node->children_[1] != NULL && tmp_node->children_[1]->type_ == T_VALUE_LIST) { + ParseNode* tmp_node1 = tmp_node->children_[1]; + if (tmp_node1->children_[0] != NULL && tmp_node1->children_[0]->type_ == T_VALUE_VECTOR) { + ParseNode* tmp_node2 = tmp_node1->children_[0]; + for (int64_t i = 0; OB_SUCC(ret) && ctx.column_name_array_.count() == tmp_node2->num_child_ && i < tmp_node2->num_child_; i++) { + if (tmp_node2->children_[i] != NULL) { + if (tmp_node2->children_[i]->type_ == T_INT) { + SqlField field; + field.column_name_.set(ctx.column_name_array_.at(i)); + field.value_type_ = TOKEN_INT_VAL; + field.column_int_value_ = tmp_node2->children_[i]->value_; + ctx.sql_field_.fields_.push_back(field); + ctx.sql_field_.field_num_++; + } else if (tmp_node2->children_[i]->type_ == T_VARCHAR) { + SqlField field; + field.column_name_.set(ctx.column_name_array_.at(i)); + field.value_type_ = TOKEN_STR_VAL; + ObString value; + if (tmp_node2->children_[i]->str_len_ - 2 <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected len", K(ret)); + } else { + value.assign_ptr(tmp_node2->children_[i]->str_value_ + 1, static_cast(tmp_node2->children_[i]->str_len_ - 2)); + field.column_value_.set_value(value); + ctx.sql_field_.fields_.push_back(field); + ctx.sql_field_.field_num_++; + } + } + } + } + } + } + } + } + + return ret; +} + +int ObConfigProcessor::resolve_delete(const ParseNode* node, ResolveContext &ctx) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || T_DELETE != node->type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is null or wrong type", K(ret)); + } else { + ctx.stmt_type_ = OBPROXY_T_DELETE; + if (node->children_[0] != NULL && node->children_[0]->type_ == T_DELETE_TABLE_NODE) { + ParseNode *tmp_node = node->children_[0]; + if (tmp_node->children_[1] != NULL && tmp_node->children_[1]->type_ == T_TABLE_REFERENCES) { + ParseNode *tmp_node1 = tmp_node->children_[1]; + if (tmp_node1->children_[0] != NULL && tmp_node1->children_[0]->type_ == T_ORG) { + ParseNode *tmp_node2 = tmp_node1->children_[0]; + if (tmp_node2->children_[0] != NULL && tmp_node2->children_[0]->type_ == T_RELATION_FACTOR) { + ctx.table_name_.assign_ptr(tmp_node2->children_[0]->str_value_, static_cast(tmp_node2->children_[0]->str_len_)); + if (OB_FAIL(table_handler_map_.get_refactored(ctx.table_name_, ctx.handler_))) { + LOG_DEBUG("fail to get handler by table name", K(ret), K(ctx.table_name_)); + ret = OB_SUCCESS; + } else { + ctx.config_type_ = ctx.handler_.config_type_; + } + } + } + } + } + + if (!ctx.table_name_.empty() && node->children_[1] != NULL && node->children_[1]->type_ == T_WHERE_CLAUSE) { + ParseNode* tmp_node = node->children_[1]; + if (tmp_node->children_[0] != NULL && tmp_node->children_[0]->type_ == T_OP_AND) { + ParseNode* tmp_node1 = tmp_node->children_[0]; + for (int64_t i = 0; OB_SUCC(ret) && i < tmp_node1->num_child_; i++) { + if (tmp_node1->children_[i] != NULL && tmp_node1->children_[i]->type_ == T_OP_EQ) { + ParseNode* tmp_node2 = tmp_node1->children_[i]; + SqlField field; + if (tmp_node2->children_[0] != NULL && tmp_node2->children_[0]->type_ == T_COLUMN_REF) { + ObString column_name; + column_name.assign_ptr(tmp_node2->children_[0]->str_value_, static_cast(tmp_node2->children_[0]->str_len_)); + field.column_name_.set(column_name); + LOG_DEBUG("parse column name", K(column_name), K(i)); + } + + if (tmp_node2->children_[1] != NULL) { + if (tmp_node2->children_[1]->type_ == T_INT) { + field.value_type_ = TOKEN_INT_VAL; + field.column_int_value_ = tmp_node2->children_[1]->value_; + ctx.sql_field_.fields_.push_back(field); + ctx.sql_field_.field_num_++; + } else if (tmp_node2->children_[1]->type_ == T_VARCHAR) { + if (tmp_node2->children_[1]->str_len_ - 2 <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected len", K(ret)); + } else { + ObString value; + value.assign_ptr(tmp_node2->children_[1]->str_value_ + 1, static_cast(tmp_node2->children_[1]->str_len_ - 2)); + field.value_type_ = TOKEN_STR_VAL; + field.column_value_.set_value(value); + ctx.sql_field_.fields_.push_back(field); + ctx.sql_field_.field_num_++; + LOG_DEBUG("parse column value", K(value), K(i)); + } + } + } + } + } + } + } + } + return ret; +} + +int ObConfigProcessor::handle_dml_stmt(ObString &sql, ParseResult& parse_result) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + const int64_t buf_len = 64 * 1024; + int64_t pos = 0; + char tree_str_buf[buf_len]; + ObSqlParseResult::ob_parse_resul_to_string(parse_result, sql, tree_str_buf, buf_len, pos); + ObString tree_str(pos, tree_str_buf); + LOG_DEBUG("result_tree_ is \n", K(tree_str)); + } + + ParseNode* node = parse_result.result_tree_; + if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is unexpected null", K(ret)); + } else if (T_STMT_LIST != node->type_ || node->num_child_ < 1) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", "node_type", get_type_name(node->type_), K(node->num_child_), K(ret)); + } else if (OB_ISNULL(node = node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_DEBUG("unexpected null child", K(ret)); + } else { + // At present, our grammar parsing is relatively fixed. There can only be two equal conditions + // after the delete statement where, otherwise an error will be reported + // The new requirement is changed to the implementation of ob_proxy_stmt + ResolveContext ctx; + switch(node->type_) { + case T_INSERT: + { + if (OB_FAIL(resolve_insert(node, ctx))) { + LOG_WARN("resove insert failed", K(ret)); + } + break; + } + case T_DELETE: + { + if (OB_FAIL(resolve_delete(node, ctx))) { + LOG_WARN("resolve delete failed", K(ret)); + } + break; + } + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unsupported type", K(node->type_)); + } + + ObCloudFnParams params; + bool is_success = true; + bool is_execute = false; + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret) && OBPROXY_CONFIG_CLOUD == ctx.config_type_ && !ctx.table_name_.empty()) { + params.config_type_ = ctx.config_type_; + params.stmt_type_ = ctx.stmt_type_; + params.table_name_ = ctx.table_name_; + params.fields_ = &ctx.sql_field_; + const char* cluster_name_str = "cluster_name"; + const char* tenant_name_str = "tenant_name"; + for (int64_t i = 0; i < ctx.sql_field_.field_num_; i++) { + SqlField &field = ctx.sql_field_.fields_.at(i); + if (field.column_name_.string_ == cluster_name_str) { + params.cluster_name_ = field.column_value_.config_string_; + } else if (field.column_name_.string_ == tenant_name_str) { + params.tenant_name_ = field.column_value_.config_string_; + } + } + + if (OB_FAIL(ctx.handler_.execute_func_(¶ms))) { + is_execute = true; + LOG_WARN("execute fn failed", K(ret)); + } else { + is_execute = true; + } + tmp_ret = ret; + } + + if (OB_SUCC(ret)) { + char *err_msg = NULL; + char *sql_buf = (char*)ob_malloc(sql.length() + 1); + if (NULL == sql_buf) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(sql.length())); + } else { + memcpy(sql_buf, sql.ptr(), sql.length()); + sql_buf[sql.length()] = '\0'; + if (SQLITE_OK != sqlite3_exec(proxy_config_db_, sql_buf, NULL, 0, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sqlite3 exec failed", K(sql), "err_msg", err_msg); + sqlite3_free(err_msg); + } + ob_free(sql_buf); + } + tmp_ret = ret; + } + + if (OB_FAIL(ret)) { + is_success = false; + } + + if (is_execute) { + if (OB_FAIL(ctx.handler_.commit_func_(is_success))) { + LOG_WARN("commit failed", K(ret), K(is_success)); + } + } + ret = (OB_SUCCESS != tmp_ret ? tmp_ret : ret); + } + + return ret; +} + +int ObConfigProcessor::sqlite3_callback(void *data, int argc, char **argv, char **column_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == data || NULL == argv || NULL == column_name)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("argument is null unexpected", K(ret)); + } else { + obproxy::ObConfigV2Handler *v2_handler = reinterpret_cast(data); + bool need_add_column_name = (v2_handler->sqlite3_column_name_.count() == 0); + ObSEArray value_array; + for (int i = 0; i < argc; i++) { + if (need_add_column_name) { + ObFixSizeString name; + name.set(column_name[i]); + if (OB_FAIL(v2_handler->sqlite3_column_name_.push_back(name))) { + LOG_WARN("sqlite3 column name push back failed", K(ret)); + } + } + + if (OB_SUCC(ret)) { + ObProxyVariantString value; + int32_t len = (NULL == argv[i] ? 0 : static_cast(strlen(argv[i]))); + value.set_value(len, argv[i]); + if (OB_FAIL(value_array.push_back(value))) { + LOG_WARN("value array push back failed", K(ret)); + } + } + } + + if (OB_SUCC(ret) && OB_FAIL(v2_handler->sqlite3_column_value_.push_back(value_array))) { + LOG_WARN("column value push back failed", K(ret)); + } + } + + return ret; +} + +int ObConfigProcessor::handle_select_stmt(ObString& sql, obproxy::ObConfigV2Handler *v2_handler) +{ + int ret = OB_SUCCESS; + char *sql_str = (char*)ob_malloc(sql.length() + 1); + if (NULL == sql_str) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(sql.length())); + } else { + char *err_msg = NULL; + memcpy(sql_str, sql.ptr(), sql.length()); + sql_str[sql.length()] = '\0'; + + if (SQLITE_OK != sqlite3_exec(proxy_config_db_, sql_str, sqlite3_callback, (void*)v2_handler, &err_msg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sqlite3 exec sql failed", K(ret), K(sql), "err_msg", err_msg); + sqlite3_free(err_msg); + } + + ob_free(sql_str); + } + + return ret; +} + +int ObConfigProcessor::register_callback(const common::ObString &table_name, ObConfigHandler&handler) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(table_handler_map_.set_refactored(table_name, handler))) { + LOG_WARN("set refactored failed", K(ret), K(table_name)); + } + + return ret; +} + +bool ObConfigProcessor::is_table_in_service(const ObString &table_name) +{ + bool is_in_service = false; + if (table_name == all_table_version_table_name + || table_name == white_list_table_name + || table_name == resource_unit_table_name + || table_name == ssl_config_table_name) { + is_in_service = true; + } + + return is_in_service; +} + +int ObConfigProcessor::store_cloud_config(const ObString &table_name, + const ObString &cluster_name, + const ObString &tenant_name, + const ObString &name, + const ObString &value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cluster_name.empty() || tenant_name.empty() || name.empty() || value.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(cluster_name), K(tenant_name), K(name), K(value)); + } else { + const char *sql = "replace into %.*s(cluster_name, tenant_name, name, value) values('%.*s', '%.*s', '%.*s', '%.*s')"; + int64_t buf_len = cluster_name.length() + tenant_name.length() + name.length() + value.length() + strlen(sql); + char *sql_buf = (char*)ob_malloc(buf_len + 1, ObModIds::OB_PROXY_CONFIG_TABLE); + if (OB_ISNULL(sql_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(buf_len)); + } else { + ObString sql_string; + int64_t len = 0; + len = static_cast(snprintf(sql_buf, buf_len, sql, table_name.length(), table_name.ptr(), + cluster_name.length(), cluster_name.ptr(), + tenant_name.length(), tenant_name.ptr(), + name.length(), name.ptr(), + value.length(), value.ptr())); + sql_string.assign_ptr(sql_buf, static_cast(len)); + if (OB_UNLIKELY(len <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to fill sql", K(len), K(ret)); + } else if (OB_FAIL(execute(sql_string, OBPROXY_T_REPLACE, NULL))) { + LOG_WARN("execute sql failed", K(ret)); + } + } + + if (OB_NOT_NULL(sql_buf)) { + ob_free(sql_buf); + } + } + + return ret; +} + +} // end of obutils +} // end of obproxy +} // end of oceanbase diff --git a/src/obproxy/obutils/ob_config_processor.h b/src/obproxy/obutils/ob_config_processor.h new file mode 100644 index 0000000000000000000000000000000000000000..5e425681346a5250671f00d6da6c7d132f503b1c --- /dev/null +++ b/src/obproxy/obutils/ob_config_processor.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_CONFIG_PROCESSOR_H_ +#define OB_CONFIG_PROCESSOR_H_ + +#include + +#include "lib/hash/ob_hashmap.h" +#include "lib/string/ob_string.h" +#include "lib/string/ob_fixed_length_string.h" +#include "lib/ptr/ob_ptr.h" +#include "iocore/eventsystem/ob_lock.h" +#include "obutils/ob_proxy_sql_parser.h" +#include "lib/lock/ob_drw_lock.h" + +namespace oceanbase +{ +namespace obproxy +{ + +class ObConfigV2Handler; +namespace obutils +{ + +enum ObConfigType +{ + OBPROXY_CONFIG_INVALID = -1, + OBPROXY_CONFIG_CLOUD, + OBPROXY_CONFIG_MAX +}; + +typedef int (*config_processor_execute) (void *); +typedef int (*config_processor_commit) (bool is_success); + +struct ObConfigHandler +{ + config_processor_execute execute_func_; + config_processor_commit commit_func_; + ObConfigType config_type_; +}; + +typedef common::hash::ObHashMap, ObConfigHandler> ConfigHandlerHashMap; + +class ObFnParams +{ +public: + ObFnParams(); + ~ObFnParams(); + +public: + ObConfigType config_type_; + ObProxyBasicStmtType stmt_type_; + common::ObString table_name_; + SqlFieldResult *fields_; +}; + +class ObCloudFnParams : public ObFnParams +{ +public: + ObCloudFnParams(); + ~ObCloudFnParams(); + +public: + common::ObString cluster_name_; + common::ObString tenant_name_; +}; + +struct ResolveContext +{ + ResolveContext() : stmt_type_(OBPROXY_T_INVALID), table_name_(), + config_type_(OBPROXY_CONFIG_INVALID), column_name_array_(), + sql_field_(), handler_() + {} + + ObProxyBasicStmtType stmt_type_; + common::ObString table_name_; + ObConfigType config_type_; + common::ObSEArray column_name_array_; + obutils::SqlFieldResult sql_field_; + ObConfigHandler handler_; +}; + +class ObConfigProcessor +{ +public: + ObConfigProcessor(); + ~ObConfigProcessor(); + + int init(); + int execute(common::ObString &sql, const ObProxyBasicStmtType stmt_type, obproxy::ObConfigV2Handler *handler); + int register_callback(const common::ObString &table_name, ObConfigHandler&handler); + bool is_table_in_service(const common::ObString &table_name); + int store_cloud_config(const common::ObString &table_name, + const common::ObString &cluster_name, + const common::ObString& tenant_name, + const common::ObString& name, + const common::ObString &value); + +private: + int init_config_from_disk(); + int check_and_create_table(); + int resolve_insert(const ParseNode* node, ResolveContext &ctx); + int resolve_delete(const ParseNode* node, ResolveContext &ctx); + int handle_dml_stmt(common::ObString &sql, ParseResult& parse_result); + int handle_select_stmt(common::ObString &sql, obproxy::ObConfigV2Handler *v2_handler); + + static int init_callback(void *data, int argc, char **argv, char **column_name); + static int sqlite3_callback(void *data, int argc, char **argv, char **column_name); +private: + sqlite3 *proxy_config_db_; + ConfigHandlerHashMap table_handler_map_; + common::DRWLock config_lock_; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigProcessor); +}; + +ObConfigProcessor &get_global_config_processor(); + +} // end of obutils +} // end of obproxy +} // end of oceanbase +#endif diff --git a/src/obproxy/obutils/ob_congestion_manager.cpp b/src/obproxy/obutils/ob_congestion_manager.cpp index 77e788e6d8996d2114c5e829c17b6cb11d0e8c31..ee58fb8fb1b71d851c7a54c2c1f5585317890f45 100644 --- a/src/obproxy/obutils/ob_congestion_manager.cpp +++ b/src/obproxy/obutils/ob_congestion_manager.cpp @@ -579,8 +579,8 @@ int ObCongestionManager::process(const int64_t buck_id, ObCongestRequestParam *p case ObCongestRequestParam::REVALIDATE_SERVER: { ObCongestionEntry *entry = lookup_entry(param->hash_, param->key_); if (NULL != entry) { + entry->server_state_ = param->server_state_; switch (param->server_state_) { - entry->server_state_ = param->server_state_; case ObCongestionEntry::ACTIVE: entry->check_and_set_alive(); break; @@ -956,7 +956,7 @@ int ObCongestionManager::revalidate_server(const ObIpEndpoint &ip, } else { entry = lookup_entry(hash, ip); LOG_DEBUG("lock bucket succ, find the entry", "server_state", - ObCongestionEntry::get_server_state_name(server_state), K(*entry)); + ObCongestionEntry::get_server_state_name(server_state), KP(entry)); if (NULL != entry) { entry->server_state_ = server_state; entry->renew_last_revalidate_time(); diff --git a/src/obproxy/obutils/ob_hot_upgrade_processor.cpp b/src/obproxy/obutils/ob_hot_upgrade_processor.cpp index 036bbfe81c8f8538a25ebb3f378ef8067ef06faf..5ddfbc27d9a5ceea2fe0614652cfa779d8b3fb80 100644 --- a/src/obproxy/obutils/ob_hot_upgrade_processor.cpp +++ b/src/obproxy/obutils/ob_hot_upgrade_processor.cpp @@ -27,12 +27,14 @@ #include "obutils/ob_async_common_task.h" #include "proxy/client/ob_mysql_proxy.h" #include "proxy/mysqllib/ob_proxy_auth_parser.h" +#include "prometheus/ob_prometheus_processor.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::event; using namespace oceanbase::obproxy::net; using namespace oceanbase::obproxy::proxy; +using namespace oceanbase::obproxy::prometheus; namespace oceanbase { @@ -96,22 +98,7 @@ int ObProxyHotUpgrader::fill_exec_ctx(ObExecCtx &ctx) if (OB_FAIL(get_envp(ctx.envp_))) { LOG_ERROR("fail to get envp", K(ret)); } else { - const bool is_server_service_mode = !get_global_proxy_config().is_client_service_mode(); - const int64_t idle_idx = (is_server_service_mode ? 2 : 1); ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); - if (info.is_local_restart()) {//need update inherited_argv_ - const int64_t length = snprintf(info.restart_buf_, ObHotUpgraderInfo::MAX_RESTART_BUF_SIZE, "-oproxy_local_cmd=2"); - if (OB_UNLIKELY(length <= 0) || OB_UNLIKELY(length >= ObHotUpgraderInfo::MAX_RESTART_BUF_SIZE)) { - ret = OB_BUF_NOT_ENOUGH; - LOG_WARN("buf not enought", K(length), LITERAL_K(ObHotUpgraderInfo::MAX_RESTART_BUF_SIZE), K(ret)); - } else { - info.inherited_argv_[idle_idx] = info.restart_buf_; - info.inherited_argv_[idle_idx + 1] = NULL; - } - } else { - //clear - info.inherited_argv_[idle_idx] = NULL; - } ctx.path_ = info.argv_[0]; ctx.name_ = "new obproxy binary process"; ctx.argv_ = info.inherited_argv_; @@ -263,7 +250,7 @@ int get_binary_md5(const char *binary, char *md5_buf, const int64_t md5_buf_len) /******************************ObHotUpgradeProcessor************************************/ ObHotUpgradeProcessor::ObHotUpgradeProcessor() : is_inited_(false), is_timeout_rollback_(false), is_self_md5_available_(false), - is_self_binary_(false), cmd_(HUC_NONE), mysql_proxy_(NULL), hu_cont_(NULL), + is_self_binary_(false), cmd_(HUC_NONE), mysql_proxy_(NULL), hu_cont_(NULL), hot_upgrade_cont_(NULL), info_(get_global_hot_upgrade_info()), proxy_port_(0), upgrade_failures_(0), check_available_failures_(0), timeout_rollback_timeout_at_(0), wait_cr_finish_timeout_at_(0) { @@ -279,7 +266,10 @@ void ObHotUpgradeProcessor::destroy() { int ret = OB_SUCCESS; if (OB_FAIL(ObAsyncCommonTask::destroy_repeat_task(hu_cont_))) { - LOG_WARN("fail to destroy hut upgrader cont", K(ret)); + LOG_WARN("fail to destroy hot upgrader cont", K(ret)); + } + if (OB_FAIL(ObAsyncCommonTask::destroy_repeat_task(hot_upgrade_cont_))) { + LOG_WARN("fail to destroy hot upgrader cont", K(ret)); } if (OB_LIKELY(is_inited_)) { is_inited_ = false; @@ -660,15 +650,6 @@ int ObHotUpgradeProcessor::check_timeout_rollback() } else if (0 < (diff_time = timeout_rollback_timeout_at_ - get_hrtime_internal())) { LOG_DEBUG("it is not time to do hot upgrade timeout rollback job", K_(timeout_rollback_timeout_at), K(diff_time)); - - } else if (info_.is_in_wait_cr_state() - && (info_.is_local_commit() - || info_.is_local_rollback() - || get_global_proxy_config().is_local_commit() - || get_global_proxy_config().is_local_rollback())) { - LOG_INFO("receive local cmd, ignore timeout rollback this moment", K_(timeout_rollback_timeout_at), - K(diff_time), "cmd", ObHotUpgraderInfo::get_cmd_string(info_.cmd_)); - } else { LOG_WARN("it is time, but we still not receive rollback cmd, we need do timeout rollback job", K_(timeout_rollback_timeout_at), K(diff_time)); @@ -682,7 +663,7 @@ int ObHotUpgradeProcessor::check_timeout_rollback() // we will try 3 times to update_proxy_hu_cmd. int64_t retry_times = 3; while (OB_LIKELY(retry_times > 0) && OB_SUCC(ret)) { - if (OB_FAIL(ObProxyTableProcessorUtils::update_proxy_hu_cmd(*mysql_proxy_, proxy_ip_, + if (OB_NOT_NULL(mysql_proxy_) && OB_FAIL(ObProxyTableProcessorUtils::update_proxy_hu_cmd(*mysql_proxy_, proxy_ip_, proxy_port_, target_cmd, orig_cmd, orig_cmd_either, orig_cmd_another)) && OB_ERR_UNEXPECTED != ret) { LOG_WARN("fail to update hot upgrade cmd, unknown error code, we will retry update", @@ -717,9 +698,6 @@ int ObHotUpgradeProcessor::state_wait_hu_cmd(const ObProxyServerInfo &proxy_info "receive cmd", ObHotUpgraderInfo::get_cmd_string(info_.cmd_)); switch (info_.cmd_) { - case HUC_LOCAL_RESTART: - //restart from config cmd - check_available_failures_ = 0; case HUC_RESTART: //restart also arrive here, but no need check_arguments case HUC_UPGRADE_BIN: @@ -774,13 +752,11 @@ int ObHotUpgradeProcessor::state_wait_hu_cmd(const ObProxyServerInfo &proxy_info } break; } - case HUC_LOCAL_EXIT: - //try to report status, but do not care result case HUC_EXIT: { //when recv "exit" cmd, we need do follow things: //1. update status in table //2. do quick exit in 5 seconds - if (OB_FAIL(ObProxyTableProcessorUtils::update_proxy_exited_status(*mysql_proxy_, + if (OB_NOT_NULL(mysql_proxy_) && OB_FAIL(ObProxyTableProcessorUtils::update_proxy_exited_status(*mysql_proxy_, proxy_ip_, proxy_port_, HU_STATUS_DO_QUICK_EXIT, HUC_NONE))) { LOG_WARN("fail to update hot upgrade status and cmd to DO_GRACEFUL_EXIT, NONE", K_(info), K(ret)); } @@ -894,7 +870,6 @@ int ObHotUpgradeProcessor::state_parent_wait_cr_cmd(const ObProxyServerInfo &pro } switch (info_.cmd_) { - case HUC_LOCAL_COMMIT: case HUC_COMMIT: { //when recv commit, parent must do follow things: //1. disable net_accept @@ -904,7 +879,7 @@ int ObHotUpgradeProcessor::state_parent_wait_cr_cmd(const ObProxyServerInfo &pro info_.disable_net_accept(); info_.graceful_exit_start_time_ = get_hrtime_internal(); - info_.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_graceful_exit_timeout) + info_.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_exit_timeout) + info_.graceful_exit_start_time_;// nanosecond info_.update_both_status(HU_STATUS_RECV_COMMIT_AND_EXIT, HU_STATUS_COMMIT_SUCC); info_.update_state(HU_STATE_WAIT_CR_FINISH); @@ -914,7 +889,6 @@ int ObHotUpgradeProcessor::state_parent_wait_cr_cmd(const ObProxyServerInfo &pro break; } - case HUC_LOCAL_ROLLBACK: case HUC_ROLLBACK: { //when recv rollback, parent must do follow things: //1. update status, state and wait_cr_finish_timeout_at_ @@ -941,7 +915,7 @@ int ObHotUpgradeProcessor::state_parent_wait_cr_cmd(const ObProxyServerInfo &pro info_.update_state(HU_STATE_WAIT_CR_FINISH); info_.update_sub_status(is_timeout_rollback_ ? HU_STATUS_RECV_TIMEOUT_ROLLBACK_AND_EXIT : HU_STATUS_RECV_ROLLBACK_AND_EXIT); wait_cr_finish_timeout_at_ = get_hrtime_internal() - + HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_graceful_exit_timeout) + + HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_exit_timeout) + HRTIME_USECONDS(get_global_proxy_config().proxy_info_check_interval); } upgrade_failures_ = 0;// reset to zero if ever succ to fork new proxy @@ -977,12 +951,6 @@ int ObHotUpgradeProcessor::state_parent_wait_cr_cmd(const ObProxyServerInfo &pro } break; } - case HUC_LOCAL_RESTART: { - if (OB_FAIL(handle_local_restart())) { - LOG_WARN("fail to handle local restart", K_(info), K(ret)); - } - break; - } default: { ret = OB_ERR_UNEXPECTED; LOG_ERROR("unknown ObHotUpgradeCmd, it should not enter here", K_(info), K(ret)); @@ -1018,7 +986,6 @@ int ObHotUpgradeProcessor::state_sub_wait_cr_cmd() } switch (info_.cmd_) { - case HUC_LOCAL_COMMIT: case HUC_COMMIT: { //when recv commit, sub will do follow things: //1. update status, state and wait_cr_finish_timeout_at_ @@ -1038,12 +1005,12 @@ int ObHotUpgradeProcessor::state_sub_wait_cr_cmd() info_.update_state(HU_STATE_WAIT_CR_FINISH); info_.update_both_status(HU_STATUS_RECV_COMMIT_AND_EXIT, HU_STATUS_COMMIT_SUCC); wait_cr_finish_timeout_at_ = get_hrtime_internal() - + HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_graceful_exit_timeout) + + HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_exit_timeout) + HRTIME_USECONDS(get_global_proxy_config().proxy_info_check_interval); } upgrade_failures_ = 0;// reset to zero if ever succ to fork new proxy cancel_timeout_rollback(); - if (!is_self_binary() && !info_.is_local_commit() && OB_FAIL(backup_old_binary())) {//!!it maybe cost long time + if (!is_self_binary() && OB_FAIL(backup_old_binary())) {//!!it maybe cost long time LOG_WARN("fail to backup old binary, but it has nothing affect", K(ret)); } info_.cmd_ = HUC_COMMIT;//reset it to commit @@ -1051,7 +1018,6 @@ int ObHotUpgradeProcessor::state_sub_wait_cr_cmd() break; } - case HUC_LOCAL_ROLLBACK: case HUC_ROLLBACK: { //when recv rollback, sub must do follow things: //1. disable net_accept @@ -1061,7 +1027,7 @@ int ObHotUpgradeProcessor::state_sub_wait_cr_cmd() info_.disable_net_accept(); info_.graceful_exit_start_time_ = get_hrtime_internal(); - info_.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_graceful_exit_timeout) + info_.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().hot_upgrade_exit_timeout) + info_.graceful_exit_start_time_; // nanosecond info_.update_state(HU_STATE_WAIT_CR_FINISH); if (is_timeout_rollback_) { @@ -1077,7 +1043,6 @@ int ObHotUpgradeProcessor::state_sub_wait_cr_cmd() case HUC_NONE: case HUC_EXIT: case HUC_RESTART: - case HUC_LOCAL_RESTART: case HUC_AUTO_UPGRADE: case HUC_HOT_UPGRADE: { LOG_DEBUG("it is invalid cmd, no need to perform", K_(info)); @@ -1498,29 +1463,6 @@ int ObHotUpgradeProcessor::send_commit_via_subprocess() return ret; } -//parent proxy need send local cmd to subprocess -int ObHotUpgradeProcessor::send_local_cmd_to_subprocess(const ObProxyLocalCMDType type) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ObProxyConfig::is_local_cmd(type))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid input argument", K(type), K(ret)); - } else { - const int64_t MAX_RETRY_TIMES = 16; - char sql[OB_SHORT_SQL_LENGTH]; - const int64_t len = snprintf(sql, OB_SHORT_SQL_LENGTH, "alter proxyconfig set proxy_local_cmd=%d", type); - if (OB_UNLIKELY(len <= 0) || OB_UNLIKELY(len >= OB_SHORT_SQL_LENGTH)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to fill sql", K(len), K(sql), K(ret)); - } else if (OB_FAIL(send_cmd_and_check_response(sql, USER_TYPE_PROXYSYS, MAX_RETRY_TIMES))) { - LOG_WARN("fail to send local cmd to subprocess", K(sql), K(USER_TYPE_METADB), K(ret)); - } else { - //do nothing - } - } - return ret; -} - int ObHotUpgradeProcessor::check_subprocess_available() { int ret = OB_SUCCESS; @@ -1602,32 +1544,158 @@ int ObHotUpgradeProcessor::check_subprocess_available() return ret; } -int ObHotUpgradeProcessor::handle_local_restart() +int ObHotUpgradeProcessor::do_hot_upgrade_repeat_task() +{ + return get_global_hot_upgrade_processor().do_hot_upgrade_work(); +} + +void ObHotUpgradeProcessor::update_hot_upgrade_interval() +{ + ObAsyncCommonTask *cont = get_global_hot_upgrade_processor().get_new_hot_upgrade_cont(); + if (OB_LIKELY(NULL != cont)) { + int64_t interval_us = ObRandomNumUtils::get_random_half_to_full( + get_global_proxy_config().proxy_hot_upgrade_check_interval); + cont->set_interval(interval_us); + } +} + +int ObHotUpgradeProcessor::start_hot_upgrade_task() { int ret = OB_SUCCESS; - LOG_INFO("parent proxy need check subprocess available", K_(info)); - bool need_rollback = false; - if (OB_FAIL(check_subprocess_available())) { - if (++check_available_failures_ < OB_MAX_CHECK_SUBPROCESS_FAILURES) { - LOG_WARN("subprocess is unavailable, will try it next time", K(check_available_failures_), K_(info), K(ret)); + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("obproxy table processor is not inited", K(ret)); + } else if (OB_UNLIKELY(NULL != hot_upgrade_cont_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("hot upgrade cont has been scheduled", K(ret)); + } else { + int64_t interval_us = ObRandomNumUtils::get_random_half_to_full( + get_global_proxy_config().proxy_hot_upgrade_check_interval); + if (OB_ISNULL(hot_upgrade_cont_ = ObAsyncCommonTask::create_and_start_repeat_task(interval_us, + "proxy_info_check_task", + ObHotUpgradeProcessor::do_hot_upgrade_repeat_task, + ObHotUpgradeProcessor::update_hot_upgrade_interval))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to create and start proxy table check task", K(ret)); } else { - need_rollback = true; - LOG_WARN("subprocess is unavailable, do rollback next time", K(check_available_failures_), K_(info)); + LOG_INFO("succ to start proxy table check task", K(interval_us)); } - } else if (OB_FAIL(send_local_cmd_to_subprocess(OB_LOCAL_CMD_COMMIT))) { - need_rollback = true; - LOG_WARN("fail to send commit to subprocess, do rollback next time", K_(info)); - } else { - info_.cmd_ = HUC_LOCAL_COMMIT; - LOG_INFO("subprocess is available, do commit next time", K_(info)); } + return ret; +} - if (need_rollback) { - if (OB_FAIL(send_local_cmd_to_subprocess(OB_LOCAL_CMD_ROLLBACK))) { - LOG_WARN("fail to send rollback to subprocess, ignore it, let subprocess timeout rollback", K(ret)); +int ObHotUpgradeProcessor::do_hot_upgrade_work() +{ + int ret = OB_SUCCESS; + + ObProxyLocalCMDType cmd_type = get_global_proxy_config().get_local_cmd_type(); + switch (cmd_type) { + case OB_LOCAL_CMD_NONE: { + //do nothing + break; + } + case OB_LOCAL_CMD_EXIT: { + get_global_proxy_config().reset_local_cmd(); + if (info_.is_in_idle_state() && info_.is_parent()) { + info_.cmd_ = HUC_LOCAL_EXIT; + LOG_INFO("proxy will do quick exit from local cmd, no need check proxy info", K_(info)); + } else { + LOG_WARN("it is doing hot upgrading now, CAN NOT do quick exit from local cmd", K_(info)); + } + + break; + } + case OB_LOCAL_CMD_RESTART: { + get_global_proxy_config().reset_local_cmd(); + if (info_.is_in_idle_state() && info_.is_parent()) { + info_.cmd_ = HUC_LOCAL_RESTART; + LOG_INFO("proxy will do restart from local cmd", K_(info)); + } else { + LOG_WARN("it is doing hot upgrading now, CAN NOT do quick restart from local cmd", K_(info)); + } + break; + } + default: { + LOG_WARN("unknown ObProxyLocalCMDType, reset it default value", K(cmd_type)); + get_global_proxy_config().reset_local_cmd(); } - info_.cmd_ = HUC_LOCAL_ROLLBACK; } + + switch (info_.get_state()) { + case HU_STATE_WAIT_HU_CMD: { + LOG_DEBUG("enter HU_STATE_WAIT_HU_CMD state", "pid", getpid(), + "current status", ObHotUpgraderInfo::get_status_string(info_.status_), + "receive cmd", ObHotUpgraderInfo::get_cmd_string(info_.cmd_)); + + switch (info_.cmd_) { + case HUC_LOCAL_RESTART: { + g_ob_prometheus_processor.destroy_exposer(); + if (OB_FAIL(ObProxyHotUpgrader::spawn_process())) { + LOG_WARN("fail to spawn sub process", K_(info), K(ret)); + g_ob_prometheus_processor.create_exposer(); + } else { + schedule_timeout_rollback(); + info_.disable_net_accept(); // disable accecpt new connection + info_.update_sub_status(HU_STATUS_NONE); + info_.update_state(HU_STATE_WAIT_LOCAL_CR_FINISH); + LOG_INFO("succ to fork new proxy, start turn to HU_STATE_WAIT_CR_FINISH", K_(info)); + } + break; + } + case HUC_LOCAL_EXIT: { + info_.disable_net_accept(); + info_.graceful_exit_start_time_ = get_hrtime_internal(); + info_.graceful_exit_end_time_ = HRTIME_USECONDS(get_global_proxy_config().delay_exit_time) + + info_.graceful_exit_start_time_; // nanosecond + g_proxy_fatal_errcode = OB_SERVER_IS_STOPPING; + LOG_WARN("parent process stop accepting new connection, stop check timer", K_(info), K(g_proxy_fatal_errcode), K(ret)); + break; + } + case HUC_NONE: + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unknown ObHotUpgradeCmd, it should not enter here", K_(info), K(ret)); + } + } + info_.cmd_ = HUC_NONE; + break; + } + case HU_STATE_WAIT_LOCAL_CR_FINISH: { + if (OB_LIKELY(common::OB_SUCCESS == lib::mutex_acquire(&info_.hot_upgrade_mutex_))) { + if (info_.is_sub_exited()) { + info_.enable_net_accept(); + g_ob_prometheus_processor.create_exposer(); + info_.update_sub_status(HU_STATUS_NONE); + info_.update_state(HU_STATE_WAIT_HU_CMD); + } else { + if (OB_LIKELY(!info_.parent_hot_upgrade_flag_)) { + ObHRTime diff_time = 0; + if (OB_LIKELY(timeout_rollback_timeout_at_ <= 0)) { + // maybe has not started + } else if (0 < (diff_time = timeout_rollback_timeout_at_ - get_hrtime_internal())) { + LOG_DEBUG("it is not time to do hot upgrade timeout rollback job", + K_(timeout_rollback_timeout_at), K(diff_time)); + } else { + kill(info_.sub_pid_, SIGKILL); + info_.enable_net_accept(); + g_ob_prometheus_processor.create_exposer(); + info_.update_state(HU_STATE_WAIT_HU_CMD); + info_.parent_hot_upgrade_flag_ = true; + LOG_INFO("upgrade timeout, will send SIGKILL to subprocess", K_(info)); + } + } + } + lib::mutex_release(&info_.hot_upgrade_mutex_); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("it should not enter here", K_(info)); + } + } + return ret; } diff --git a/src/obproxy/obutils/ob_hot_upgrade_processor.h b/src/obproxy/obutils/ob_hot_upgrade_processor.h index 2dd2d7f96ba42552bb434ca12473aa95b4e96de8..86798808ae6d2cdf2f27c5bb784417012c4430e2 100644 --- a/src/obproxy/obutils/ob_hot_upgrade_processor.h +++ b/src/obproxy/obutils/ob_hot_upgrade_processor.h @@ -71,6 +71,7 @@ public: static int do_repeat_task(); int handle_hot_upgrade(); ObAsyncCommonTask *get_hot_upgrade_cont() { return hu_cont_; } + ObAsyncCommonTask *get_new_hot_upgrade_cont() { return hot_upgrade_cont_; } bool is_same_hot_binary(const common::ObString &name, const common::ObString &md5) const { return (name == hot_binary_name_ && md5 == hot_binary_md5_); @@ -80,6 +81,11 @@ public: return (name == cold_binary_name_ && md5 == cold_binary_md5_); } + static int do_hot_upgrade_repeat_task(); + static void update_hot_upgrade_interval(); + int start_hot_upgrade_task(); + int do_hot_upgrade_work(); + DECLARE_TO_STRING; private: @@ -105,9 +111,7 @@ private: int send_cmd_and_check_response(const char *sql, const ObProxyLoginUserType type, const int64_t retry_times = 1, const int64_t expected_affected_rows = 0); int send_commit_via_subprocess(); - int send_local_cmd_to_subprocess(const ObProxyLocalCMDType type); int check_subprocess_available(); - int handle_local_restart(); public: static const int64_t OB_MAX_CHECK_SUBPROCESS_FAILURES = 32; @@ -127,6 +131,7 @@ private: ObHotUpgradeCmd cmd_; proxy::ObMysqlProxy *mysql_proxy_; ObAsyncCommonTask *hu_cont_; + ObAsyncCommonTask *hot_upgrade_cont_; ObHotUpgraderInfo &info_; char proxy_ip_[common::OB_IP_STR_BUFF]; // ip primary key char proxy_self_md5_[OB_DEFAULT_PROXY_MD5_LEN + 1]; diff --git a/src/obproxy/obutils/ob_log_file_processor.cpp b/src/obproxy/obutils/ob_log_file_processor.cpp index e6e5c85628a453ed2e3ac4daf6d90371c779a287..29e741a3aa9c225ace5dcba262154c685cec862d 100644 --- a/src/obproxy/obutils/ob_log_file_processor.cpp +++ b/src/obproxy/obutils/ob_log_file_processor.cpp @@ -30,6 +30,14 @@ namespace oceanbase { namespace obproxy { +/* Log files are divided into blocks and then compressed. The default block size is (2M - 1K).*/ +static const int32_t DEFAULT_COMPRESSION_BLOCK_SIZE = OB_MALLOC_BIG_BLOCK_SIZE; +/* To prevent extreme cases where the files become larger after compression, + * the size of the decompression buffer needs to be larger than the original data. + * Specific size can refer to the ZSTD code implementation. */ +static const int32_t DEFAULT_COMPRESSION_BUFFER_SIZE = + DEFAULT_COMPRESSION_BLOCK_SIZE + DEFAULT_COMPRESSION_BLOCK_SIZE / 128 + 512 + 19; + //------ ObProxyLogFileStruct------ DEF_TO_STRING(ObProxyLogFileStruct) { @@ -47,7 +55,7 @@ ObLogFileProcessor &get_global_log_file_processor() } ObLogFileProcessor::ObLogFileProcessor() - : is_inited_(false), cleanup_cont_(NULL) + : is_inited_(false), zstd_compressor_1_3_8_(), cleanup_cont_(NULL) { } @@ -181,13 +189,13 @@ int ObLogFileProcessor::cleanup_log_file() int ret = OB_SUCCESS; int64_t begin = ObTimeUtility::current_time(); int64_t total_size = 0; - int64_t cur_process_log_size = 0; ObArray log_array; + ObArray compress_log_array; ObArray invalid_log_array; struct dirent *ent = NULL; const char *layout_log_dir = NULL; DIR *log_dir = NULL; - const bool is_in_single_service = get_global_hot_upgrade_info().is_in_single_service(); + bool enable_syslog_file_compress = get_global_proxy_config().enable_syslog_file_compress; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -211,18 +219,19 @@ int ObLogFileProcessor::cleanup_log_file() LOG_WARN("fail to filter log file", K(ret)); } else if (need_further_handle) { need_further_handle = false; - total_size += log_st.size_; bool skip_file = false; bool self_process_file = false; + bool is_wf_file = false; ObString full_file_name; full_file_name.assign_ptr(log_st.full_path_, static_cast(STRLEN(log_st.full_path_))); ObString current_file_name; ObString current_wf_file_name; char wf_filename[OB_MAX_FILE_NAME_LENGTH + 1] = "\0"; + ObLogFDType type = FD_DEFAULT_FILE; - for (ObLogFDType type = FD_DEFAULT_FILE; !skip_file && type < MAX_FD_FILE; type = (ObLogFDType)(type + 1)) { + for (; type < MAX_FD_FILE; type = (ObLogFDType)(type + 1)) { const ObLogFileStruct& log_file = OB_LOGGER.get_log_file(type); current_file_name.assign_ptr(log_file.filename_, static_cast(STRLEN(log_file.filename_))); if (log_file.is_enable_wf_file() && log_file.wf_fd_ > 0) { @@ -232,47 +241,51 @@ int ObLogFileProcessor::cleanup_log_file() current_wf_file_name.reset(); } - if (type == FD_DEFAULT_FILE - && ((!current_file_name.empty() && full_file_name.prefix_match(current_file_name)) - || (!current_wf_file_name.empty() && full_file_name.prefix_match(current_wf_file_name)))) { - cur_process_log_size += log_st.size_; - self_process_file = true; - } - if (full_file_name == current_file_name || full_file_name == current_wf_file_name) { skip_file = true; + break; + } + + if (!current_wf_file_name.empty() && full_file_name.prefix_match(current_wf_file_name)){ + self_process_file = true; + is_wf_file = true; + break; + } else if (!current_file_name.empty() && full_file_name.prefix_match(current_file_name)) { + self_process_file = true; + break; } } if (skip_file) { //skip current using log - } else if (!is_in_single_service) { - //when in hot upgrade progress, just clean up log of self process - if (self_process_file) { - need_further_handle = true; + total_size += log_st.size_; + } else if (!self_process_file) { + // handle other invalid obproxy.xxx.log or obproxy.xxx.log.time + if (OB_FAIL(invalid_log_array.push_back(log_st))) { + LOG_WARN("fail to add file into invalid log array", K(ret)); } } else { - need_further_handle = true; - if ((file_name.after('.')).after('.') == "log") { - // handle other invalid obproxy.xxx.log - if (OB_FAIL(invalid_log_array.push_back(log_st))) { - LOG_WARN("fail to add file into invalid log array", K(ret)); - } else { - need_further_handle = false; + if (enable_syslog_file_compress && !is_wf_file + && (type == FD_DEFAULT_FILE || type == FD_XFLUSH_FILE || type == FD_DIGEST_FILE)) { + if (OB_FAIL(compress_log_array.push_back(log_st))) { + LOG_WARN("fail to add file into compress log array", K(compress_log_array), K(log_st), K(ret)); } + } else if (OB_FAIL(log_array.push_back(log_st))) { + LOG_WARN("fail to add file into log array", K(log_array), K(log_st), K(ret)); + } + if (OB_SUCC(ret)) { + total_size += log_st.size_; } - } - if (need_further_handle && OB_FAIL(log_array.push_back(log_st))) { - LOG_WARN("fail to add file into log array", K(log_array), K(log_st), K(ret)); } } }//end of while } if (OB_SUCC(ret)) { - std::sort(log_array.begin(), log_array.end()); if (OB_FAIL(do_cleanup_invalid_log_file(invalid_log_array))) { LOG_WARN("fail to do cleanup invalid log file", K(ret)); - } else if (OB_FAIL(do_cleanup_log_file(log_array, is_in_single_service, cur_process_log_size, total_size))) { + } else if (enable_syslog_file_compress && OB_FAIL(do_cleanup_compress_log_file(log_array, compress_log_array, total_size))) { + LOG_WARN("fail to do cleanup compress log file", K(ret)); + } else if (OB_FAIL(do_cleanup_log_file(log_array, total_size))) { LOG_WARN("fail to do cleanup log file", K(ret)); } } @@ -289,9 +302,7 @@ int ObLogFileProcessor::cleanup_log_file() int ObLogFileProcessor::do_cleanup_invalid_log_file(ObArray &log_array) { int ret = OB_SUCCESS; - std::sort(log_array.begin(), log_array.end()); - int64_t delete_num = log_array.count() - MAX_INVALID_PROXY_LOG_NUM; - for (int64_t i = 0; OB_SUCC(ret) && i < delete_num; ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < log_array.count(); ++i) { if (OB_UNLIKELY(0 != ::unlink(log_array[i].full_path_))) { ret = OB_IO_ERROR; LOG_WARN("fail to unlink file", K(log_array[i].full_path_), KERRMSGS, K(ret)); @@ -300,10 +311,161 @@ int ObLogFileProcessor::do_cleanup_invalid_log_file(ObArray &log_array, const bool is_in_single_service, - const int64_t cur_process_log_size, int64_t &total_size) +int ObLogFileProcessor::log_compress_block(char *dest, size_t dest_size, + const char *src, size_t src_size, + size_t &return_size) +{ + int ret = OB_SUCCESS; + int64_t size = -1; + if (OB_FAIL(zstd_compressor_1_3_8_.compress(src, src_size, dest, dest_size, size))) { + LOG_WARN("Failed to compress", K(ret)); + } else { + return_size = size; + } + return ret; +} + +int ObLogFileProcessor::log_compress(ObProxyLogFileStruct &log_st, const ObString &file_name, const ObString &compression_file_name, + int64_t &total_size, char *src_buf, int src_size, char *dest_buf, int dest_size) { int ret = OB_SUCCESS; + static int sleep_us = 50 * 1000; // 50ms, 40M/s, 6s per file(256M) + FILE *input_file = NULL; + FILE *output_file = NULL; + + if (compression_file_name.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get_compression_file_name", K(ret)); + } else if (NULL == (input_file = fopen(file_name.ptr(), "r"))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Failed to fopen", "error_code", errno, K(ret)); + } else if (NULL == (output_file = fopen(compression_file_name.ptr(), "w"))) { + ret = OB_ERR_UNEXPECTED; + fclose(input_file); + LOG_WARN("Failed to fopen", "error_code", errno, K(ret)); + } else { + size_t read_size = 0; + size_t write_size = 0; + while (OB_SUCC(ret) && !feof(input_file)) { + if ((read_size = fread(src_buf, 1, src_size, input_file)) > 0) { + if (OB_FAIL(log_compress_block(dest_buf, dest_size, src_buf, read_size, write_size))) { + LOG_WARN("Failed to log_compress_block", K(ret)); + } else if (write_size != fwrite(dest_buf, 1, write_size, output_file)) { + ret = OB_ERR_SYS; + LOG_WARN("Failed to fwrite", "err_code=", errno, K(ret)); + } + } + usleep(sleep_us); + } + fclose(input_file); + fclose(output_file); + if (0 != access(file_name.ptr(), F_OK) || OB_SUCCESS != ret) { + unlink(compression_file_name.ptr()); + } else { + unlink(file_name.ptr()); + struct stat st; + if (0 != (stat(compression_file_name.ptr(), &st))) { + ret = OB_IO_ERROR; + LOG_WARN("fail to stat dir", K(compression_file_name), KERRMSGS, K(ret)); + } else { + total_size += (st.st_size - log_st.size_); + log_st.size_ = st.st_size; + MEMCPY(log_st.full_path_, compression_file_name.ptr(), compression_file_name.length()); + } + } + } + + return ret; +} + +int ObLogFileProcessor::do_cleanup_compress_log_file(ObArray &log_array, + ObArray &compress_log_array, + int64_t &total_size) +{ + int ret = OB_SUCCESS; + + ObProxyConfig &config = get_global_proxy_config(); + int64_t max_syslog_file_count = config.max_syslog_file_count; + int64_t delete_num = max_syslog_file_count > 0 ? compress_log_array.count() - max_syslog_file_count : 0; + + int64_t max_syslog_file_time = config.max_syslog_file_time; + max_syslog_file_time = max_syslog_file_time > 0 ? usec_to_sec(max_syslog_file_time) : 0; + time_t min_time = time(NULL) - max_syslog_file_time; + + int src_size = DEFAULT_COMPRESSION_BLOCK_SIZE; + int dest_size = DEFAULT_COMPRESSION_BUFFER_SIZE; + char *src_buf = NULL; + char *dest_buf = NULL; + char compression_file_name_str[OB_MAX_FILE_NAME_LENGTH] = "\0"; + int64_t i = 0; + + std::sort(compress_log_array.begin(), compress_log_array.end()); + for (; OB_SUCC(ret) && i < compress_log_array.count(); ++i) { + if ((max_syslog_file_time > 0 && compress_log_array[i].mtime_ < min_time) + || (i < delete_num)) { + if (OB_UNLIKELY(0 != ::unlink(compress_log_array[i].full_path_))) { + ret = OB_IO_ERROR; + LOG_WARN("fail to unlink file", K(compress_log_array[i].full_path_), KERRMSGS, K(ret)); + } else { + total_size -= compress_log_array[i].size_; + LOG_INFO("succ to cleanup file", K(compress_log_array[i].full_path_), K(ret)); + } + } else { + const char *idx = NULL; + ObString suffix_str = ".zst"; + ObString file_name = ObString::make_string(compress_log_array[i].full_path_); + ObString compression_file_name; + int size = file_name.length(); + if (size && 0 == file_name[size - 1]) { + size -= 1; + } + if ((size + 1 + suffix_str.length() > OB_MAX_FILE_NAME_LENGTH) + || (size > 4 && NULL != (idx = file_name.reverse_find('.')) && idx != file_name.ptr() && + 0 == file_name.after(--idx).compare(suffix_str))) { + // skip this file + } else { + if (NULL == src_buf) { + src_buf = (char *)ob_malloc(src_size + dest_size, ObModIds::OB_COMPRESSOR); + if (OB_ISNULL(src_buf)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to ob_malloc", K(ret)); + } else { + dest_buf = src_buf + src_size; + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (FALSE_IT(compression_file_name.assign_buffer(compression_file_name_str, OB_MAX_FILE_NAME_LENGTH))) { + // do nothing + } else if (OB_UNLIKELY(size != compression_file_name.write(file_name.ptr(), size))) { + // do nothing + } else if (OB_UNLIKELY(5 != compression_file_name.write(".zst\0", 5))) { + // do nothing + } else if (OB_FAIL(log_compress(compress_log_array[i], file_name, compression_file_name, + total_size, src_buf, src_size, dest_buf, dest_size))) { + LOG_WARN("fail to compress log", K(file_name), K(compression_file_name), K(ret)); + } + } + + if (OB_SUCC(ret)) { + log_array.push_back(compress_log_array[i]); + } + } + } + + if (src_buf) { + ob_free(src_buf); + } + + return ret; +} + +int ObLogFileProcessor::do_cleanup_log_file(ObArray &log_array, + int64_t &total_size) +{ + int ret = OB_SUCCESS; + int64_t avail_size = 0; static const int64_t min_avail_size = 1024 * 1024 * 1024; // 1GB if (OB_FAIL(get_disk_size(get_global_layout().get_log_dir(), avail_size))) { @@ -314,13 +476,6 @@ int ObLogFileProcessor::do_cleanup_log_file(ObArray &log_a ObProxyConfig &config = get_global_proxy_config(); int64_t thresh_hold = config.log_dir_size_threshold; int64_t log_file_percentage = config.log_file_percentage; - if (!is_in_single_service) { - if (total_size > 0) { - double proportion = static_cast(cur_process_log_size) / static_cast(total_size); - thresh_hold = static_cast(static_cast(thresh_hold) * proportion); - total_size = cur_process_log_size; - } - } if (OB_LIKELY(avail_size >= 0) && OB_LIKELY(log_file_percentage > 0)) { if (avail_size <= min_avail_size) { LOG_WARN("disk avail size is too small!!!", K(avail_size)); @@ -328,6 +483,7 @@ int ObLogFileProcessor::do_cleanup_log_file(ObArray &log_a thresh_hold = std::min(thresh_hold, ((total_size + avail_size) / 100) * log_file_percentage); } if (thresh_hold < total_size) { + std::sort(log_array.begin(), log_array.end()); LOG_INFO("begin to delete log file", K(thresh_hold), K(total_size)); int64_t low_water_mark = thresh_hold / 2; int64_t i = 0; diff --git a/src/obproxy/obutils/ob_log_file_processor.h b/src/obproxy/obutils/ob_log_file_processor.h index 0cf43c79a91a18bc6639b65907ffb51e0f99a66f..9d7bb84cbee4740b9a6c5c14cd7f6635a390e618 100644 --- a/src/obproxy/obutils/ob_log_file_processor.h +++ b/src/obproxy/obutils/ob_log_file_processor.h @@ -13,6 +13,7 @@ #ifndef OBPROXY_LOG_FILE_PROCESSOR_H #define OBPROXY_LOG_FILE_PROCESSOR_H #include "lib/container/ob_array.h" +#include "lib/compress/zstd_1_3_8/ob_zstd_compressor_1_3_8.h" #include "utils/ob_layout.h" #include "iocore/eventsystem/ob_buf_allocator.h" @@ -72,14 +73,23 @@ private: //sorted array until total_size_ is smaller than threshhold //if in hot upgrade status, log_size_threshhold is divided by both parent and child //process according to their proportions in total size - int do_cleanup_log_file(common::ObArray &log_array, const bool is_in_single_service, - const int64_t cur_process_log_size, int64_t &total_size); + int do_cleanup_log_file(common::ObArray &log_array, + int64_t &total_size); int do_cleanup_invalid_log_file(common::ObArray &log_array); + int do_cleanup_compress_log_file(common::ObArray &log_array, + common::ObArray &compress_log_array, + int64_t &total_size); + int log_compress_block(char *dest, size_t dest_size, + const char *src, size_t src_size, + size_t &return_size); + int log_compress(ObProxyLogFileStruct &log_st, const common::ObString &file_name, const common::ObString &compression_file_name, + int64_t &total_size, char *src_buf, int src_size, char *dest_buf, int dest_size); private: static const int64_t MAX_INVALID_PROXY_LOG_NUM = 10; bool is_inited_; + common::zstd_1_3_8::ObZstdCompressor_1_3_8 zstd_compressor_1_3_8_; obutils::ObAsyncCommonTask *cleanup_cont_; DISALLOW_COPY_AND_ASSIGN(ObLogFileProcessor); }; diff --git a/src/obproxy/obutils/ob_proxy_buf.h b/src/obproxy/obutils/ob_proxy_buf.h index 006ea1a5f7ff0d1821ad7bcaa9d8096d72f6de82..98aa0f6643c5d6a87e2ca385184e3aac6bf8f878 100644 --- a/src/obproxy/obutils/ob_proxy_buf.h +++ b/src/obproxy/obutils/ob_proxy_buf.h @@ -58,10 +58,8 @@ template inline void ObVariableLenBuffer::reset() { if (OB_LIKELY(is_inited_)) { - if (OB_LIKELY(NULL != buf_)) { - if (total_len_ > BUF_LEN) { - op_fixed_mem_free(buf_, total_len_); - } + if (total_len_ > BUF_LEN && NULL != buf_) { + op_fixed_mem_free(buf_, total_len_); } buf_ = NULL; total_len_ = 0; @@ -175,7 +173,7 @@ public: int64_t len() const { return valid_len_; } int64_t total_len() const { return BUF_LEN; } - void reset() { valid_len_ = 0; } + inline void reset() { valid_len_ = 0; } private: char buf_[BUF_LEN]; diff --git a/src/obproxy/obutils/ob_proxy_config.h b/src/obproxy/obutils/ob_proxy_config.h index 60d46b822a921a29a64b75e49ffb9121760484ff..a27f28d11b88ddba84de091e991e2c01e69586ea 100644 --- a/src/obproxy/obutils/ob_proxy_config.h +++ b/src/obproxy/obutils/ob_proxy_config.h @@ -57,8 +57,6 @@ enum ObProxyLocalCMDType OB_LOCAL_CMD_NONE = 0, OB_LOCAL_CMD_EXIT, OB_LOCAL_CMD_RESTART, - OB_LOCAL_CMD_COMMIT, - OB_LOCAL_CMD_ROLLBACK, OB_LOCAL_CMD_MAX }; @@ -129,8 +127,6 @@ public: ObProxyLocalCMDType get_local_cmd_type() const; void reset_local_cmd(); bool is_local_restart(); - bool is_local_commit(); - bool is_local_rollback(); bool is_metadb_used() const { return with_config_server_ && enable_metadb_used; } bool is_control_plane_used() const { return with_control_plane_; } @@ -155,6 +151,7 @@ public: //repeat task interval related DEF_TIME(proxy_info_check_interval, "60s", "[1s,1h]", "proxy info check task interval, [1s, 1h]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_TIME(proxy_hot_upgrade_check_interval, "5s", "[1s,1h]", "proxy info check task interval, [1s, 1h]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(cache_cleaner_clean_interval, "20s", "[1s, 1d]", "the interval for cache cleaner to clean cache, [1s, 1d]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(server_state_refresh_interval, "20s", "[10ms, 1h]", "the interval to refresh server state for getting zone or server newest state, [10ms, 1h]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(metadb_server_state_refresh_interval, "60s", "[10ms, 1h]", "the interval to refresh metadb server state for getting zone or server newest state, [10ms, 1h]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); @@ -171,13 +168,17 @@ public: DEF_TIME(fetch_proxy_bin_timeout, "120s", "[1s,1200s]", "default hot upgrade fetch binary timeout, proxy will stop fetching after such long time, [1s, 1200s]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_INT(hot_upgrade_failure_retries, "5", "[1,20]", "default hot upgrade failure retries, proxy will stop handle hot_upgrade command after such retries, [1, 20]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(hot_upgrade_rollback_timeout, "24h", "[1s,30d]", "default hot upgrade rollback timeout, proxy will do rollback if receive no rollback command in such long time, [1s, 30d]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_TIME(hot_upgrade_graceful_exit_timeout, "120s", "[0s,30d]", "graceful exit timeout, [0s, 30d], if set a value <= 0, proxy treat it as 0", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(hot_upgrade_exit_timeout, "30000000", "[-1,86400000000]", "graceful exit timeout, unit is us, default 30s, " + "-1 means no timeout, 0 means quit now, > 0 means wait time", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(delay_exit_time, "100ms", "[100ms,500ms]", "delay exit time, [100ms,500ms]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); //log cleanup related DEF_INT(log_file_percentage, "80", "[0, 100]", "max percentage of avail size occupied by proxy log file, [0, 90], 0 means ignore such limit", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_TIME(log_cleanup_interval, "10m","[5s,30d]", "log file clean up task schedule interval, set 1 day or longer, [5s, 30d]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_TIME(log_cleanup_interval, "1m","[5s,30d]", "log file clean up task schedule interval, set 1 day or longer, [5s, 30d]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_CAP(log_dir_size_threshold, "64GB", "[256M,1T]", "max usable space size of log dir, used to decide whether should clean up log file, [256MB, 1T]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_TIME(max_syslog_file_time, "7d","[1s,]", "Maximum retention time of archive logs, 0 means ignore such limit", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(max_syslog_file_count, "0", "[0,]", "Maximum number of archive files to keep, 0 means ignore such limit", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_BOOL(enable_syslog_file_compress, "false", "Whether to enable archive log compression", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); //resource pool related DEF_BOOL(need_convert_vip_to_tname, "false", "convert vip to tenant name, which is useful in cloud", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); @@ -188,7 +189,7 @@ public: //client session related DEF_BOOL(enable_client_connection_lru_disconnect, "false", "if client connections reach throttle, true is that new connection will be accepted, and eliminate lru client connection, false is that new connection will disconnect, and err packet will be returned", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); - DEF_INT(client_max_connections, "8192", "[0,65535]", "client max connections for one obproxy, [0, 65535]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_max_connections, "8192", "[0,]", "client max connections for one obproxy, [0, +∞]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_TIME(observer_query_timeout_delta, "20s", "[1s,30s]", "the delta value for @@ob_query_timeout, to cover net round trip time(proxy<->server) and task schedule time(server), [1s, 30s]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_BOOL(enable_cluster_checkout, "true", "if enable cluster checkout, proxy will send cluster name when login and server will check it", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_BOOL(enable_proxy_scramble, "false", "if enable proxy scramble, proxy will send client its variable scramble num, not support old observer", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); @@ -204,20 +205,20 @@ public: DEF_TIME(default_inactivity_timeout, "180000s", "[1s,30d]", "default inactivity timeout, [1s, 30d]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_CAP(sock_recv_buffer_size_out, "0", "[0,8MB]", "sock param, recv buffer size, [0, 8MB], if set a negative value, proxy treat it as 0", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_CAP(sock_send_buffer_size_out, "0", "[0,8MB]", "sock param, send buffer size, [0, 8MB], if set a negative value, proxy treat it as 0", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(sock_option_flag_out, "1", "[0,]","sock param, option flag out, bit 1: NO_DELAY, bit 2: KEEP_ALIVE, bit 3: LINGER_ON", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(sock_option_flag_out, "3", "[0,]","sock param, option flag out, bit 1: NO_DELAY, bit 2: KEEP_ALIVE, bit 3: LINGER_ON", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_INT(sock_packet_mark_out, "0", "[0,]","sock param, packet mark out, [0, 1]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_INT(sock_packet_tos_out, "0", "[0,]","sock param, packet tos out, [0, 1]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_INT(server_tcp_init_cwnd, "0", "[0,64]", "the initial tcp congestion window, [0, 64]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(server_tcp_keepidle, "0", "[0,7200]", "tcp keepalive idle time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(server_tcp_keepintvl, "0", "[0,75]", "tcp keepalive interval time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(server_tcp_keepcnt, "0", "[0,9]", "tcp keepalive probe count, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(server_tcp_user_timeout, "5", "[0,20]", "tcp user timeout, unit is s, 0 means no user timeout", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(server_tcp_keepidle, "5", "[0,7200]", "tcp keepalive idle time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(server_tcp_keepintvl, "5", "[0,75]", "tcp keepalive interval time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(server_tcp_keepcnt, "5", "[0,9]", "tcp keepalive probe count, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(server_tcp_user_timeout, "0", "[0,20]", "tcp user timeout, unit is s, 0 means no user timeout", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(client_sock_option_flag_out, "0", "[0,]","client sock param, option flag out, bit 1: NO_DELAY, bit 2: KEEP_ALIVE", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(client_tcp_keepidle, "0", "[0,7200]", "client tcp keepalive idle time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(client_tcp_keepintvl, "0", "[0,75]", "client tcp keepalive interval time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(client_tcp_keepcnt, "0", "[0,9]", "client tcp keepalive probe count, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); - DEF_INT(client_tcp_user_timeout, "5", "[0,20]", "client tcp user timeout, unit is s, 0 means no user timeout", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_sock_option_flag_out, "2", "[0,]","client sock param, option flag out, bit 1: NO_DELAY, bit 2: KEEP_ALIVE", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_tcp_keepidle, "5", "[0,7200]", "client tcp keepalive idle time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_tcp_keepintvl, "5", "[0,75]", "client tcp keepalive interval time, unit is second, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_tcp_keepcnt, "5", "[0,9]", "client tcp keepalive probe count, 0 means use default value by kernel", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_INT(client_tcp_user_timeout, "0", "[0,20]", "client tcp user timeout, unit is s, 0 means no user timeout", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); //proxy init related DEF_CAP(proxy_mem_limited, "2G", "[100MB,100G]", "proxy memory limited, [100MB, 100G]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); @@ -286,6 +287,7 @@ public: DEF_BOOL(enable_ob_protocol_v2, "true", "if enabled, proxy will use oceanbase protocol 2.0 with server", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_BOOL(enable_reroute, "false", "if this and protocol_v2 enabled, proxy will reroute when routing error", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_BOOL(enable_pl_route, "true", "if enabled, pl will be accurate routing", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_BOOL(enable_cached_server, "true", "if enabled, use cached server session when no table entry", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); // location rate limit DEF_INT(normal_pl_update_threshold, "100", "[0,]", "max partition location update task processing per second", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); @@ -304,7 +306,7 @@ public: DEF_BOOL(enable_async_log, "true", "if enabled, use async logging way, maybe lost some log when busy", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); // proxy cmd - DEF_INT(proxy_local_cmd, "0", "[0,]", "proxy local cmd type: 0->none(default), 1->exit, 2->restart, 3->commit, 4->rollback", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_MEMORY); + DEF_INT(proxy_local_cmd, "0", "[0,2]", "proxy local cmd type: 0->none(default), 1->exit, 2->restart", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_MEMORY); //ldc DEF_STR(proxy_idc_name, "", "idc name for proxy ldc route. If is empty or invalid, treat as do not use ldc. User session vars 'proxy_session_ldc' can cover it", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); @@ -353,6 +355,7 @@ public: DEF_INT(sql_table_cache_expire_relative_time, "0", "[-36000000,36000000]", "the unit is ms, 0 means do not expire, others will expire sql table cache base on relative time", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); DEF_CAP(sql_table_cache_mem_limited, "128MB", "[1KB,100G]", "max size of proxy sql table cache size. [1KB, 100G]", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_BOOL(enable_cloud_full_username, "false", "used for cloud user, if set false, treat all login user as username", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); + DEF_BOOL(enable_full_username, "false", "used for non-cloud user, if set true, username must have tenant and cluster", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); DEF_BOOL(skip_proxyro_check, "false", "used for proxro@sys, if set false, access denied", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); DEF_BOOL(skip_proxy_sys_private_check, "true", "skip_proxy_sys_private_check", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); @@ -401,6 +404,9 @@ public: DEF_STR(pod_namespace, "", "sidecar pod namespace", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_STR(instance_ip, "", "ip addr string", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); DEF_STR(runtime_env, "", "runtime env", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_STR(mng_url, "", "ob sharding console url", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_STR(cloud_instance_id, "", "ob sharding cloud instance id", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); + DEF_BOOL(auto_scan_all, "false", "if enabled, need scan all", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); // session pool DEF_BOOL(is_pool_mode, "false", "if enabled means useing session pool", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_USER); @@ -438,6 +444,14 @@ public: DEF_STR(obproxy_sys_password, "", "password for obproxy sys user", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); DEF_STR(observer_sys_password, "", "password for observer sys user", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); DEF_STR(observer_sys_password1, "", "password for observer sys user", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); + + // performance_mode has two functions: competition and verification, it will turn off some auxiliary functions, + // but these functions are required in production; another case is to verify whether the optimization method is effective + DEF_BOOL(enable_performance_mode, "false", "if enabled, for performance situation", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); + DEF_BOOL(enable_trace, "true", "if enabled, log will print trace info", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); + DEF_BOOL(enable_stat, "true", "if enabled, will collect stat info", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); + + DEF_INT(digest_sql_length, "1024", "0 means use default print sql len, otherwise is digest_sql_length", CFG_NO_NEED_REBOOT, CFG_SECTION_OBPROXY, CFG_VISIBLE_LEVEL_SYS); }; ObProxyConfig &get_global_proxy_config(); @@ -569,16 +583,6 @@ inline bool ObProxyConfig::is_local_restart() return (OB_LOCAL_CMD_RESTART == get_local_cmd_type()); } -inline bool ObProxyConfig::is_local_commit() -{ - return (OB_LOCAL_CMD_COMMIT == get_local_cmd_type()); -} - -inline bool ObProxyConfig::is_local_rollback() -{ - return (OB_LOCAL_CMD_ROLLBACK == get_local_cmd_type()); -} - inline ObProxyLocalCMDType ObProxyConfig::get_local_cmd_type() const { return static_cast(proxy_local_cmd.get()); diff --git a/src/obproxy/obutils/ob_proxy_config_processor.cpp b/src/obproxy/obutils/ob_proxy_config_processor.cpp index 3b8d7379645af493f249cb94eba6dac1afd43aca..9595a8398b38d2f40f89ff11b5ce3f1e895e9224 100644 --- a/src/obproxy/obutils/ob_proxy_config_processor.cpp +++ b/src/obproxy/obutils/ob_proxy_config_processor.cpp @@ -24,6 +24,8 @@ #include "utils/ob_proxy_utils.h" #include "utils/ob_proxy_monitor_utils.h" #include "iocore/net/ob_ssl_processor.h" +#include "obutils/ob_config_processor.h" +#include "omt/ob_ssl_config_table_processor.h" using namespace oceanbase::common; using namespace oceanbase::json; @@ -32,6 +34,7 @@ using namespace oceanbase::obproxy::event; using namespace oceanbase::obproxy::qos; using namespace oceanbase::obproxy::net; using namespace oceanbase::obmysql; +using namespace oceanbase::obproxy::omt; namespace oceanbase { @@ -411,6 +414,11 @@ int ObProxyBaseConfig::load_from_local(const char *file_path, ObProxyAppConfig & } else if (OB_FAIL(app_config.update_config(json_str, type_, is_from_local))) { LOG_WARN("fail to load local config", K(json_str), K(file_path), K(ret)); } + + if (NULL != buf) { + ob_free(buf); + } + return ret; } @@ -602,6 +610,14 @@ int ObProxyDynamicConfig::assign(const ObProxyDynamicConfig &other) } //---------------ObProxyLimitControlConfig-------------- +ObProxyLimitConfig::~ObProxyLimitConfig() +{ + int64_t count = cond_array_.count(); + for (int64_t i = 0; i < count; i++) { + cond_array_.at(i)->~ObProxyQosCond(); + } +} + int64_t ObProxyLimitConfig::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; @@ -616,7 +632,7 @@ int64_t ObProxyLimitConfig::to_string(char *buf, const int64_t buf_len) const int ObProxyLimitConfig::init(ObArenaAllocator &allocator) { int ret = OB_SUCCESS; - if (OB_FAIL(limit_rule_map_.create(LIMIT_RULE_MAP_BUCKET, ObModIds::OB_HASH_BUCKET_CONF_CONTAINER))) { + if (OB_FAIL(limit_rule_map_.create(LIMIT_RULE_MAP_BUCKET, ObModIds::OB_PROXY_QOS, ObModIds::OB_PROXY_QOS))) { LOG_WARN("fail to init limit rule map", K(ret)); } else { allocator_ = &allocator; @@ -916,6 +932,14 @@ int ObProxyLimitConfig::calc(ObMysqlTransact::ObTransState &trans_state, const O return ret; } +ObProxyLimitControlConfig::~ObProxyLimitControlConfig() +{ + int64_t count = limit_config_array_.count(); + for (int64_t i = 0; i < count; i++) { + limit_config_array_.at(i)->~ObProxyLimitConfig(); + } +} + int64_t ObProxyLimitControlConfig::to_string(char *buf, const int64_t buf_len) const { int64_t pos = 0; @@ -1476,15 +1500,8 @@ int ObProxyConfigProcessor::get_app_config_string(const ObString &app_name, ret = app_config->fuse_control_config_.to_json_str(buf); break; case SECURITY_CONFIG: - if (!security_config_.ca_.empty() && !security_config_.public_key_.empty() - && !security_config_.private_key_.empty()) { - if (security_config_.source_type_.get_string() == "DBMESH") { - ret = buf.append_fmt("SSL INFO DBMESH"); - } else if (security_config_.source_type_.get_string() == "FILE") { - ret = buf.append_fmt("SSL INFO FILE"); - } else { - ret = OB_ERR_UNEXPECTED; - } + if (get_global_ssl_config_table_processor().is_ssl_key_info_valid("*", "*")) { + ret = buf.append_fmt("SSL INFO VALID"); } else { ret = buf.append_fmt("SSL INFO INVALID"); } @@ -2157,12 +2174,31 @@ int ObProxyConfigProcessor::update_app_security_config(const ObString &appname, if (!security_config_.source_type_.get_string().empty() && !security_config_.ca_.get_string().empty() && !security_config_.public_key_.get_string().empty() && !security_config_.private_key_.get_string().empty()) { - if (OB_FAIL(g_ssl_processor.update_key(security_config_.source_type_.get_string(), - security_config_.ca_.get_string(), - security_config_.public_key_.get_string(), - security_config_.private_key_.get_string()))) { - LOG_WARN("update key failed", K(ret), K(security_config_.source_type_), K(security_config_.ca_), - K(security_config_.public_key_), K(security_config_.private_key_)); + const int64_t json_len = security_config_.source_type_.get_string().length() + + security_config_.ca_.get_string().length() + + security_config_.public_key_.get_string().length() + + security_config_.private_key_.get_string().length() + 256; + char *json_buf = (char*)ob_malloc(json_len + 1, ObModIds::OB_PROXY_CONFIG_TABLE); + if (OB_ISNULL(json_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate json_buf failed", K(ret), K(json_len)); + } else { + const char *json_string = "{\"sourceType\": \"%.*s\", \"CA\": \"%.*s\", \"publicKey\": \"%.*s\", \"privateKey\": \"%.*s\"}"; + int64_t len = static_cast(snprintf(json_buf, json_len, json_string, + security_config_.source_type_.get_string().length(), security_config_.source_type_.get_string().ptr(), + security_config_.ca_.get_string().length(), security_config_.ca_.get_string().ptr(), + security_config_.public_key_.get_string().length(), security_config_.public_key_.get_string().ptr(), + security_config_.private_key_.get_string().length(), security_config_.private_key_.get_string().ptr())); + if (OB_UNLIKELY(len <= 0 || len > json_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fill sql failed", K(len), K(json_len), K(ret)); + } else if (OB_FAIL(get_global_config_processor().store_cloud_config("ssl_config", "*", "*", "key_info", json_buf))) { + LOG_WARN("execute sql failed", K(ret)); + } + } + + if (OB_NOT_NULL(json_buf)) { + ob_free(json_buf); } } } diff --git a/src/obproxy/obutils/ob_proxy_config_processor.h b/src/obproxy/obutils/ob_proxy_config_processor.h index 610561cdf8e02e8a2a548c897e0fc301d6dce737..cb036bf9fd57aaff58cc5899657ac903fed94b58 100644 --- a/src/obproxy/obutils/ob_proxy_config_processor.h +++ b/src/obproxy/obutils/ob_proxy_config_processor.h @@ -209,6 +209,7 @@ class ObProxyLimitConfig { public: ObProxyLimitConfig() : action_(NULL), + cond_array_(ObModIds::OB_PROXY_QOS, OB_MALLOC_NORMAL_BLOCK_SIZE), limit_mode_(LIMIT_MODE_INVALID), limit_priority_(-1), limit_qps_(-1), limit_status_(LIMIT_STATUS_INVALID), limit_time_window_(-1), limit_conn_(0), limit_fuse_time_(0), allocator_(NULL) {}; @@ -307,9 +308,11 @@ private: class ObProxyLimitControlConfig : public ObProxyBaseConfig { public: - ObProxyLimitControlConfig() : ObProxyBaseConfig(LIMIT_CONTROL_CONFIG) {} - ObProxyLimitControlConfig(ObProxyConfigType type) : ObProxyBaseConfig(type) {} - virtual ~ObProxyLimitControlConfig() {} + ObProxyLimitControlConfig() : ObProxyBaseConfig(LIMIT_CONTROL_CONFIG), allocator_(ObModIds::OB_PROXY_QOS), + limit_config_array_(ObModIds::OB_PROXY_QOS, OB_MALLOC_NORMAL_BLOCK_SIZE) {} + ObProxyLimitControlConfig(ObProxyConfigType type) : ObProxyBaseConfig(type), allocator_(ObModIds::OB_PROXY_QOS), + limit_config_array_(ObModIds::OB_PROXY_QOS, OB_MALLOC_NORMAL_BLOCK_SIZE) {} + virtual ~ObProxyLimitControlConfig(); int64_t to_string(char *buf, const int64_t buf_len) const; virtual int parse_config_spec(json::Value &json_value); diff --git a/src/obproxy/obutils/ob_proxy_create_server_conn_cont.cpp b/src/obproxy/obutils/ob_proxy_create_server_conn_cont.cpp index ac4247815827ef2f827e5d55d2082b85bc45653f..cea2ea89a41d469eae4647690547ac217afde0c9 100644 --- a/src/obproxy/obutils/ob_proxy_create_server_conn_cont.cpp +++ b/src/obproxy/obutils/ob_proxy_create_server_conn_cont.cpp @@ -420,7 +420,7 @@ int ObProxyCreateServerConnCont::do_create_server_conn() K(schema_key_conn_info_->addr_)); } } else if (DB_MYSQL == server_type) { - if (OB_FAIL(proxy->rebuild_client_pool(schema_key.shard_conn_, + if (OB_FAIL(proxy->rebuild_client_pool(schema_key.shard_conn_, NULL, is_meta_mysql_client, user_name, passwd_string, database_name, "", &client_pool_option))) { LOG_WARN("fail to create mysql client pool", K(user_name), K(database_name), K(ret)); } else { diff --git a/src/obproxy/obutils/ob_proxy_refresh_server_addr_cont.h b/src/obproxy/obutils/ob_proxy_refresh_server_addr_cont.h index 0ec2ce9eff22f2a8cec193d4eeda4266da5971ba..e1f98f1964cf68a8b652231e367f0962d819dbfd 100644 --- a/src/obproxy/obutils/ob_proxy_refresh_server_addr_cont.h +++ b/src/obproxy/obutils/ob_proxy_refresh_server_addr_cont.h @@ -79,4 +79,4 @@ int global_proxy_session_create_task(); } // end of namespace obutils } // end of namespace obproxy } // end of namespace oceanbase -#endif //OB_PROXY_FRESH_SERVER_ADDR_CONT_H_ \ No newline at end of file +#endif //OB_PROXY_FRESH_SERVER_ADDR_CONT_H_ diff --git a/src/obproxy/obutils/ob_proxy_sequence_entry_cont.cpp b/src/obproxy/obutils/ob_proxy_sequence_entry_cont.cpp index d007f6b38e11a2722d12dbeb5157dcaa324a6393..eea8862b0759282dbb614a27b739a9dc10a3130f 100644 --- a/src/obproxy/obutils/ob_proxy_sequence_entry_cont.cpp +++ b/src/obproxy/obutils/ob_proxy_sequence_entry_cont.cpp @@ -166,7 +166,7 @@ bool ObProxySequenceEntryCont::need_create_cluster_resource() { if (OB_UNLIKELY(OB_ISNULL(client_pool))) { LOG_WARN("client_pool is NULL", K(sequence_info_.seq_id_)); } else { - ObClusterResource *cluster_resource = dynamic_cast(client_pool->acquire_connection_param()); + ObClusterResource *cluster_resource = client_pool->acquire_cluster_resource(); if (OB_UNLIKELY(OB_ISNULL(cluster_resource))) { LOG_WARN("cluster_resource is NULL", K(sequence_info_.seq_id_)); } else if (cluster_resource->is_avail()) { @@ -292,7 +292,7 @@ int ObProxySequenceEntryCont::rebuild_proxy(ObMysqlProxy *proxy, ObSharedRefCoun K_(database_name), K(ret)); } } else { - if (OB_FAIL(proxy->rebuild_client_pool(dynamic_cast(param), is_meta_mysql_client, + if (OB_FAIL(proxy->rebuild_client_pool(dynamic_cast(param), NULL, is_meta_mysql_client, username, passwd, database_name_))) { LOG_WARN("fail to create mysql client pool", K(username), K_(database_name), K(ret)); } @@ -882,6 +882,7 @@ int ObProxySequenceEntryCont::main_handler(int event, void *data) data = NULL; // fail through, do not break } + __attribute__ ((fallthrough)); case CLIENT_TRANSPORT_MYSQL_RESP_EVENT: { if (OB_FAIL(handle_client_resp(data))) { LOG_WARN("fail to handle client resp", K(ret), K(sequence_info_.seq_id_)); diff --git a/src/obproxy/obutils/ob_proxy_sql_parser.cpp b/src/obproxy/obutils/ob_proxy_sql_parser.cpp index cc470b07186b714e1e4849306e48a95e55a1942e..eab47c2bfb56d229c5cc4d93191e421da9d023bc 100644 --- a/src/obproxy/obutils/ob_proxy_sql_parser.cpp +++ b/src/obproxy/obutils/ob_proxy_sql_parser.cpp @@ -16,6 +16,7 @@ #include "obutils/ob_proxy_stmt.h" #include "opsql/parser/ob_proxy_parser.h" #include "dbconfig/ob_proxy_db_config_info.h" +#include "proxy/shard/obproxy_shard_utils.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::opsql; @@ -103,13 +104,13 @@ inline int ObSqlParseResult::set_db_table_name(const ObProxyParseString &databas } table_name_.assign_ptr(dml_buf_.table_name_buf_, table_name.str_len_); table_name_quote_ = table_name.quote_type_; - if (!drop_origin_db_table_name) { + if (OB_LIKELY(!drop_origin_db_table_name)) { MEMCPY(origin_dml_buf_.table_name_buf_, table_name.str_, table_name.str_len_); origin_table_name_.assign_ptr(origin_dml_buf_.table_name_buf_, table_name.str_len_); } // assign package name when table name is valid - if (NULL != package_name.str_ && 0 != package_name.str_len_) { + if (OB_UNLIKELY(NULL != package_name.str_ && 0 != package_name.str_len_)) { if (OB_UNLIKELY(package_name.str_len_ > OB_MAX_TABLE_NAME_LENGTH) || OB_UNLIKELY(package_name.str_len_ < 0)) { ret = OB_INVALID_ARGUMENT; @@ -131,7 +132,7 @@ inline int ObSqlParseResult::set_db_table_name(const ObProxyParseString &databas } } - if (NULL != alias_name.str_ && 0 != alias_name.str_len_) { + if (OB_UNLIKELY(NULL != alias_name.str_ && 0 != alias_name.str_len_)) { if (OB_UNLIKELY(alias_name.str_len_ > OB_MAX_TABLE_NAME_LENGTH) || OB_UNLIKELY(alias_name.str_len_ < 0)) { ret = OB_INVALID_ARGUMENT; @@ -191,7 +192,7 @@ inline int ObSqlParseResult::set_call_prarms(const ObProxyCallParseInfo &call_pa } else if (OB_UNLIKELY(call_parse_info.node_count_ < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(call_parse_info.node_count_), K(ret)); - } else if (call_parse_info.node_count_ > 0) { + } else if (call_parse_info.node_count_ >= 0) { call_info_.is_param_valid_ = true; call_info_.param_count_ = call_parse_info.node_count_; ObProxyCallParam tmp_param; @@ -566,27 +567,27 @@ int ObSqlParseResult::load_result(const ObProxyParseResult &parse_result, parsed_length_ = static_cast(parse_result.end_pos_ - parse_result.start_pos_); text_ps_inner_stmt_type_ = parse_result.text_ps_inner_stmt_type_; - if (NULL != parse_result.table_info_.table_name_.str_ && parse_result.table_info_.table_name_.str_len_ > 0) { + if (OB_UNLIKELY(is_sharding_request && NULL != parse_result.table_info_.table_name_.str_ && parse_result.table_info_.table_name_.str_len_ > 0)) { dbmesh_route_info_.tb_pos_ = parse_result.table_info_.table_name_.str_ - parse_result.start_pos_; } - if (NULL != parse_result.trace_id_.str_ + if (OB_UNLIKELY(NULL != parse_result.trace_id_.str_ && 0 < parse_result.trace_id_.str_len_ - && OB_MAX_OBPROXY_TRACE_ID_LENGTH > parse_result.trace_id_.str_len_) { + && OB_MAX_OBPROXY_TRACE_ID_LENGTH > parse_result.trace_id_.str_len_)) { MEMCPY(trace_id_buf_, parse_result.trace_id_.str_, parse_result.trace_id_.str_len_); trace_id_.assign_ptr(trace_id_buf_, parse_result.trace_id_.str_len_); } - if (NULL != parse_result.rpc_id_.str_ + if (OB_UNLIKELY(NULL != parse_result.rpc_id_.str_ && 0 < parse_result.rpc_id_.str_len_ - && OB_MAX_OBPROXY_TRACE_ID_LENGTH > parse_result.rpc_id_.str_len_) { + && OB_MAX_OBPROXY_TRACE_ID_LENGTH > parse_result.rpc_id_.str_len_)) { MEMCPY(rpc_id_buf_, parse_result.rpc_id_.str_, parse_result.rpc_id_.str_len_); rpc_id_.assign_ptr(rpc_id_buf_, parse_result.rpc_id_.str_len_); } // if is dml stmt, then set db/table name - if (is_dml_stmt() || is_call_stmt() || (is_text_ps_stmt() && is_text_ps_inner_dml_stmt()) - || is_show_create_table_stmt() || is_desc_table_stmt()) { + if (OB_LIKELY(is_dml_stmt() || is_call_stmt() || (is_text_ps_stmt() && is_text_ps_inner_dml_stmt()) + || is_show_create_table_stmt() || is_desc_table_stmt())) { if (OB_FAIL(set_db_table_name(parse_result.table_info_.database_name_, parse_result.table_info_.package_name_, parse_result.table_info_.table_name_, @@ -594,12 +595,12 @@ int ObSqlParseResult::load_result(const ObProxyParseResult &parse_result, use_lower_case_name, drop_origin_db_table_name))) { LOG_WARN("failed to set db table name", K(use_lower_case_name), K(ret)); - } else if (OB_FAIL(set_dbmesh_route_info(parse_result))) { + } else if (OB_UNLIKELY(is_sharding_request && OB_FAIL(set_dbmesh_route_info(parse_result)))) { LOG_WARN("fail to set dbmesh route info", K(ret)); if (is_sharding_request) { stmt_type_ = OBPROXY_T_INVALID; } - } else if (is_call_stmt()) { + } else if (OB_UNLIKELY(is_call_stmt())) { if (OB_FAIL(set_call_prarms(parse_result.call_parse_info_))) { LOG_WARN("failed to set_call_prarms", K(ret)); } @@ -665,7 +666,7 @@ int ObSqlParseResult::load_result(const ObProxyParseResult &parse_result, } } } - } + } return ret; } @@ -850,7 +851,7 @@ int ObProxySqlParser::get_parse_allocator(ObArenaAllocator *&allocator) { int ret = OB_SUCCESS; static __thread ObArenaAllocator *arena_allocator = NULL; - if (NULL == arena_allocator) { + if (OB_UNLIKELY(NULL == arena_allocator)) { if (NULL == (arena_allocator = new (std::nothrow) ObArenaAllocator(common::ObModIds::OB_PROXY_SQL_PARSE))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc arena allocator", K(ret)); @@ -890,7 +891,8 @@ int ObProxySqlParser::parse_sql(const ObString &sql, int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = obproxy_parser.parse(sql, obproxy_parse_result, connection_collation))) { LOG_INFO("fail to parse sql, will go on anyway", K(sql), K(tmp_ret)); - } else if (OB_SUCCESS != (tmp_ret = sql_parse_result.load_result(obproxy_parse_result, use_lower_case_name, + } else if (OB_SUCCESS != (tmp_ret = sql_parse_result.load_result(obproxy_parse_result, + use_lower_case_name, drop_origin_db_table_name, is_sharding_request))) { LOG_INFO("fail to load result, will go on anyway", K(sql), K(use_lower_case_name), K(tmp_ret)); } else { @@ -899,25 +901,9 @@ int ObProxySqlParser::parse_sql(const ObString &sql, allocator->reuse(); } - //int64_t end_time = ObTimeUtility::current_time(); - //LOG_TRACE("finish parse sql", "total cost time(us)", end_time - start_time, - // K(sql), K(sql_parse_result), K(ret)); - if (OB_SUCC(ret) && need_parser_by_obparser(sql_parse_result)) { - if (OB_FAIL(parse_sql_by_obparser(sql, parse_mode, sql_parse_result))) { - LOG_WARN("parse_sql_by_obparser failed", K(ret), K(sql)); - } - } return ret; } -bool ObProxySqlParser::need_parser_by_obparser(ObSqlParseResult &sql_parse_result) -{ - return (sql_parse_result.is_select_stmt() && sql_parse_result.get_dbp_route_info().scan_all_) - || (sql_parse_result.get_dbmesh_route_info().testload_ != oceanbase::obproxy::dbconfig::TESTLOAD_NON - && sql_parse_result.get_dbmesh_route_info().testload_ != OBPROXY_MAX_DBMESH_ID) - ; //ADDED BY FUTURE -} - int ObProxySqlParser::init_ob_parser_node(ObArenaAllocator &allocator, ObParseNode *&ob_node) { int ret = OB_SUCCESS; @@ -1109,7 +1095,7 @@ int ObProxySqlParser::ob_load_testload_parse_node(ParseNode *node, const int lev return ret; } -int ObSqlParseResult::get_result_tree_str(ParseNode *root, const int level, char* buf, int& pos, int64_t length) +int ObSqlParseResult::get_result_tree_str(ParseNode *root, const int level, char* buf, int64_t& pos, int64_t length) { int ret = OB_SUCCESS; if (NULL == root || NULL == buf || length < 0) { @@ -1117,20 +1103,36 @@ int ObSqlParseResult::get_result_tree_str(ParseNode *root, const int level, char } else { for (int i = 0 ; i < level *2; i++) { pos += snprintf(buf + pos, length - pos, "-"); + if (OB_UNLIKELY(pos >= length)) { + pos = length - 1; + break; + } } pos += snprintf (buf + pos, length - pos, " %s %s:pos:%ld, text_len:%ld, num_child_:%d, token_off:%d, token_len:%d\n", get_type_name(root->type_), root->str_value_, root->pos_, root->text_len_, root->num_child_, root->token_off_, root->token_len_); + if (OB_UNLIKELY(pos >= length)) { + pos = length - 1; + } + for (int i = 0; OB_SUCC(ret) && i < root->num_child_; i++) { if (NULL == root->children_[i]) { for (int i = 0 ; i < (level + 1) *2; i++) { pos += snprintf(buf + pos, length - pos, "-"); + if (OB_UNLIKELY(pos >= length)) { + pos = length - 1; + break; + } } pos += snprintf(buf + pos, length - pos, " There is one NULL child node\n"); - } else if (OB_FAIL(get_result_tree_str(root->children_[i], level + 1, buf, pos, length - pos))) { - LOG_WARN("get_result_tree_str failed", K(ret), K(i)); + if (OB_UNLIKELY(pos >= length)) { + pos = length - 1; + break; + } + } else if (OB_FAIL(get_result_tree_str(root->children_[i], level + 1, buf, pos, length))) { + LOG_WARN("get_result_tree_str failed", K(ret), K(i), K(pos), K(length)); } } } @@ -1138,10 +1140,9 @@ int ObSqlParseResult::get_result_tree_str(ParseNode *root, const int level, char } int ObSqlParseResult::ob_parse_resul_to_string(const ParseResult &parse_result, const common::ObString& sql, - char* buf, int64_t buf_len) + char* buf, int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; - int pos = 0; if (OB_FAIL(get_result_tree_str(parse_result.result_tree_, 0, buf, pos, buf_len))) { LOG_WARN("get_result_tree_str failed", K(ret), K(sql)); } else if (parse_result.comment_list_ != NULL) { @@ -1149,23 +1150,30 @@ int ObSqlParseResult::ob_parse_resul_to_string(const ParseResult &parse_result, TokenPosInfo& token_info = parse_result.comment_list_[j]; pos += snprintf(buf + pos, buf_len - pos, "comment[%d] is %.*s\n", j, token_info.token_len_, sql.ptr()+ token_info.token_off_); + if (OB_UNLIKELY(pos >= buf_len)) { + pos = buf_len - 1; + break; + } } } return ret; } int ObSqlParseResult::load_ob_parse_result(const ParseResult &parse_result, - const common::ObString& sql) + const common::ObString& sql, + const bool need_handle_result) { int ret = OB_SUCCESS; - const int64_t buf_len = 64 * 1024; - char tree_str_buf[buf_len]; //just for debug , ignore ret - ob_parse_resul_to_string(parse_result, sql, tree_str_buf, buf_len); - ObString tree_str(buf_len, tree_str_buf); - LOG_DEBUG("result_tree_ is \n", K(tree_str)); + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + const int64_t buf_len = 64 * 1024; + int64_t pos = 0; + char tree_str_buf[buf_len]; + ob_parse_resul_to_string(parse_result, sql, tree_str_buf, buf_len, pos); + ObString tree_str(pos, tree_str_buf); + LOG_DEBUG("result_tree_ is \n", K(tree_str)); + } ParseNode* node = parse_result.result_tree_; - ObProxySelectStmt* select_stmt= NULL; if (OB_ISNULL(node)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("node is unexpected null", K(ret)); @@ -1176,25 +1184,31 @@ int ObSqlParseResult::load_ob_parse_result(const ParseResult &parse_result, ret = OB_ERR_UNEXPECTED; LOG_DEBUG("unexpected null child", K(ret)); } else { - char* buf = NULL; switch(node->type_) { case T_SELECT: - if (OB_ISNULL(buf = (char*)allocator_.alloc(sizeof(ObProxySelectStmt)))) { - LOG_WARN("failed to alloc buf"); - ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_ISNULL(select_stmt = new (buf) ObProxySelectStmt())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to new ObProxySelectStmt", K(ret)); - } else if (OB_FAIL(select_stmt->init())) { - LOG_WARN("init failed", K(ret)); - } else { - select_stmt->set_sql_string(sql); - select_stmt->set_stmt_type(OBPROXY_T_SELECT); - select_stmt->set_allocator(&allocator_); - select_stmt->field_results_ = &fileds_result_; - proxy_stmt_ = select_stmt; - if (OB_FAIL(proxy_stmt_->handle_parse_result(parse_result))) { - LOG_WARN("handle_parse_result failed", K(ret)); + if (need_handle_result) { + char* buf = NULL; + ObProxySelectStmt* select_stmt= NULL; + if (OB_ISNULL(buf = (char*)allocator_.alloc(sizeof(ObProxySelectStmt)))) { + LOG_WARN("failed to alloc buf"); + ret = OB_ALLOCATE_MEMORY_FAILED; + } else if (OB_ISNULL(select_stmt = new (buf) ObProxySelectStmt())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new ObProxySelectStmt", K(ret)); + } else if (OB_FAIL(select_stmt->init())) { + LOG_WARN("init failed", K(ret)); + } else { + select_stmt->set_sql_string(sql); + select_stmt->set_stmt_type(OBPROXY_T_SELECT); + select_stmt->set_allocator(&allocator_); + select_stmt->field_results_ = &fileds_result_; + select_stmt->set_table_name(origin_table_name_); + proxy_stmt_ = select_stmt; + if (OB_FAIL(proxy_stmt_->handle_parse_result(parse_result))) { + LOG_WARN("handle_parse_result failed", K(ret)); + } else { + has_for_update_ = select_stmt->has_for_update(); + } } } break; @@ -1211,7 +1225,8 @@ int ObSqlParseResult::load_ob_parse_result(const ParseResult &parse_result, int ObProxySqlParser::parse_sql_by_obparser(const ObString &sql, const ObProxyParseMode parse_mode, - ObSqlParseResult &sql_parse_result) + ObSqlParseResult &sql_parse_result, + const bool need_handle_result) { int ret = OB_SUCCESS; //int64_t start_time = ObTimeUtility::current_time(); @@ -1232,8 +1247,11 @@ int ObProxySqlParser::parse_sql_by_obparser(const ObString &sql, ParseResult &ob_parse_result = *parser_result; if (OB_FAIL(obproxy_parser.obparse(sql, ob_parse_result))) { + if (OB_ERR_PARSE_SQL == ret) { + ret = OB_ERR_PARSER_SYNTAX; + } LOG_INFO("fail to parse sql", K(sql), K(ret)); - } else if (OB_FAIL(sql_parse_result.load_ob_parse_result(ob_parse_result, sql))) { + } else if (OB_FAIL(sql_parse_result.load_ob_parse_result(ob_parse_result, sql, need_handle_result))) { LOG_WARN("fail to load_ob_parse_result", K(sql), K(ret)); } else { LOG_DEBUG("success to do parse_sql_by_obparser", K(sql)); diff --git a/src/obproxy/obutils/ob_proxy_sql_parser.h b/src/obproxy/obutils/ob_proxy_sql_parser.h index a8559c87f1334e54a2a2357168484bcd3b20c028..9a763c992fb6e7dddd65b1024dc6d85d9a2f76a6 100644 --- a/src/obproxy/obutils/ob_proxy_sql_parser.h +++ b/src/obproxy/obutils/ob_proxy_sql_parser.h @@ -23,6 +23,7 @@ #include "common/ob_hint.h" #include "lib/utility/ob_print_utils.h" #include "utils/ob_proxy_lib.h" +#include "obutils/ob_proxy_string_utils.h" #include #include #include @@ -279,7 +280,7 @@ struct ObProxySimpleRouteInfo && part_key_offset_ > 0 && part_key_len_ > 0; } - void reset() + inline void reset() { table_offset_ = 0; table_len_ = 0; @@ -339,7 +340,7 @@ struct SqlField { // TODO: erase deprecated ObProxyTokenType value_type_; int64_t column_int_value_; - ObFixSizeString column_value_; + obutils::ObProxyVariantString column_value_; }; struct SqlFieldResult { @@ -514,7 +515,8 @@ typedef struct _ObProxyDualParseResult struct ObSqlParseResult { - ObSqlParseResult() : ob_parser_result_(NULL), proxy_stmt_(NULL) { reset(); } + ObSqlParseResult() : allocator_(common::ObModIds::OB_PROXY_SHARDING_PARSE), + ob_parser_result_(NULL), proxy_stmt_(NULL) { reset(); } ~ObSqlParseResult() { reset(); } void clear_proxy_stmt(); void reset(bool is_reset_origin_db_table = true); @@ -543,6 +545,7 @@ struct ObSqlParseResult bool is_show_trace_stmt() const { return OBPROXY_T_SHOW_TRACE == stmt_type_; } bool is_show_session_stmt() const { return OBPROXY_T_ICMD_SHOW_SESSION == stmt_type_; } bool is_select_tx_ro() const { return OBPROXY_T_SELECT_TX_RO == stmt_type_; } + bool is_select_proxy_version() const { return OBPROXY_T_SELECT_PROXY_VERSION == stmt_type_; } bool is_set_autocommit_0() const { return OBPROXY_T_SET_AC_0 == stmt_type_; } bool is_select_route_addr() const { return OBPROXY_T_SELECT_ROUTE_ADDR == stmt_type_; } bool is_set_route_addr() const { return OBPROXY_T_SET_ROUTE_ADDR == stmt_type_; } @@ -598,7 +601,7 @@ struct ObSqlParseResult bool is_text_ps_prepare_stmt() const { return OBPROXY_T_TEXT_PS_PREPARE == stmt_type_; } bool is_text_ps_execute_stmt() const { return OBPROXY_T_TEXT_PS_EXECUTE == stmt_type_; } - bool is_internal_select() const { return is_select_tx_ro(); } + bool is_internal_select() const { return is_select_tx_ro() || is_select_proxy_version(); } bool is_dual_request() const {return is_dual_request_;} bool is_internal_request() const; bool is_dml_stmt() const; @@ -628,6 +631,7 @@ struct ObSqlParseResult bool has_simple_route_info() const { return has_simple_route_info_; } bool has_shard_comment() const { return has_shard_comment_; } bool has_anonymous_block() const { return has_anonymous_block_; } + bool has_for_update() const { return has_for_update_; } bool is_simple_route_info_valid() const { return route_info_.is_valid(); } @@ -698,10 +702,11 @@ struct ObSqlParseResult ObProxyBasicStmtType get_text_ps_inner_stmt_type() const { return text_ps_inner_stmt_type_; } int load_ob_parse_result(const ParseResult &parse_result, - const common::ObString& sql); + const common::ObString& sql, + const bool need_handle_result); static int ob_parse_resul_to_string(const ParseResult &parse_result, const common::ObString& sql, - char* buf, int64_t buf_len); - static int get_result_tree_str(ParseNode *root, const int level, char* buf, int& pos, int64_t length); + char* buf, int64_t buf_len, int64_t &pos); + static int get_result_tree_str(ParseNode *root, const int level, char* buf, int64_t &pos, int64_t length); SqlFieldResult& get_sql_filed_result() { return fileds_result_; } int64_t get_batch_insert_values_count() { return batch_insert_values_count_; } void set_batch_insert_values_count(int64_t count) { batch_insert_values_count_ = count; } @@ -726,6 +731,7 @@ struct ObSqlParseResult has_explain_ = other.has_explain_; has_simple_route_info_ = other.has_simple_route_info_; has_anonymous_block_ = other.has_anonymous_block_; + has_for_update_ = other.has_for_update_; stmt_type_ = other.stmt_type_; hint_query_timeout_ = other.hint_query_timeout_; parsed_length_ = other.parsed_length_; @@ -817,6 +823,7 @@ private: bool has_shard_comment_; bool is_dual_request_; bool has_anonymous_block_; + bool has_for_update_; ObProxyBasicStmtType stmt_type_; int64_t hint_query_timeout_; int64_t parsed_length_; // next parser can starts with (orig_sql + parsed_length_) @@ -894,8 +901,8 @@ public: int parse_sql_by_obparser(const common::ObString &sql, const ObProxyParseMode parse_mode, - ObSqlParseResult &sql_parse_result); - bool need_parser_by_obparser(ObSqlParseResult &sql_parse_result); + ObSqlParseResult &sql_parse_result, + const bool need_handle_result); typedef common::hash::ObHashMap AliasTableMap; static int ob_load_testload_parse_node(ParseNode *root, const int level, common::ObSEArray &relation_table_node, @@ -918,6 +925,7 @@ inline void ObSqlParseResult::reset(bool is_reset_origin_db_table /* true */) has_shard_comment_ = false; is_dual_request_ = false; has_anonymous_block_ = false; + has_for_update_ = false; stmt_type_ = OBPROXY_T_INVALID; cmd_sub_type_ = OBPROXY_T_SUB_INVALID; cmd_err_type_ = OBPROXY_T_ERR_INVALID; diff --git a/src/obproxy/obutils/ob_proxy_stmt.cpp b/src/obproxy/obutils/ob_proxy_stmt.cpp index 3e807bc5b104fdb211e57aa8c7bc069391a26a0f..efd3f78881bfa1a9ee2321ae8ef4b045b23edb17 100644 --- a/src/obproxy/obutils/ob_proxy_stmt.cpp +++ b/src/obproxy/obutils/ob_proxy_stmt.cpp @@ -24,80 +24,77 @@ namespace obutils { int ObProxyDMLStmt::condition_exprs_to_sql_string(common::ObSqlString& sql_string) { + UNUSED(sql_string); int ret = OB_SUCCESS; - if (condition_exprs_.count() <= 0) { - // do nothing - } else if (condition_exprs_.count() != 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected count", K(condition_exprs_.count())); - } else if (OB_FAIL(sql_string.append(" WHERE "))) { - LOG_WARN("append failed", K(ret)); - } else { - for (int i = 0; OB_SUCC(ret) && i < condition_exprs_.count(); i++) { - if (OB_FAIL(condition_exprs_.at(i)->to_sql_string(sql_string))) { - LOG_WARN("to sql_string failed", K(i), K(ret)); - } - } - } return ret; } int ObProxyDMLStmt::limit_to_sql_string(common::ObSqlString& sql_string) { int ret = OB_SUCCESS; - if (limit_offset_ > 0) { - if (OB_FAIL(sql_string.append_fmt(" LIMIT %d, %d", limit_start_, limit_offset_))) { + if (limit_size_ > 0) { + if (OB_FAIL(sql_string.append_fmt(" LIMIT %d, %d", limit_offset_, limit_size_))) { LOG_WARN("fail to append", K(ret)); } } return ret; } -ObProxySelectStmt::ObProxySelectStmt() : is_inited_(false), parse_phase_(SELECT_FIELD_PHASE), has_rollup_(false) +ObProxySelectStmt::ObProxySelectStmt() : is_inited_(false), has_rollup_(false), + has_for_update_(false), from_token_off_(-1) { } ObProxySelectStmt::~ObProxySelectStmt() { - alias_col_map_.destroy(); + for (int64_t i = 0; i < select_exprs_.count(); i++) { + ObProxyExpr *expr = select_exprs_.at(i); + expr->~ObProxyExpr(); + } + + for (int64_t i = 0; i < group_by_exprs_.count(); i++) { + ObProxyGroupItem *group_expr = group_by_exprs_.at(i); + group_expr->~ObProxyGroupItem(); + } + + for (int64_t i = 0; i < order_by_exprs_.count(); i++) { + ObProxyOrderItem *order_expr = order_by_exprs_.at(i); + order_expr->~ObProxyOrderItem(); + } + + ExprMap::iterator iter = table_exprs_map_.begin(); + ExprMap::iterator end = table_exprs_map_.end(); + for (; iter != end; iter++) { + ObProxyExpr *expr = iter->second; + expr->~ObProxyExpr(); + } + + table_exprs_map_.destroy(); + alias_table_map_.destroy(); } + int ObProxySelectStmt::init() { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("init twice", K(ret)); - } else if (OB_FAIL(alias_col_map_.create(BUCKET_SIZE, ObModIds::OB_HASH_BUCKET_PROXY_MAP))) { - LOG_WARN("hash map init failed", K(ret)); + } else if (OB_FAIL(table_exprs_map_.create(BUCKET_SIZE, ObModIds::OB_HASH_BUCKET_PROXY_MAP))) { + LOG_WARN("fail to init table expr map", K(ret)); + } else if (OB_FAIL(alias_table_map_.create(BUCKET_SIZE, ObModIds::OB_HASH_BUCKET_PROXY_MAP))) { + LOG_WARN("fail to init alias table set", K(ret)); } else { is_inited_ = true; } return ret; } -int ObProxySelectStmt::handle_where_end_pos(ParseNode* node) -{ - int ret = OB_SUCCESS; - where_end_pos_ = sql_string_.length(); - if (node->num_child_ > PARSE_SELECT_GROUP && node->children_[PARSE_SELECT_GROUP] != NULL) { - where_end_pos_ = node->children_[PARSE_SELECT_GROUP]->token_off_; - } else if (node->num_child_ > PARSE_SELECT_HAVING && node->children_[PARSE_SELECT_HAVING] != NULL) { - where_end_pos_ = node->children_[PARSE_SELECT_HAVING]->token_off_; - } else if (node->num_child_ > PARSE_SELECT_ORDER && node->children_[PARSE_SELECT_ORDER] != NULL) { - where_end_pos_ = node->children_[PARSE_SELECT_ORDER]->token_off_; - } else if (node->num_child_ >PARSE_SELECT_LIMIT && node->children_[PARSE_SELECT_LIMIT] != NULL) { - where_end_pos_ = node->children_[PARSE_SELECT_LIMIT]->token_off_; - } - return ret; -} int ObProxySelectStmt::handle_parse_result(const ParseResult &parse_result) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; ParseNode* node = parse_result.result_tree_->children_[0]; - if (OB_FAIL(handle_where_end_pos(node))) { - LOG_WARN("handle_where_end_pos failed", K(ret), K(sql_string_)); - } + for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; if (NULL == tmp_node) { @@ -105,6 +102,22 @@ int ObProxySelectStmt::handle_parse_result(const ParseResult &parse_result) } else if (i == PARSE_SELECT_HAVING) { ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; LOG_WARN("having not support", K(ret), K(sql_string_)); + } else { + switch(tmp_node->type_) { + case T_FROM_LIST: + from_token_off_ = tmp_node->token_off_; + ret = handle_from_list(tmp_node); + break; + default: + break; + } + } + } + + for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { + tmp_node = node->children_[i]; + if (NULL == tmp_node) { + // do nothing } else { switch(tmp_node->type_) { case T_HINT_OPTION_LIST: @@ -114,7 +127,6 @@ int ObProxySelectStmt::handle_parse_result(const ParseResult &parse_result) ret = handle_project_list(tmp_node); break; case T_FROM_LIST: - ret = handle_from_list(tmp_node); break; case T_WHERE_CLAUSE: ret = handle_where_clause(tmp_node); @@ -129,6 +141,9 @@ int ObProxySelectStmt::handle_parse_result(const ParseResult &parse_result) case T_LIMIT_CLAUSE: ret = handle_limit_clause(tmp_node); break; + case T_SFU_INT: + has_for_update_ = true; + break; case T_QEURY_EXPRESSION_LIST: //distinct not support now default: ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; @@ -168,14 +183,8 @@ int ObProxySelectStmt::comments_to_sql_string(common::ObSqlString& sql_string) int ObProxySelectStmt::hint_exprs_to_sql_string(common::ObSqlString& sql_string) { + UNUSED(sql_string); int ret = OB_SUCCESS; - if (hint_string_.empty()) { - // do nothing - } else if (OB_FAIL(sql_string.append("/*"))) { - LOG_WARN("fail to append", K(ret), K(sql_string_)); - } else if (OB_FAIL(sql_string.append(hint_string_))) { - LOG_WARN("fail to append", K(ret), K(hint_string_), K(sql_string_)); - } return ret; } int ObProxySelectStmt::select_exprs_to_sql_string(common::ObSqlString& sql_string) @@ -199,23 +208,8 @@ int ObProxySelectStmt::select_exprs_to_sql_string(common::ObSqlString& sql_strin } int ObProxySelectStmt::table_exprs_to_sql_string(common::ObSqlString& sql_string) { + UNUSED(sql_string); int ret = OB_SUCCESS; - if (table_exprs_.count() != 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected count ,now only support 1", K(table_exprs_.count()), K(sql_string_)); - } else if (OB_FAIL(sql_string.append(" FROM "))) { - LOG_WARN("fail to append", K(ret), K(sql_string_)); - } else { - for (int i = 0; OB_SUCC(ret) && i < table_exprs_.count(); i++) { - if (OB_FAIL(table_exprs_.at(i)->to_sql_string(sql_string))) { - LOG_WARN("to sql_string failed", K(i), K(ret), K(sql_string_)); - } else if (i >= table_exprs_.count() - 1) { - // no need add , do nothing - } else if (OB_FAIL(sql_string.append(", "))) { - LOG_WARN("fail to append", K(i), K(ret), K(sql_string_)); - } - } - } return ret; } int ObProxySelectStmt::group_by_exprs_to_sql_string(common::ObSqlString& sql_string) @@ -295,8 +289,26 @@ int ObProxySelectStmt::to_sql_string(common::ObSqlString& sql_string) int ObProxySelectStmt::handle_hint_clause(ParseNode* node) { int ret = OB_SUCCESS; - hint_string_.assign_ptr(sql_string_.ptr() + node->token_off_, node->token_len_); - LOG_DEBUG("handle_hint_clause", K(hint_string_)); + ParseNode* tmp_node = NULL; + + for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { + tmp_node = node->children_[i]; + if (NULL == tmp_node) { + // do nothing + } else { + switch(tmp_node->type_) { + case T_RELATION_FACTOR: + if (OB_FAIL(handle_table_and_db_in_hint(tmp_node))) { + LOG_WARN("fail to handle table and db in hint", K(ret)); + } + break; + default: + ret = handle_hint_clause(tmp_node); + break; + } + } + } + return ret; } @@ -305,6 +317,7 @@ int ObProxySelectStmt::handle_limit_clause(ParseNode* node) int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; int64_t tmp_val = 0; + limit_token_off_ = node->token_off_; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; if (NULL == tmp_node) { @@ -315,14 +328,14 @@ int ObProxySelectStmt::handle_limit_clause(ParseNode* node) if(OB_FAIL(get_int_value(tmp_node->str_value_, tmp_val))) { LOG_WARN("get_int_value failed", K(ret), K(tmp_node->str_value_), K(sql_string_)); } else { - limit_start_ = static_cast(tmp_val); + limit_offset_ = static_cast(tmp_val); } break; case T_LIMIT_INT: if(OB_FAIL(get_int_value(tmp_node->str_value_, tmp_val))) { LOG_WARN("get_int_value failed", K(ret), K(tmp_node->str_value_), K(sql_string_)); } else { - limit_offset_ = static_cast(tmp_val); + limit_size_ = static_cast(tmp_val); } break; default: @@ -339,7 +352,6 @@ int ObProxySelectStmt::handle_project_list(ParseNode* node) int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; ObProxyExpr* expr = NULL; - parse_phase_ = SELECT_FIELD_PHASE; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; expr = NULL; @@ -349,33 +361,34 @@ int ObProxySelectStmt::handle_project_list(ParseNode* node) switch(tmp_node->type_) { case T_PROJECT_STRING: if (OB_FAIL(project_string_to_expr(tmp_node, expr))) { - LOG_WARN("project_string_to_expr failed", K(ret), K(sql_string_)); + LOG_WARN("project_string_to_expr failed", K(sql_string_), K(ret)); } else if (OB_FAIL(select_exprs_.push_back(expr))) { - LOG_WARN("push to array failed", K(ret), K(sql_string_)); - } else { - LOG_DEBUG("add expr succ", K(tmp_node->str_value_)); + LOG_WARN("push to array failed", K(sql_string_), K(ret)); } break; default: ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; LOG_WARN("unsupport type", "node_type", get_type_name(tmp_node->type_), K(sql_string_)); } + + if (OB_FAIL(ret) && OB_NOT_NULL(expr)) { + expr->~ObProxyExpr(); + } } } return ret; } -int ObProxySelectStmt::get_sharding_const_expr(ParseNode* node, ObProxyExpr* &expr, bool is_column_ref) +int ObProxySelectStmt::get_const_expr(ParseNode* node, ObProxyExpr* &expr) { int ret = OB_SUCCESS; - ObProxyExpr* real_alias_col_expr = NULL; - ObProxyExprShardingConst* expr_const = NULL; + ObProxyExprConst* expr_const = NULL; if (OB_ISNULL(node)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("unexpected null", K(sql_string_), K(ret)); - } else if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_SHARDING_CONST))) { + } else if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_CONST))) { LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); - } else if (OB_ISNULL(expr_const = dynamic_cast(expr))) { + } else if (OB_ISNULL(expr_const = dynamic_cast(expr))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("dynamic_cast failed", K(ret)); } else { @@ -390,25 +403,33 @@ int ObProxySelectStmt::get_sharding_const_expr(ParseNode* node, ObProxyExpr* &ex LOG_DEBUG("add int value", K(obj)); } } else { - obj.set_varchar(node->str_value_); + ObString var(node->token_len_, node->str_value_); + obj.set_varchar(var); expr_const->set_object(obj); - expr_const->set_is_column(is_column_ref); LOG_DEBUG("add varchar value", K(obj)); } - if (parse_phase_ >= GROUP_BY_PHASE) { - ObString col_name(node->str_value_); - if (OB_SUCCESS == alias_col_map_.get_refactored(col_name, real_alias_col_expr)) { - // is alias - expr_const->expr_ = real_alias_col_expr; - expr_const->has_alias_ = 1; - expr_const->set_is_alias(true); - LOG_DEBUG("succ set expr for alias", K(col_name)); - } - } - expr = expr_const; } return ret; } + +int ObProxySelectStmt::get_sharding_const_expr(ParseNode* node, ObProxyExpr* &expr) +{ + int ret = OB_SUCCESS; + ObProxyExprShardingConst* expr_const = NULL; + if (OB_ISNULL(node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null", K(sql_string_), K(ret)); + } else if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_SHARDING_CONST))) { + LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); + } else if (OB_ISNULL(expr_const = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + expr_const->set_expr_name(node->str_value_, node->token_len_); + } + return ret; +} + ObProxyExprType ObProxySelectStmt::get_expr_type_by_node_type(const ObItemType& item_type) { ObProxyExprType expr_type = OB_PROXY_EXPR_TYPE_NONE; @@ -462,11 +483,20 @@ int ObProxySelectStmt::get_expr_by_type(ObProxyExpr* &expr, ObProxyExprType type } switch(type) { + case OB_PROXY_EXPR_TYPE_CONST: + ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprConst); + break; case OB_PROXY_EXPR_TYPE_SHARDING_CONST: ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprShardingConst); break; - case OB_PROXY_EXPR_TYPE_SHARDING_ALIAS: - ALLOC_PROXY_EXPR_BY_TYPE(ObProxyShardingAliasExpr); + case OB_PROXY_EXPR_TYPE_TABLE: + ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprTable); + break; + case OB_PROXY_EXPR_TYPE_COLUMN: + ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprColumn); + break; + case OB_PROXY_EXPR_TYPE_STAR: + ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprStar); break; case OB_PROXY_EXPR_TYPE_FUNC_ADD: ALLOC_PROXY_EXPR_BY_TYPE(ObProxyExprAdd); @@ -498,6 +528,9 @@ int ObProxySelectStmt::get_expr_by_type(ObProxyExpr* &expr, ObProxyExprType type case OB_PROXY_EXPR_TYPE_FUNC_ORDER: ALLOC_PROXY_EXPR_BY_TYPE(ObProxyOrderItem); break; + case OB_PROXY_EXPR_TYPE_FUNC_GROUP: + ALLOC_PROXY_EXPR_BY_TYPE(ObProxyGroupItem); + break; default: ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; LOG_WARN("unexpected type", K(type)); @@ -506,76 +539,152 @@ int ObProxySelectStmt::get_expr_by_type(ObProxyExpr* &expr, ObProxyExprType type return ret; } -int ObProxySelectStmt::column_ref_to_expr(ParseNode* node, ObProxyExpr* &expr) +int ObProxySelectStmt::handle_table_and_db_node(ParseNode* node, ObProxyExprTable* &expr_table) { int ret = OB_SUCCESS; - ParseNode* tmp_node = NULL; - for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { - tmp_node = node->children_[i]; - if (NULL == tmp_node) { - } else if (T_IDENT != tmp_node->type_) { + + ParseNode* db_node = node->children_[0]; + ParseNode* table_node = node->children_[1]; + if (OB_ISNULL(table_node)) { + // do nothing + } else if (OB_UNLIKELY(T_IDENT != table_node->type_ || NULL == table_node->str_value_ || 0 >= table_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table node", "node type", table_node->type_, "token len", table_node->token_len_, K(ret)); + } else if (NULL != db_node + && OB_UNLIKELY(T_IDENT != db_node->type_ || NULL == db_node->str_value_ || 0 >= db_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected db node", "node type", db_node->type_, "token len", db_node->token_len_, K(ret)); + } else { + ObProxyExpr* expr = NULL; + ObString table_name = ObString::make_string(table_node->str_value_); + if (OB_SUCCESS == alias_table_map_.get_refactored(table_name, expr)) { + if (OB_ISNULL(expr_table = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } + } else if (OB_SUCCESS == table_exprs_map_.get_refactored(table_name, expr)) { + if (OB_ISNULL(expr_table = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + // Only the real table name needs to be rewritten, not the alias + ObProxyExprTablePos expr_table_pos; + if (NULL != db_node) { + expr_table_pos.set_database_pos(db_node->token_off_); + } + expr_table_pos.set_table_pos(table_node->token_off_); + expr_table_pos.set_table_expr(expr_table); + if (OB_FAIL(table_pos_array_.push_back(expr_table_pos))) { + LOG_WARN("fail to push expr table pos", K(ret)); + } + } + } else { + ret = OB_ERR_BAD_FIELD_ERROR; + LOG_WARN("table name of column is not alias name or real table name", K(table_name), K(ret)); + } + } + + return ret; +} + +int ObProxySelectStmt::handle_table_and_db_in_hint(ParseNode* node) +{ + int ret = OB_SUCCESS; + + ObProxyExprTable* expr_table = NULL; + + if (OB_T_RELATION_FACTOR_NUM_CHILD /* 2 */ != node->num_child_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children node is not 2", "child num", node->num_child_, K(ret)); + } else if (OB_FAIL(handle_table_and_db_node(node, expr_table))) { + LOG_WARN("fail to handle table and db node", K(ret)); + } + + return ret; +} + +int ObProxySelectStmt::column_ref_to_expr(ParseNode* node, ObProxyExpr* &expr, ObProxyExprTable* &expr_table) +{ + int ret = OB_SUCCESS; + + if (OB_T_COLUMN_REF_NUM_CHILD /* 3 */ != node->num_child_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_COLUMN_REF children node is not 3", "child num", node->num_child_, K(ret)); + } else if (OB_FAIL(handle_table_and_db_node(node, expr_table))) { + LOG_WARN("fail to handle table and db node", K(ret)); + } + + if (OB_SUCC(ret)) { + ParseNode* table_node = node->children_[1]; + ParseNode* column_node = node->children_[2]; + if (OB_ISNULL(column_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_COLUMN_REF unexpected column entry", K(ret)); + } else if (T_IDENT != column_node->type_ && T_STAR != column_node->type_) { // now column_ref child should be T_IDENT ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unsupport type", "node_type", get_type_name(tmp_node->type_), K(i), K(tmp_node->str_value_), K(sql_string_)); - } else if (NULL != expr) { - //column_ref should have only one T_IDENT - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr_const already allocate, unexpected", K(i), K(tmp_node->str_value_), K(node->num_child_)); - } else if (OB_FAIL(get_sharding_const_expr(tmp_node, expr, true))) { - LOG_WARN("get_sharding_const_expr failed", K(ret), K(sql_string_)); + LOG_WARN("T_COLUMN_REF unexpected column entry", "node_type", get_type_name(column_node->type_), K(ret)); + } else if (T_IDENT == column_node->type_) { + ObProxyExprColumn* expr_column = NULL; + if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_COLUMN))) { + LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); + } else if (OB_ISNULL(expr_column = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + expr_column->set_column_name(column_node->str_value_, column_node->token_len_); + if (OB_NOT_NULL(table_node)) { + expr_column->set_table_name(table_node->str_value_, table_node->token_len_); + if (NULL != expr_table) { + expr_column->set_real_table_name(expr_table->get_table_name()); + } + } + } + } else if (T_STAR == column_node->type_) { + ObProxyExprStar* expr_star = NULL; + if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_STAR))) { + LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); + } else if (OB_ISNULL(expr_star = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + if (OB_NOT_NULL(table_node)) { + expr_star->set_table_name(table_node->str_value_, table_node->token_len_); + } + } } } + return ret; } + int ObProxySelectStmt::alias_node_to_expr(ParseNode* node, ObProxyExpr* &expr, ParseNode* string_node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - ObProxyShardingAliasExpr* alias_expr = NULL; - ObProxyExpr* tmp_expr = NULL; - if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_SHARDING_ALIAS))) { - LOG_WARN("get_expr_by_type failed", K(ret)); - } else if (OB_ISNULL(alias_expr = dynamic_cast(expr))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("dynamic_cast failed", K(ret)); - } - ObString alias_col; + expr = NULL; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; - tmp_expr = NULL; if (NULL == tmp_node) { //do nothing } else { switch(tmp_node->type_) { case T_IDENT: - // here is alias node - if (OB_FAIL(get_sharding_const_expr(tmp_node, tmp_expr, true))) { //here is_column true - LOG_WARN("get_sharding_const_expr failed", K(i), K(ret)); - } else if (parse_phase_ == SELECT_FIELD_PHASE) { - // in select record alias - alias_col.assign_ptr(tmp_node->str_value_, static_cast(strlen(tmp_node->str_value_))); - if (OB_FAIL(alias_col_map_.set_refactored(alias_col, alias_expr))) { //put origin expr into map - LOG_WARN("set set_refactored failed", K(alias_col), K(ret)); - } else { - LOG_DEBUG("add alias to map", K(alias_col)); - } + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get expr table", K(ret)); + } else if (OB_UNLIKELY(NULL == tmp_node->str_value_ || 0 >= tmp_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("alias token meet some wrong", "token len", tmp_node->token_len_, K(ret)); + } else { + expr->set_alias_name(tmp_node->str_value_, tmp_node->token_len_); } break; default: - if (OB_FAIL(string_node_to_expr(tmp_node, tmp_expr, string_node))) { + if (OB_FAIL(string_node_to_expr(tmp_node, expr, string_node))) { LOG_WARN("string_node_to_expr failed", K(i), K(ret)); } } - if (OB_FAIL(ret)) { - // already log before, do nothing - } else if (OB_FAIL(alias_expr->add_param_expr(tmp_expr))) { - LOG_WARN("add_param_expr failed", K(ret), K(sql_string_)); - } else { - if (tmp_expr->has_agg_) { - alias_expr->has_agg_ = 1; - } - LOG_DEBUG("add alias node success", K(tmp_node->str_value_), K(alias_expr->has_agg_)); - } } } return ret; @@ -585,8 +694,8 @@ int ObProxySelectStmt::func_node_to_expr(ParseNode* node, ObProxyExpr* &expr) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - ObProxyFuncExpr* func_expr = NULL; ObProxyExpr* tmp_expr = NULL; + ObProxyFuncExpr* func_expr = NULL; ObProxyExprType expr_type = get_expr_type_by_node_type(node->type_); if (OB_FAIL(get_expr_by_type(expr, expr_type))) { LOG_WARN("get_expr_by_type failed", K(ret)); @@ -595,16 +704,16 @@ int ObProxySelectStmt::func_node_to_expr(ParseNode* node, ObProxyExpr* &expr) LOG_WARN("dynamic_cast failed", K(ret)); } else { switch(node->type_) { - case T_FUN_SUM: - case T_FUN_COUNT: - case T_FUN_MAX: - case T_FUN_MIN: - case T_FUN_AVG: - func_expr->has_agg_ = 1; - break; - default: - func_expr->has_agg_ = 0; - break; + case T_FUN_SUM: + case T_FUN_COUNT: + case T_FUN_MAX: + case T_FUN_MIN: + case T_FUN_AVG: + func_expr->has_agg_ = 1; + break; + default: + func_expr->has_agg_ = 0; + break; } } for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { @@ -622,12 +731,13 @@ int ObProxySelectStmt::func_node_to_expr(ParseNode* node, ObProxyExpr* &expr) if (tmp_expr->has_agg_) { func_expr->has_agg_ = 1; } - if (tmp_expr->has_alias_) { - func_expr->has_alias_ = 1; - } - LOG_DEBUG("add node success", K(get_expr_type_name(expr_type)), K(func_expr->has_agg_), K(func_expr->has_alias_)); } } + + if (OB_SUCC(ret)) { + func_expr->set_expr_name(node->str_value_, node->token_len_); + } + return ret; } int ObProxySelectStmt::check_node_has_agg(ParseNode* node) @@ -649,7 +759,7 @@ int ObProxySelectStmt::check_node_has_agg(ParseNode* node) case T_FUN_MAX: case T_FUN_MIN: case T_FUN_AVG: - ret = OB_ERROR_UNSUPPORT_HAS_AGG_EXPR_TYPE; + ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; LOG_WARN("node has agg", "node_type", get_type_name(tmp_node->type_), K(ret), K(sql_string_)); break; default: @@ -669,10 +779,10 @@ int ObProxySelectStmt::func_sys_node_to_expr(ParseNode* node, ObProxyExpr* &expr if (OB_FAIL(check_node_has_agg(node))) { LOG_WARN("check_node_has_agg failed", K(ret)); } else if (string_node == NULL) { - if (OB_FAIL(get_sharding_const_expr(node, expr, true))) { + if (OB_FAIL(get_sharding_const_expr(node, expr))) { LOG_WARN("get_sharding_const_expr faield", K(ret)); } - } else if (OB_FAIL(get_sharding_const_expr(string_node, expr, true))) { + } else if (OB_FAIL(get_sharding_const_expr(string_node, expr))) { LOG_WARN("get_sharding_const_expr faield", K(ret)); } return ret; @@ -684,45 +794,40 @@ int ObProxySelectStmt::string_node_to_expr(ParseNode* node, ObProxyExpr* &expr, int i = 0; switch(node->type_) { case T_STAR: - node->str_value_ = "*"; - node->str_len_ = 1; - if (OB_SUCC(get_sharding_const_expr(node, expr, true))) { - LOG_DEBUG("get sharding const expr succ", "node_str", node->str_value_); + if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_STAR))) { + LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); } break; - case T_COLUMN_REF: - if (OB_SUCC(column_ref_to_expr(node, expr))) { - LOG_DEBUG("column_ref to expr succ", "node_str", node->str_value_); + case T_COLUMN_REF: { + ObProxyExprTable* expr_table = NULL; + if (OB_FAIL(column_ref_to_expr(node, expr, expr_table))) { + LOG_WARN("fail to column ref to expr", K(ret)); } break; + } case T_ALL: break; case T_ALIAS: - if (OB_SUCC(alias_node_to_expr(node, expr, string_node))) { - LOG_DEBUG("alias_node to expr succ", "node_str", node->str_value_); + if (OB_FAIL(alias_node_to_expr(node, expr, string_node))) { + LOG_WARN("fail to alias node to expr", K(ret)); } break; case T_FUN_SYS: // not support fun sys, as string_node - if (OB_SUCC(func_sys_node_to_expr(node, expr, string_node))) { - LOG_DEBUG("func_sys node to expr succ", "node_str", node->str_value_); - } - break; - case T_RELATION_FACTOR: - if (OB_SUCC(get_sharding_const_expr(node, expr, true))) { - LOG_DEBUG("get sharding const expr succ", "node_str", node->str_value_); + if (OB_FAIL(func_sys_node_to_expr(node, expr, string_node))) { + LOG_WARN("fail to func sys node to expr", K(ret)); } break; case T_IDENT: case T_INT: - if (OB_SUCC(get_sharding_const_expr(node, expr, false))) { - LOG_DEBUG("get sharding const expr succ", "node_str", node->str_value_); + if (OB_FAIL(get_const_expr(node, expr))) { + LOG_WARN("fail to get sharding const expr", K(ret)); } break; case T_VARCHAR: //varchar has varchar child, if child is 0 not have child if (node->num_child_ == 0) { - if (OB_SUCC(get_sharding_const_expr(node, expr, false))) { - LOG_DEBUG("get sharding const expr succ", "node_str", node->str_value_); + if (OB_FAIL(get_const_expr(node, expr))) { + LOG_WARN("fail to get sharding const expr succ", K(ret)); } } else { for (i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { @@ -731,8 +836,8 @@ int ObProxySelectStmt::string_node_to_expr(ParseNode* node, ObProxyExpr* &expr, } else if (expr != NULL) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr should null", K(ret), K(i), K(node->str_value_), K(sql_string_)); - } else if (OB_SUCC(get_sharding_const_expr(node->children_[i], expr, false))) { - LOG_DEBUG("get sharding const expr succ", "node_str", node->children_[i]->str_value_); + } else if (OB_FAIL(get_const_expr(node->children_[i], expr))) { + LOG_WARN("fail to get sharding const expr succ", K(ret)); } } } @@ -746,8 +851,8 @@ int ObProxySelectStmt::string_node_to_expr(ParseNode* node, ObProxyExpr* &expr, case T_FUN_MAX: case T_FUN_MIN: case T_FUN_AVG: - if (OB_SUCC(func_node_to_expr(node, expr))) { - LOG_DEBUG("add add node expr succ", "node_type", get_type_name(node->type_)); + if (OB_FAIL(func_node_to_expr(node, expr))) { + LOG_WARN("fail to add func node expr succ", K(ret)); } break; default: @@ -756,7 +861,7 @@ int ObProxySelectStmt::string_node_to_expr(ParseNode* node, ObProxyExpr* &expr, } else if (string_node == NULL) { ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; LOG_WARN("unsupport type", "node_type", get_type_name(node->type_), K(node->str_value_), K(sql_string_)); - } else if (OB_FAIL(get_sharding_const_expr(string_node, expr, true))) { + } else if (OB_FAIL(get_sharding_const_expr(string_node, expr))) { LOG_WARN("get_sharding_const_expr failed", K(ret)); } } @@ -780,19 +885,109 @@ int ObProxySelectStmt::project_string_to_expr(ParseNode* node, ObProxyExpr* &exp return ret; } -int ObProxySelectStmt::org_node_to_expr(ParseNode* node, ObProxyExpr* &expr) +int ObProxySelectStmt::get_table_and_db_expr(ParseNode* node, ObProxyExprTable* &expr_table) +{ + int ret = OB_SUCCESS; + + ObObj obj; + ParseNode* table_node = NULL; + ParseNode* db_node = NULL; + ObProxyExpr* expr = NULL; + if (OB_ISNULL(node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_T_RELATION_FACTOR_NUM_CHILD /* 2 */ != node->num_child_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_RELATION_FACTOR children node is not 2", K(node->num_child_), K(ret)); + } else if (FALSE_IT(db_node = node->children_[0])) { + } else if (NULL != db_node + && OB_UNLIKELY(T_IDENT != db_node->type_ || NULL == db_node->str_value_ || 0 >= db_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_RELATION_FACTOR unexpected table entry", "node type", db_node->type_, + "token len", db_node->token_len_, K(ret)); + } else if (OB_ISNULL(table_node = node->children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_UNLIKELY(T_IDENT != table_node->type_ || NULL == table_node->str_value_ || 0 >= table_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("T_RELATION_FACTOR unexpected table entry", "node type", table_node->type_, + "token len", table_node->token_len_, K(ret)); + } else { + ObString table_name(table_node->token_len_, table_node->str_value_); + if (OB_FAIL(table_exprs_map_.get_refactored(table_name, expr))) { /* same table keep last one. */ + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_TABLE))) { + LOG_WARN("get_expr_by_type failed", K(ret)); + } else if (OB_ISNULL(expr_table = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + if (NULL != db_node) { + expr_table->set_database_name(db_node->str_value_, db_node->token_len_); + } + expr_table->set_table_name(table_node->str_value_, table_node->token_len_); + + if (OB_FAIL(table_exprs_map_.set_refactored(table_name, expr_table))) { /* same table keep last one. */ + LOG_WARN("fail to add table expr", K(table_name), K(ret)); + } + } + } else { + LOG_WARN("fail to get table expr", K(table_name), K(ret)); + } + } else { + if (OB_ISNULL(expr_table = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else if (NULL != db_node && expr_table->get_database_name().empty()) { + expr_table->set_database_name(db_node->str_value_, db_node->token_len_); + } + } + } + + + if (OB_SUCC(ret)) { + ObProxyExprTablePos expr_table_pos; + if (NULL != db_node) { + expr_table_pos.set_database_pos(db_node->token_off_); + } + expr_table_pos.set_table_pos(table_node->token_off_); + expr_table_pos.set_table_expr(expr_table); + if (OB_FAIL(table_pos_array_.push_back(expr_table_pos))) { + LOG_WARN("fail to push expr table pos", K(ret)); + } + } + + return ret; +} + +int ObProxySelectStmt::handle_table_node_to_expr(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; + ObProxyExprTable* expr_table = NULL; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; if (NULL == tmp_node) { - // do nothing - } else { + //do nothing + } else { switch(tmp_node->type_) { case T_RELATION_FACTOR: - if (OB_FAIL(get_sharding_const_expr(tmp_node, expr, true))) { - LOG_WARN("get_sharding_const_expr", K(sql_string_), K(ret)); + if (OB_FAIL(get_table_and_db_expr(tmp_node, expr_table))) { + LOG_WARN("fail to get table expr", K(ret)); + } + break; + case T_IDENT: + if (OB_ISNULL(expr_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get expr table", K(ret)); + } else if (OB_UNLIKELY(NULL == tmp_node->str_value_ || 0 >= tmp_node->token_len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("alias token meet some wrong", "token len", tmp_node->token_len_, K(ret)); + } else { + ObString alias_table = ObString::make_string(tmp_node->str_value_); + if (OB_FAIL(alias_table_map_.set_refactored(alias_table, expr_table))) { + LOG_WARN("fail to add alias table set", K(alias_table), K(ret)); + } } break; default: @@ -803,39 +998,46 @@ int ObProxySelectStmt::org_node_to_expr(ParseNode* node, ObProxyExpr* &expr) } return ret; } + int ObProxySelectStmt::handle_from_list(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - LOG_DEBUG("handle_from_list"); - parse_phase_ = FROM_TABLE_PHASE; - ObProxyExpr* expr = NULL; + for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; - expr = NULL; if (NULL == tmp_node) { // do nothing } else { switch(tmp_node->type_) { case T_ORG: - if (OB_FAIL(org_node_to_expr(tmp_node, expr))) { - LOG_WARN("org_node_to_expr failed", K(ret), K(sql_string_)); + case T_ALIAS: + if (OB_FAIL(handle_table_node_to_expr(tmp_node))) { + LOG_WARN("fail to handle table node to expr", K(sql_string_), K(ret)); } break; - case T_ALIAS: - if (OB_FAIL(alias_node_to_expr(tmp_node, expr))) { - LOG_WARN("alias_node_to_expr failed", K(ret), K(sql_string_)); + case T_OP_EQ: + case T_OP_IN: + if (OB_FAIL(handle_where_node(tmp_node))) { + LOG_WARN("fail to handle where node", K(sql_string_), K(ret)); + } + break; + case T_COLUMN_REF: { + ObProxyExprTable* expr_table = NULL; + ObProxyExpr* expr = NULL; + if (OB_FAIL(column_ref_to_expr(tmp_node, expr, expr_table))) { + LOG_WARN("fail to column ref to expr", K(ret)); + } + if (OB_NOT_NULL(expr)) { + expr->~ObProxyExpr(); } break; + } default: - ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unsupport type", "node_type", get_type_name(tmp_node->type_), K(tmp_node->str_value_)); - } - if (OB_SUCC(ret) && OB_FAIL(table_exprs_.push_back(expr))) { - LOG_WARN("push_back failed", K(ret), K(sql_string_)); - } else if (table_exprs_.count() > 1) { - ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unexpected count ,now only support 1", K(table_exprs_.count()), K(sql_string_)); + if (OB_FAIL(handle_from_list(tmp_node))) { + LOG_WARN("fail to handle from list", K(sql_string_), K(ret)); + } + break; } } } @@ -879,51 +1081,6 @@ int ObProxySelectStmt::handle_varchar_node_in_where_condition(ParseNode* node, S return ret; } -int ObProxySelectStmt::handle_where_eq_node(ParseNode* node) -{ - int ret = OB_SUCCESS; - ParseNode* tmp_node = NULL; - SqlField sql_field; - SqlColumnValue column_value; - if (OB_ISNULL(field_results_)) { - LOG_WARN("unexpected null"); - ret = OB_ERR_UNEXPECTED; - } else { - for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { - tmp_node = node->children_[i]; - if (NULL == tmp_node) { - //do nothing - } else { - switch(tmp_node->type_) { - case T_COLUMN_REF: - sql_field.column_name_.set(tmp_node->str_value_); - break; - case T_INT: - case T_NUMBER: - column_value.value_type_ = TOKEN_STR_VAL; - column_value.column_value_.set(tmp_node->str_value_); - sql_field.column_values_.push_back(column_value); - break; - case T_VARCHAR: - ret = handle_varchar_node_in_where_condition(tmp_node, sql_field); - break; - default: - ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unexpected expr type", "node_type", get_type_name(tmp_node->type_), K(sql_string_)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(field_results_->fields_.push_back(sql_field))) { - LOG_WARN("push_back failed", K(ret), K(sql_string_)); - } else { - ++field_results_->field_num_; - LOG_DEBUG("add sql_field", K(sql_field), K(field_results_->field_num_)); - } - } - } - return ret; -} int ObProxySelectStmt::handle_where_expr_list_node(ParseNode* node, SqlField& sql_field) { int ret = OB_SUCCESS; @@ -951,12 +1108,16 @@ int ObProxySelectStmt::handle_where_expr_list_node(ParseNode* node, SqlField& sq } return ret; } -int ObProxySelectStmt::handle_where_in_node(ParseNode* node) + +int ObProxySelectStmt::handle_where_node(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; + ObProxyExpr* expr = NULL; + bool have_column = false; SqlField sql_field; SqlColumnValue column_value; + bool is_skip_field = false; if (OB_ISNULL(field_results_)) { LOG_WARN("unexpected null"); ret = OB_ERR_UNEXPECTED; @@ -967,8 +1128,40 @@ int ObProxySelectStmt::handle_where_in_node(ParseNode* node) //do nothing } else { switch(tmp_node->type_) { - case T_COLUMN_REF: - sql_field.column_name_.set(tmp_node->str_value_); + case T_COLUMN_REF: { + ObProxyExprColumn* expr_column = NULL; + ObProxyExprTable* expr_table = NULL; + if (OB_FAIL(column_ref_to_expr(tmp_node, expr, expr_table))) { + LOG_WARN("fal to get column expr", K(ret)); + } else if (OB_ISNULL(expr_column = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else if (have_column) { + // where t1.c1 = t2.c2 + is_skip_field = true; + } else { + if (NULL == expr_table + || 0 == expr_table->get_table_name().case_compare(table_name_)) { + sql_field.column_name_.set(expr_column->get_column_name()); + } else { + is_skip_field = true; + } + have_column = true; + } + + if (OB_NOT_NULL(expr)) { + expr->~ObProxyExpr(); + } + break; + } + case T_INT: + case T_NUMBER: + column_value.value_type_ = TOKEN_STR_VAL; + column_value.column_value_.set(tmp_node->str_value_); + sql_field.column_values_.push_back(column_value); + break; + case T_VARCHAR: + ret = handle_varchar_node_in_where_condition(tmp_node, sql_field); break; case T_EXPR_LIST: if (OB_FAIL(handle_where_expr_list_node(tmp_node, sql_field))) { @@ -977,74 +1170,24 @@ int ObProxySelectStmt::handle_where_in_node(ParseNode* node) break; default: ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unsupport expr type", "node_type", get_type_name(tmp_node->type_), K(sql_string_)); - break; + LOG_WARN("unexpected expr type", "node_type", get_type_name(tmp_node->type_), K(sql_string_)); } } } - if (OB_FAIL(field_results_->fields_.push_back(sql_field))) { - LOG_WARN("push_back failed", K(ret)); - } else { - LOG_DEBUG("succ add sql_field", K(sql_field)); - } - } - return ret; -} -int ObProxySelectStmt::handle_where_condition_node(ParseNode* node) -{ - int ret = OB_SUCCESS; - switch(node->type_) { - case T_OP_OR: - case T_OP_AND: - if (OB_FAIL(handle_where_nodes(node))) { - LOG_WARN("handle_where_nodes failed", K(ret), K(sql_string_)); - } - break; - case T_OP_EQ: - if (OB_FAIL(handle_where_eq_node(node))) { - LOG_WARN("handle_where_eq_node failed", K(ret), K(sql_string_)); - } - break; - case T_OP_IN: - if (OB_FAIL(handle_where_in_node(node))) { - LOG_WARN("handle_where_in_node failed", K(ret), K(sql_string_)); + + if (OB_SUCC(ret) && !is_skip_field) { + if (OB_FAIL(field_results_->fields_.push_back(sql_field))) { + LOG_WARN("push_back failed", K(ret), K(sql_string_)); + } else { + ++field_results_->field_num_; + LOG_DEBUG("add sql_field", K(sql_field), K(field_results_->field_num_)); } - break; - default: - LOG_DEBUG("type not handle", "node_type", get_type_name(node->type_), K(sql_string_)); + } } return ret; } + int ObProxySelectStmt::handle_where_clause(ParseNode* node) -{ - int ret = OB_SUCCESS; - parse_phase_ = WHERE_CONDTION_PHASE; - int offset = node->token_off_ + node->token_len_ + 1; - ObString where_str(where_end_pos_ - offset, sql_string_.ptr() + offset); - ObObj obj; - obj.set_varchar(where_str); - LOG_DEBUG("handle_where_clause", K(where_str)); - ObProxyExpr* expr = NULL; - ObProxyExprShardingConst* sharding_expr = NULL; - if (OB_FAIL(get_expr_by_type(expr, OB_PROXY_EXPR_TYPE_SHARDING_CONST))) { - LOG_WARN("get_expr_by_type failed", K(ret), K(sql_string_)); - } else if (OB_ISNULL(sharding_expr = dynamic_cast(expr))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("dynamic_cast failed", K(ret)); - } else if (FALSE_IT(sharding_expr->set_is_column(true))) { - // never here - } else if (FALSE_IT(sharding_expr->set_object(obj))) { - // never here - } else if (OB_FAIL(condition_exprs_.push_back(sharding_expr))){ - LOG_WARN("push_back failed", K(ret), K(sql_string_)); - } else if (OB_FAIL(handle_where_nodes(node))) { - LOG_WARN("handle_where_nodes failed", K(ret), K(sql_string_)); - } else { - LOG_DEBUG("after handle_where_clause", KPC(field_results_), K(condition_exprs_.count())); - } - return ret; -} -int ObProxySelectStmt::handle_where_nodes(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; @@ -1056,18 +1199,41 @@ int ObProxySelectStmt::handle_where_nodes(ParseNode* node) tmp_node = node->children_[i]; if (NULL == tmp_node) { // do nothing - } else if (OB_FAIL(handle_where_condition_node(tmp_node))) { - LOG_WARN("handle_where_condition_node", K(ret), K(sql_string_)); + } else { + switch(tmp_node->type_) { + case T_OP_EQ: + case T_OP_IN: + if (OB_FAIL(handle_where_node(tmp_node))) { + LOG_WARN("fail to handle where node", K(sql_string_), K(ret)); + } + break; + case T_COLUMN_REF: { + ObProxyExprTable* expr_table = NULL; + ObProxyExpr* expr = NULL; + if (OB_FAIL(column_ref_to_expr(tmp_node, expr, expr_table))) { + LOG_WARN("fail to column ref to expr", K(ret)); + } + if (OB_NOT_NULL(expr)) { + expr->~ObProxyExpr(); + } + break; + } + default: + if (OB_FAIL(handle_where_clause(tmp_node))) { + LOG_WARN("handle_where_nodes failed", K(sql_string_), K(ret)); + } + break; + } } } } return ret; } + int ObProxySelectStmt::handle_sort_key_node(ParseNode* node, ObProxyExpr* &expr, const SortListType& sort_list_type) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - ObProxyOrderItem* order_item_expr = NULL; ObProxyExpr* tmp_expr = NULL; ObProxyOrderDirection order_direction = NULLS_FIRST_ASC; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { @@ -1083,31 +1249,51 @@ int ObProxySelectStmt::handle_sort_key_node(ParseNode* node, ObProxyExpr* &expr, order_direction = NULLS_LAST_DESC; break; default: - if (OB_FAIL(string_node_to_expr(tmp_node, expr))) { + if (OB_FAIL(string_node_to_expr(tmp_node, tmp_expr))) { LOG_WARN("string_node_to_expr failed", K(ret), K(sql_string_)); + } else if (T_INT == tmp_node->type_) { + ObProxyExprConst* expr_const = NULL; + if (OB_ISNULL(expr_const = dynamic_cast(tmp_expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast failed", K(ret)); + } else { + int64_t index = expr_const->get_object().get_int(); + if (index > 0) { + tmp_expr->set_index(index - 1); + } + } } } } } if (OB_SUCC(ret)) { - if (sort_list_type != SORT_LSIT_IN_ORDER_BY) { + ObProxyGroupItem* group_item_expr = NULL; + ObProxyExprType expr_type; + if (sort_list_type == SORT_LSIT_IN_ORDER_BY) { + expr_type = OB_PROXY_EXPR_TYPE_FUNC_ORDER; + } else { + expr_type = OB_PROXY_EXPR_TYPE_FUNC_GROUP; + } // do nothing - } else if (OB_FAIL(get_expr_by_type(tmp_expr, OB_PROXY_EXPR_TYPE_FUNC_ORDER))) { - LOG_WARN("get_expr_by_type", K(ret), K(sql_string_)); - } else if (OB_ISNULL(order_item_expr = dynamic_cast(tmp_expr))) { + if (OB_FAIL(get_expr_by_type(expr, expr_type))) { + LOG_WARN("get_expr_by_type", K(expr_type), K(sql_string_), K(ret)); + } else if (OB_ISNULL(group_item_expr = dynamic_cast(expr))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("dynamic_cast to ObProxyOrderItem failed", K(ret)); + LOG_WARN("dynamic_cast to ObProxyGroupItem failed", K(ret)); } else { - if (expr->has_agg_) { - order_item_expr->has_agg_ = 1; + if (tmp_expr->has_agg_) { + group_item_expr->has_agg_ = 1; } - if (expr->has_alias_) { - order_item_expr->has_alias_ = 1; + group_item_expr->set_expr(tmp_expr); + if (sort_list_type == SORT_LSIT_IN_ORDER_BY) { + ObProxyOrderItem* order_item_expr = NULL; + if (OB_ISNULL(order_item_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast to ObProxyOrderItem failed", K(ret)); + } else { + order_item_expr->order_direction_ = order_direction; + } } - order_item_expr->order_direction_ = order_direction; - order_item_expr->expr_ = expr; - expr = order_item_expr; - LOG_DEBUG("add order by sort key", K(order_direction), K(expr->has_agg_), K(expr->has_alias_)); } } return ret; @@ -1117,6 +1303,7 @@ int ObProxySelectStmt::handle_sort_list_node(ParseNode* node, const SortListType int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; ObProxyExpr* expr = NULL; + ObProxyGroupItem* group_item_expr = NULL; ObProxyOrderItem* order_item_expr = NULL; for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; @@ -1127,12 +1314,15 @@ int ObProxySelectStmt::handle_sort_list_node(ParseNode* node, const SortListType switch(tmp_node->type_) { case T_SORT_KEY: if (OB_FAIL(handle_sort_key_node(tmp_node, expr, sort_list_type))) { - LOG_WARN("handle_sort_key_node", K(ret), K(sql_string_)); + LOG_WARN("handle_sort_key_node", K(sql_string_), K(ret)); } else { switch(sort_list_type) { case SORT_LIST_IN_GROUP_BY: - if (OB_FAIL(group_by_exprs_.push_back(expr))) { - LOG_WARN("push_back failed", K(ret), K(sql_string_)); + if (OB_ISNULL(group_item_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic_cast to ObProxyGroupItem failed", K(ret)); + } else if (OB_FAIL(group_by_exprs_.push_back(group_item_expr))) { + LOG_WARN("push_back failed", K(sql_string_), K(ret)); } break; case SORT_LSIT_IN_ORDER_BY: @@ -1140,21 +1330,25 @@ int ObProxySelectStmt::handle_sort_list_node(ParseNode* node, const SortListType ret = OB_ERR_UNEXPECTED; LOG_WARN("dynamic_cast to ObProxyOrderItem failed", K(ret)); } else if (OB_FAIL(order_by_exprs_.push_back(order_item_expr))) { - LOG_WARN("push_back failed", K(ret), K(sql_string_)); + LOG_WARN("push_back failed", K(sql_string_), K(ret)); } break; default: ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid sort_list_type", K(sort_list_type), K(sql_string_)); + LOG_WARN("invalid sort_list_type", K(sort_list_type), K(sql_string_), K(ret)); } } break; default: - ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; - LOG_WARN("unsupport expr type", "node_type", get_type_name(tmp_node->type_), K(sql_string_)); + ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; + LOG_WARN("unsupport expr type", "node_type", get_type_name(tmp_node->type_), K(sql_string_), K(ret)); } } } + + if (OB_FAIL(ret) && OB_NOT_NULL(expr)) { + expr->~ObProxyExpr(); + } return ret; } int ObProxySelectStmt::handle_with_rollup_in_groupby(ParseNode* node) @@ -1188,8 +1382,6 @@ int ObProxySelectStmt::handle_groupby_clause(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - parse_phase_ = GROUP_BY_PHASE; - LOG_DEBUG("handle_groupby_clause"); for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; if (NULL == tmp_node) { @@ -1201,9 +1393,7 @@ int ObProxySelectStmt::handle_groupby_clause(ParseNode* node) LOG_WARN("handle_with_rollup_in_groupby failed", K(ret), K(i), K(sql_string_)); } } - if (OB_SUCC(ret)) { - LOG_DEBUG("after handle_groupby_clause", K(group_by_exprs_.count())); - } + return ret; } @@ -1211,8 +1401,6 @@ int ObProxySelectStmt::handle_orderby_clause(ParseNode* node) { int ret = OB_SUCCESS; ParseNode* tmp_node = NULL; - parse_phase_ = ORDER_BY_PHASE; - LOG_DEBUG("handle_orderby_clause"); for (int i = 0; OB_SUCC(ret) && i < node->num_child_; i++) { tmp_node = node->children_[i]; if (NULL == tmp_node) { @@ -1224,9 +1412,7 @@ int ObProxySelectStmt::handle_orderby_clause(ParseNode* node) LOG_WARN("handle_sort_list_node", K(ret), K(i), K(sql_string_)); } } - if (OB_SUCC(ret)) { - LOG_DEBUG("after handle_orderby_clause", K(order_by_exprs_.count())); - } + return ret; } diff --git a/src/obproxy/obutils/ob_proxy_stmt.h b/src/obproxy/obutils/ob_proxy_stmt.h index ec43599d3080ad77ee804262fc98719dea072ee0..5e8a1818324e01877019ab424bfdef3531679104 100644 --- a/src/obproxy/obutils/ob_proxy_stmt.h +++ b/src/obproxy/obutils/ob_proxy_stmt.h @@ -29,6 +29,33 @@ namespace obproxy { namespace obutils { + +class ObProxyExprTablePos { +public: + ObProxyExprTablePos() : database_pos_(-1), table_pos_(-1), table_expr_(NULL) {} + virtual ~ObProxyExprTablePos() {} + + void set_database_pos(const int32_t database_pos) { database_pos_ = database_pos; } + int32_t get_database_pos() { return database_pos_; } + void set_table_pos(const int32_t table_pos) { table_pos_ = table_pos; } + int32_t get_table_pos() { return table_pos_; } + void set_table_expr(ObProxyExprTable* table_expr) { table_expr_ = table_expr; } + ObProxyExprTable *get_table_expr() { return table_expr_; } + + bool operator<(ObProxyExprTablePos &expr_table_pos) + { + return table_pos_ < expr_table_pos.get_table_pos(); + } + + TO_STRING_KV("database_pos", database_pos_, + "table_pos", table_pos_); + +private: + int32_t database_pos_; + int32_t table_pos_; + ObProxyExprTable* table_expr_; +}; + class ObProxyStmt { public: ObProxyStmt() : stmt_type_(OBPROXY_T_INVALID), allocator_(NULL) {} @@ -48,17 +75,16 @@ protected: class ObProxyDMLStmt : public ObProxyStmt { public: - ObProxyDMLStmt(): limit_start_(0), limit_offset_(-1), field_results_(NULL) {} + ObProxyDMLStmt(): limit_offset_(0), limit_size_(-1), limit_token_off_(-1), field_results_(NULL) {} virtual ~ObProxyDMLStmt() {} protected: int condition_exprs_to_sql_string(common::ObSqlString& sql_string); int limit_to_sql_string(common::ObSqlString& sql_string); public: - common::ObSEArray condition_exprs_; //where condition expr - int limit_start_; int limit_offset_; + int limit_size_; + int64_t limit_token_off_; SqlFieldResult* field_results_; // pointer to parseResult - common::ObString hint_string_; common::ObSEArray comments_; }; @@ -68,14 +94,7 @@ enum SortListType { SORT_LSIT_IN_ORDER_BY, SORT_LIST_TYPE_MAX, }; -enum ParsePhase { - SELECT_FIELD_PHASE = 0, - FROM_TABLE_PHASE = 1, - WHERE_CONDTION_PHASE = 2, - GROUP_BY_PHASE = 3, - ORDER_BY_PHASE = 4, - PHASE_MAX, -}; + class ObProxySelectStmt : public ObProxyDMLStmt { public: @@ -93,6 +112,18 @@ public: int handle_limit_clause(ParseNode* node); int handle_comment_list(const ParseResult &parse_result); virtual int to_sql_string(common::ObSqlString& sql_string); + bool has_for_update() const { return has_for_update_; } + void set_table_name(const common::ObString& table_name) { table_name_ = table_name; } + int64_t get_from_token_off() { return from_token_off_; } + +public: + typedef common::hash::ObHashMap ExprMap; + typedef common::ObSEArray TablePosArray; + +public: + ExprMap& get_table_exprs_map() { return table_exprs_map_; } + TablePosArray& get_table_pos_array() { return table_pos_array_; } + private: int project_string_to_expr(ParseNode* node, ObProxyExpr* &expr); int string_node_to_expr(ParseNode* node, ObProxyExpr* &expr, ParseNode* string_node = NULL); @@ -106,18 +137,19 @@ private: int max_node_to_expr(ParseNode* node, ObProxyExpr* &expr); int min_node_to_expr(ParseNode* node, ObProxyExpr* &expr); - int column_ref_to_expr(ParseNode* node, ObProxyExpr* &expr); - int get_sharding_const_expr(ParseNode* node, ObProxyExpr* &expr, bool is_column_ref = true); + int column_ref_to_expr(ParseNode* node, ObProxyExpr* &expr, ObProxyExprTable* &expr_table); + int get_const_expr(ParseNode* node, ObProxyExpr* &expr); + int get_sharding_const_expr(ParseNode* node, ObProxyExpr* &expr); int get_expr_by_type(ObProxyExpr* &expr, ObProxyExprType type); ObProxyExprType get_expr_type_by_node_type(const ObItemType& item_type); - int org_node_to_expr(ParseNode* node, ObProxyExpr* &expr); + int get_table_and_db_expr(ParseNode* node, ObProxyExprTable* &expr_table); + int handle_table_node_to_expr(ParseNode* node); + int handle_table_and_db_node(ParseNode* node, ObProxyExprTable* &expr_table); + int handle_table_and_db_in_hint(ParseNode* node); //for where - int handle_where_nodes(ParseNode* node); - int handle_where_condition_node(ParseNode* node); - int handle_where_eq_node(ParseNode* node); - int handle_where_in_node(ParseNode* node); + int handle_where_node(ParseNode* node); int handle_where_expr_list_node(ParseNode* node, SqlField& sql_field); int handle_varchar_node_in_where_condition(ParseNode* node, SqlField& sql_field); //for group by @@ -133,20 +165,20 @@ private: int group_by_exprs_to_sql_string(common::ObSqlString& sql_string); int order_by_exprs_to_sql_string(common::ObSqlString& sql_string); private: - int handle_where_end_pos(ParseNode* node); int check_node_has_agg(ParseNode* node); - int where_end_pos_; public: common::ObSEArray select_exprs_; - common::ObSEArray table_exprs_; - common::ObSEArray group_by_exprs_; + common::ObSEArray group_by_exprs_; //select groupby expression common::ObSEArray order_by_exprs_; private: + common::ObString table_name_; bool is_inited_; - ParsePhase parse_phase_; bool has_rollup_; - typedef common::hash::ObHashMap AliasExprMap; - AliasExprMap alias_col_map_; + bool has_for_update_; + int64_t from_token_off_; + ExprMap table_exprs_map_; + ExprMap alias_table_map_; + common::ObSEArray table_pos_array_; }; } // end of namespace obutils } // end of namespace obproxy diff --git a/src/obproxy/obutils/ob_proxy_table_processor.cpp b/src/obproxy/obutils/ob_proxy_table_processor.cpp index 0a3fce5e6ba0ec5f4f1d57ca377150b353985808..baf69d070f0da663ffda0da679dedd21463031e4 100644 --- a/src/obproxy/obutils/ob_proxy_table_processor.cpp +++ b/src/obproxy/obutils/ob_proxy_table_processor.cpp @@ -573,66 +573,7 @@ int ObProxyTableProcessor::get_proxy_info(ObProxyServerInfo &proxy_info) const char *proxy_ip = hot_upgrade_processor_.get_proxy_ip(); const int32_t proxy_port = hot_upgrade_processor_.get_proxy_port(); proxy_info.reset(); - ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); - ObProxyLocalCMDType cmd_type = get_global_proxy_config().get_local_cmd_type(); - switch (cmd_type) { - case OB_LOCAL_CMD_NONE: { - //do nothing - break; - } - case OB_LOCAL_CMD_EXIT: { - get_global_proxy_config().reset_local_cmd(); - if (info.is_in_idle_state()) { - info.cmd_ = HUC_LOCAL_EXIT; - LOG_INFO("proxy will do quick exit from local cmd, no need check proxy info", K(info)); - } else { - LOG_WARN("it is doing hot upgrading now, CAN NOT do quick exit from local cmd", K(info)); - } - - break; - } - case OB_LOCAL_CMD_RESTART: { - get_global_proxy_config().reset_local_cmd(); - if (info.is_in_idle_state()) { - info.cmd_ = HUC_LOCAL_RESTART; - hot_upgrade_processor_.reset_upgrade_failures(); - LOG_INFO("proxy will do restart from local cmd", K(info)); - } else { - LOG_WARN("it is doing hot upgrading now, CAN NOT do quick restart from local cmd", K(info)); - } - break; - } - case OB_LOCAL_CMD_COMMIT: { - get_global_proxy_config().reset_local_cmd(); - if (info.is_local_restart() && info.is_in_wait_cr_state()) { - info.cmd_ = HUC_LOCAL_COMMIT; - LOG_INFO("proxy will do commit from local cmd", K(info)); - } else { - LOG_WARN("it is not doing hot upgrading now, CAN NOT do commit from local cmd", K(info)); - } - break; - } - case OB_LOCAL_CMD_ROLLBACK: { - get_global_proxy_config().reset_local_cmd(); - if (info.is_local_restart() && info.is_in_wait_cr_state()) { - info.cmd_ = HUC_LOCAL_ROLLBACK; - LOG_INFO("proxy will do rollback from local cmd", K(info)); - } else { - LOG_WARN("it is not doing hot upgrading now, CAN NOT do rollback from local cmd", K(info)); - } - break; - } - default: { - LOG_WARN("unknown ObProxyLocalCMDType, reset it default value", K(cmd_type)); - get_global_proxy_config().reset_local_cmd(); - } - } - - //5. if this is local_cmd, no need get proxy info - if (info.is_local_cmd()) { - LOG_DEBUG("it is doing local cmd now, no need check proxy info", K(info)); - - } else if (OB_FAIL(ObProxyTableProcessorUtils::get_proxy_info(*mysql_proxy_, proxy_ip, proxy_port, proxy_info))) { + if (OB_FAIL(ObProxyTableProcessorUtils::get_proxy_info(*mysql_proxy_, proxy_ip, proxy_port, proxy_info))) { if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST == ret)) { // this means no record in observer's PROXY_INFO_TABLE_NAME, abnormal state proxy_info.row_status_ = PRS_NOT_EXSIT; @@ -659,8 +600,7 @@ int ObProxyTableProcessor::check_upgrade_state(const ObProxyServerInfo &proxy_in //any of the follower happened, we should do state_wait_hu_cmd() //1. proxy_info is AVAILABLE //2. this is local cmd from config cmd - if (OB_LIKELY(PRS_AVAILABLE == proxy_info.row_status_) - || info.is_local_cmd()) { + if (OB_LIKELY(PRS_AVAILABLE == proxy_info.row_status_)) { if (OB_FAIL(hot_upgrade_processor_.state_wait_hu_cmd(proxy_info))) { LOG_WARN("fail to handle wait_hu_cmd state", K(proxy_info), K(info), K(ret)); } @@ -687,8 +627,7 @@ int ObProxyTableProcessor::check_upgrade_state(const ObProxyServerInfo &proxy_in //4. this is restart from config cmd if (OB_LIKELY(PRS_AVAILABLE == proxy_info.row_status_) || hot_upgrade_processor_.is_timeout_rollback() - || info.is_auto_upgrade() - || info.is_local_cmd()) { + || info.is_auto_upgrade()) { if (OB_FAIL(hot_upgrade_processor_.state_wait_cr_cmd(proxy_info))) { LOG_WARN("fail to handle wait_cr_cmd state", K(info), K(proxy_info), K(ret)); } @@ -706,6 +645,8 @@ int ObProxyTableProcessor::check_upgrade_state(const ObProxyServerInfo &proxy_in } break; } + case HU_STATE_WAIT_LOCAL_CR_FINISH: + break; default: { ret = OB_ERR_UNEXPECTED; LOG_ERROR("it should not enter here", K(info)); @@ -760,12 +701,8 @@ int ObProxyTableProcessor::check_update_proxy_table(const ObProxyServerInfo &pro break; } case PRS_NONE: { - if (info.is_local_cmd()) { - LOG_INFO("this is local cmd, no need update_proxy table", K(info)); - } else { - if (OB_FAIL(register_proxy(RT_FORCE))) { - LOG_WARN("fail to register proxy, current proxy info in table maybe old", K(info), K(ret)); - } + if (OB_FAIL(register_proxy(RT_FORCE))) { + LOG_WARN("fail to register proxy, current proxy info in table maybe old", K(info), K(ret)); } break; } @@ -923,7 +860,6 @@ int ObProxyTableProcessor::do_check_work() if (OB_SUCC(ret)) { //proxy need check kv table when it is not local cmd - const bool need_check_kv = ((!get_global_hot_upgrade_info().is_local_cmd() && !get_global_proxy_config().is_local_cmd())); ObProxyKVTableInfo kv_info; if (is_meta_mysql_avail) { @@ -934,27 +870,18 @@ int ObProxyTableProcessor::do_check_work() } } - if (need_check_kv) { - //2. check proxy kv info from tables without care is_registered_ - if (OB_FAIL(check_proxy_kv_info(kv_info))) { - LOG_WARN("fail to check proxy info", K(kv_info), K(ret)); - } - - //3. get proxy info from tables if needed - if (need_check_proxy_info_table(kv_info)) { - if (OB_FAIL(check_proxy_info(kv_info))) { - LOG_WARN("fail to check proxy info", K(kv_info), K(ret)); - } - } else { - LOG_DEBUG("there is no need to get proxy info", K(need_check_kv), K(is_meta_mysql_avail), K(kv_info)); - } + //2. check proxy kv info from tables without care is_registered_ + if (OB_FAIL(check_proxy_kv_info(kv_info))) { + LOG_WARN("fail to check proxy info", K(kv_info), K(ret)); } - } - //4. get proxy info from tables if use local cmd - if (!need_check_kv) { - if (OB_FAIL(check_proxy_info(kv_info))) { - LOG_WARN("fail to check proxy info", K(kv_info), K(ret)); + //3. get proxy info from tables if needed + if (need_check_proxy_info_table(kv_info)) { + if (OB_FAIL(check_proxy_info(kv_info))) { + LOG_WARN("fail to check proxy info", K(kv_info), K(ret)); + } + } else { + LOG_DEBUG("there is no need to get proxy info", K(is_meta_mysql_avail), K(kv_info)); } } } diff --git a/src/obproxy/obutils/ob_resource_pool_processor.cpp b/src/obproxy/obutils/ob_resource_pool_processor.cpp index df65603277b350366aafd6239dfb5b16f07a111a..3e75bbc201d871f81b8ac9fdd9623b7421eabe63 100644 --- a/src/obproxy/obutils/ob_resource_pool_processor.cpp +++ b/src/obproxy/obutils/ob_resource_pool_processor.cpp @@ -44,16 +44,14 @@ namespace obutils { ObResourcePoolProcessor g_rp_processor; -const static char *CHEK_CLUSTER_NAME_SQL = - "SELECT /*+READ_CONSISTENCY(WEAK)*/ cluster FROM oceanbase.%s LIMIT 1"; const static char *CHEK_CLUSTER_ROLE_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ cluster_role, cluster_status FROM oceanbase.%s LIMIT 1"; const static char *OBPROXY_V_DATABASE_TNAME = "v$ob_cluster"; const static char *INIT_SS_INFO_SQL = - "SELECT /*+READ_CONSISTENCY(WEAK)*/ *, zs.status AS zone_status, ss.status AS server_status " - "FROM oceanbase.%s zs, oceanbase.%s ss " - "WHERE zs.zone=ss.zone " - "AND ss.svr_port > 0 LIMIT %ld;"; + "SELECT /*+READ_CONSISTENCY(WEAK)*/ *, zs.status AS zone_status, ss.status AS server_status " + "FROM oceanbase.%s ss left join oceanbase.%s zs " + "ON zs.zone = ss.zone " + "WHERE ss.svr_port > 0 ORDER BY ss.zone LIMIT %ld;"; const static char *PRIMARY_ROLE = "PRIMARY"; const static char *ROLE_VALID = "VALID"; @@ -327,99 +325,6 @@ inline void *ObClusterRoleCheckCont::get_callback_data() return static_cast(&check_result_); } -//---------------------ObClusterCheckNameCont---------------------// -class ObClusterNameCheckCont : public ObAsyncCommonTask -{ -public: - ObClusterNameCheckCont(ObClusterResource *cr, ObContinuation *cb_cont, ObEThread *submit_thread) - : ObAsyncCommonTask(cb_cont->mutex_, "cluster_name_check_task", cb_cont, submit_thread), - check_result_(false), cr_(cr) {} - virtual ~ObClusterNameCheckCont() {} - - virtual void destroy(); - virtual int init_task(); - virtual int finish_task(void *data); - virtual void *get_callback_data(); - -private: - bool check_result_; - ObClusterResource *cr_; - DISALLOW_COPY_AND_ASSIGN(ObClusterNameCheckCont); -}; - -void ObClusterNameCheckCont::destroy() -{ - if (NULL != cr_) { - // inc_ref() in add_async_task() - cr_->dec_ref(); - cr_ = NULL; - } - ObAsyncCommonTask::destroy(); -} - -int ObClusterNameCheckCont::init_task() -{ - int ret = OB_SUCCESS; - char sql[OB_SHORT_SQL_LENGTH]; - sql[0] = '\0'; - int64_t len = snprintf(sql, OB_SHORT_SQL_LENGTH, CHEK_CLUSTER_NAME_SQL, OB_ALL_VIRTUAL_ZONE_STAT_TNAME); - if (OB_UNLIKELY(len <= 0) || OB_UNLIKELY(len >= OB_SHORT_SQL_LENGTH)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to fill sql", K(len), K(ret)); - } else if (OB_FAIL(cr_->mysql_proxy_.async_read(this, sql, pending_action_))) { - LOG_WARN("fail to async read", K(ret)); - } else if (OB_ISNULL(pending_action_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("pending action can not be NULL", K_(pending_action), K(ret)); - } - - return ret; -} - -int ObClusterNameCheckCont::finish_task(void *data) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(data)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid data", K(data), K(ret)); - } else { - ObClientMysqlResp *resp = reinterpret_cast(data); - ObMysqlResultHandler handler; - handler.set_resp(resp); - if (OB_FAIL(handler.next())) { - if (OB_ITER_END == ret) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("no cluster name found", K(ret)); - } else { - LOG_WARN("fail to get cluster name", K(ret)); - } - } else { - ObString cluster_name; - PROXY_EXTRACT_VARCHAR_FIELD_MYSQL(handler, "cluster", cluster_name); - - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(OB_ITER_END != handler.next())) { // check if this is only one - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get cluster name, there is more than one record", K(ret)); - } else if (OB_UNLIKELY(cr_->get_cluster_name() != cluster_name)) { - ret = OB_OBCONFIG_APPNAME_MISMATCH; - LOG_WARN("fail to check cluster name", "local cluster name", cr_->get_cluster_name(), - "remote cluster name", cluster_name, K(ret)); - } else { - check_result_ = true; - } - } - } - } - - return ret; -} - -inline void *ObClusterNameCheckCont::get_callback_data() -{ - return static_cast(&check_result_); -} - //---------------------ObServerStateInfoInitCont---------------------// class ObServerStateInfoInitCont : public ObAsyncCommonTask { @@ -458,8 +363,8 @@ int ObServerStateInfoInitCont::init_task() char sql[OB_SHORT_SQL_LENGTH]; sql[0] = '\0'; int64_t len = snprintf(sql, OB_SHORT_SQL_LENGTH, INIT_SS_INFO_SQL, - OB_ALL_VIRTUAL_ZONE_STAT_TNAME, OB_ALL_VIRTUAL_PROXY_SERVER_STAT_TNAME, + OB_ALL_VIRTUAL_ZONE_STAT_TNAME, INT64_MAX); if (OB_UNLIKELY(len <= 0) || OB_UNLIKELY(len >= OB_SHORT_SQL_LENGTH)) { ret = OB_ERR_UNEXPECTED; @@ -498,6 +403,12 @@ int ObServerStateInfoInitCont::finish_task(void *data) int64_t tmp_real_str_len = 0; int64_t start_service_time = 0; int64_t stop_time = 0; + ObString cluster_name; + ObSEArray servers_state; + ObServerStateInfo server_state; + ObSEArray zones_state; + ObZoneStateInfo zone_state; + ObSEArray zone_names; while (OB_SUCC(ret) && OB_SUCC(result_handler.next())) { ss_info.reset(); @@ -512,6 +423,9 @@ int ObServerStateInfoInitCont::finish_task(void *data) port = 0; start_service_time = 0; stop_time = 0; + cluster_name.reset(); + server_state.reset(); + zone_state.reset(); PROXY_EXTRACT_VARCHAR_FIELD_MYSQL(result_handler, "zone", zone_name); PROXY_EXTRACT_INT_FIELD_MYSQL(result_handler, "is_merging", ss_info.is_merging_, int64_t); @@ -523,6 +437,18 @@ int ObServerStateInfoInitCont::finish_task(void *data) PROXY_EXTRACT_STRBUF_FIELD_MYSQL(result_handler, "server_status", display_status_str, MAX_DISPLAY_STATUS_LEN, tmp_real_str_len); + if (OB_SUCC(ret)) { + if (OB_LIKELY(get_global_proxy_config().with_config_server_) + && (cr_->get_cluster_name() != OB_META_DB_CLUSTER_NAME)) { + PROXY_EXTRACT_VARCHAR_FIELD_MYSQL(result_handler, "cluster", cluster_name); + if (OB_UNLIKELY(cr_->get_cluster_name() != cluster_name)) { + ret = OB_OBCONFIG_APPNAME_MISMATCH; + LOG_WARN("fail to check cluster name", "local cluster name", cr_->get_cluster_name(), + "remote cluster name", cluster_name, K(ret)); + } + } + } + if (OB_SUCC(ret)) { PROXY_EXTRACT_VARCHAR_FIELD_MYSQL(result_handler, "region", region_name); if ((OB_ERR_COLUMN_NOT_FOUND == ret) || (OB_SUCC(ret) && region_name.empty())) { @@ -584,12 +510,80 @@ int ObServerStateInfoInitCont::finish_task(void *data) LOG_DEBUG("succ to push back server_info", K(ss_info)); } } + + if (OB_SUCC(ret)) { + zone_state.is_merging_ = ss_info.is_merging_; + zone_state.zone_type_ = ss_info.zone_type_; + zone_state.zone_status_ = ObZoneStatus::get_status(zone_status); + if (OB_FAIL(zone_state.set_zone_name(zone_name))) { + LOG_WARN("fail to set zone name", K(zone_name), K(ret)); + } else if (OB_FAIL(zone_state.set_region_name(region_name))) { + LOG_WARN("fail to set region name", K(region_name), K(ret)); + } else if (OB_FAIL(zone_state.set_idc_name(idc_name))) { + LOG_WARN("fail to set idc name", K(region_name), K(ret)); + } else if (OB_FAIL(zones_state.push_back(zone_state))) { + LOG_WARN("fail to push back zone_info", K(zone_state), K(ret)); + } else { + server_state.start_service_time_ = start_service_time; + server_state.stop_time_ = stop_time; + server_state.server_status_ = server_status; + + if (OB_FAIL(server_state.add_addr(ip_str, port))) { + LOG_WARN("fail to add addr", K(ip_str), K(port), K(ret)); + //if svr_ip or svr_port in __all_virtual_proxy_server_stat is wrong, + //we can skip over this server_state. + ret = OB_SUCCESS; + continue; + } else if (OB_ISNULL(ObServerStateRefreshUtils::get_zone_info_ptr(zones_state, zone_name))) { + LOG_INFO("this server can not find it's zone, maybe it's zone has been " + "deleted, so treat this server as deleted also", K(server_state), K(zone_name), K(zones_state)); + ret = OB_SUCCESS; + continue; + } else { + if (OB_FAIL(servers_state.push_back(server_state))) { + LOG_WARN("fail to push back server_state", K(server_state), K(ret)); + } else if (OB_FAIL(zone_names.push_back(zone_name))) { + LOG_WARN("fail to push back zone_name", K(ret), K(zone_name)); + } else { + LOG_DEBUG("succ to push back server_state and zone_name", K(ret), K(server_state), K(zone_name)); + } + } + } + } }//end of while - if (ret != OB_ITER_END) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + + /* + * build zone_state_ for each server_state + * pay attention to the validity of the ptr zone_state_ + * the above judgement of get_zone_info_ptr() will not set zone_state_, to avoid the ptr being invalid. + */ + for (int i = 0; (OB_SUCC(ret)) && (i < zone_names.count()); ++i) { + ObString &each_zone_name = zone_names.at(i); + ObServerStateInfo &each_servers_state = servers_state.at(i); + if (OB_ISNULL(each_servers_state.zone_state_ = + ObServerStateRefreshUtils::get_zone_info_ptr(zones_state, each_zone_name))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected situation, the corresponding zone_state has been searched before.", + K(ret), K(each_zone_name), K(zones_state)); + } else { + if (each_servers_state.zone_state_->is_readonly_zone()) { + each_servers_state.replica_.replica_type_ = REPLICA_TYPE_READONLY; + } else if (each_servers_state.zone_state_->is_encryption_zone()) { + each_servers_state.replica_.replica_type_ = REPLICA_TYPE_ENCRYPTION_LOGONLY; + } else { + each_servers_state.replica_.replica_type_ = REPLICA_TYPE_FULL; + } + } + } + + if (OB_FAIL(ret)) { LOG_WARN("fail to get server state info", K(ret)); } else { - ret = OB_SUCCESS; + LocationList server_list; //save the ordered servers const uint64_t new_ss_version = cr_->server_state_version_ + 1; common::ObIArray &server_state_info = cr_->get_server_state_info(new_ss_version); common::DRWLock &server_state_lock = cr_->get_server_state_lock(new_ss_version); @@ -598,10 +592,55 @@ int ObServerStateInfoInitCont::finish_task(void *data) server_state_info.assign(ss_infos_); server_state_lock.wrunlock(); ATOMIC_AAF(&(cr_->server_state_version_), 1); - check_result_ = true; - LOG_INFO("succ to init server state info", "cluster_info", cr_->cluster_info_key_, - "server_state_version_", cr_->server_state_version_, - K_(ss_infos)); + + if (OB_FAIL(ObServerStateRefreshUtils::order_servers_state(servers_state, server_list))) { + LOG_WARN("fail to order servers state", K(ret)); + // add to table location + } else if (OB_LIKELY(!server_list.empty())) { + LOG_DEBUG("succ to get server list", K(server_list), "server_count", servers_state.count()); + ObTableEntry *entry = NULL; + ObTableCache &table_cache = get_global_table_cache(); + const bool is_rslist = false; + if (OB_FAIL(ObRouteUtils::build_sys_dummy_entry(cr_->get_cluster_name(), cr_->get_cluster_id(), server_list, is_rslist, entry))) { + LOG_WARN("fail to build sys dummy entry", K(server_list), "cluster_name", cr_->get_cluster_name(), + "cluster_id", cr_->get_cluster_id(), K(ret)); + } else if (OB_ISNULL(entry) || OB_UNLIKELY(!entry->is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("entry can not be NULL here", K(entry), K(ret)); + } else { + ObTableEntry *tmp_entry = NULL; + tmp_entry = entry; + tmp_entry->inc_ref();//just for print + if (OB_FAIL(ObTableCache::add_table_entry(table_cache, *entry))) { + LOG_WARN("fail to add table entry", K(*entry), K(ret)); + } else { + obsys::CWLockGuard guard(cr_->dummy_entry_rwlock_); + if (NULL != cr_->dummy_entry_) { + cr_->dummy_entry_->dec_ref(); + } + cr_->dummy_entry_ = tmp_entry; + cr_->dummy_entry_->inc_ref(); + + LOG_INFO("ObServerStateInfoInitCont, update sys tennant's __all_dummy succ", KPC(tmp_entry)); + } + tmp_entry->dec_ref(); + tmp_entry = NULL; + } + + if (OB_FAIL(ret)) { + if (NULL != entry) { + entry->dec_ref(); + entry = NULL; + } + } + }//end of add to table location + + if (OB_SUCC(ret)) { + check_result_ = true; + LOG_INFO("succ to init server state info", "cluster_info", cr_->cluster_info_key_, + "server_state_version_", cr_->server_state_version_, + K_(ss_infos), K(zones_state), K(servers_state)); + } } } @@ -796,11 +835,6 @@ int ObClusterResourceCreateCont::add_async_task() this, &self_ethread()); } break; - case CHECK_CLUSTER_NAME: - LOG_INFO("will add CHECK_CLUSTER_NAME task", K(this), K_(cluster_name), K_(cluster_id)); - created_cr_->inc_ref(); - async_cont = new(std::nothrow) ObClusterNameCheckCont(created_cr_, this, &self_ethread()); - break; case CHECK_CLUSTER_ROLE: LOG_INFO("will add CHECK_CLUSTER_ROLE task", K(this), K_(cluster_name), K_(cluster_id)); created_cr_->inc_ref(); @@ -1033,11 +1067,9 @@ int ObClusterResourceCreateCont::handle_async_task_complete(void *data) && get_global_proxy_config().enable_standby) { init_status_ = CHECK_CLUSTER_ROLE; } else { - init_status_ = CHECK_CLUSTER_NAME; + init_status_ = INIT_SS_INFO; } } else if (CHECK_CLUSTER_ROLE == init_status_) { - init_status_ = CHECK_CLUSTER_NAME; - } else if (CHECK_CLUSTER_NAME == init_status_) { init_status_ = INIT_SS_INFO; } else if (INIT_SS_INFO == init_status_) { init_status_ = INIT_IDC_LIST; @@ -1368,7 +1400,7 @@ int ObClusterResourceCreateCont::build() && get_global_proxy_config().enable_standby) { init_status_ = CHECK_CLUSTER_ROLE; } else { - init_status_ = CHECK_CLUSTER_NAME; + init_status_ = INIT_SS_INFO; } LOG_DEBUG("try next status", K_(cluster_name), K_(cluster_id), K(new_failure_count), K(rs_list), K(is_rslist_from_local_), K(init_status_)); @@ -1377,18 +1409,6 @@ int ObClusterResourceCreateCont::build() } } - if (OB_SUCC(ret) && CHECK_CLUSTER_NAME == init_status_) { - if (OB_LIKELY(get_global_proxy_config().with_config_server_) - && (cluster_name_ != OB_META_DB_CLUSTER_NAME)) { - // cluster name checking, if it is proxy meta db cluster, do not check - if (OB_FAIL(add_async_task())) { - LOG_WARN("fail to add cluster name check task", K(ret)); - } - } else { - init_status_ = INIT_SS_INFO; - } - } - if (OB_SUCC(ret) && CHECK_CLUSTER_ROLE == init_status_) { int64_t cluster_id = OB_DEFAULT_CLUSTER_ID; ObConfigServerProcessor &cs_processor = get_global_config_server_processor(); @@ -1461,8 +1481,7 @@ DEF_TO_STRING(ObClusterResource) "cr_state", get_cr_state_str(cr_state_), K_(version), K_(last_access_time_ns), K_(deleting_completed_thread_num), K_(fetch_rslist_task_count), K_(fetch_idc_list_task_count), - K_(last_idc_list_refresh_time_ns), K_(last_rslist_refresh_time_ns), K_(server_state_version), - KPC_(dummy_entry)); + K_(last_idc_list_refresh_time_ns), K_(last_rslist_refresh_time_ns), K_(server_state_version)); J_OBJ_END(); return pos; } diff --git a/src/obproxy/obutils/ob_resource_pool_processor.h b/src/obproxy/obutils/ob_resource_pool_processor.h index 31276e1f9583d3fa26e0007a88bcf03fef757b3e..dbb4b2a952567680aff37f1f5040b8312bce079e 100644 --- a/src/obproxy/obutils/ob_resource_pool_processor.h +++ b/src/obproxy/obutils/ob_resource_pool_processor.h @@ -102,7 +102,6 @@ enum ObClusterResourceInitState { INIT_BORN = 0, INIT_RS, - CHECK_CLUSTER_NAME, CHECK_CLUSTER_ROLE, INIT_SS_INFO, INIT_IDC_LIST, @@ -402,6 +401,7 @@ public: event::ObAction *&action); // Attention!! if succ, will inc_ref; ObClusterResource *acquire_avail_cluster_resource(const common::ObString &cluster_name, const int64_t cluster_id = OB_DEFAULT_CLUSTER_ID); + ObClusterResource *acquire_cluster_resource(const common::ObString &cluster_name, const int64_t cluster_id); int acquire_all_avail_cluster_resource(common::ObIArray &cr_array); int acquire_all_cluster_resource(common::ObIArray &cr_array); void release_cluster_resource(ObClusterResource *cr); @@ -442,7 +442,6 @@ public: typedef common::hash::ObBuildInHashMap ObCRHashMap; private: - ObClusterResource *acquire_cluster_resource(const common::ObString &cluster_name, const int64_t cluster_id); int init_cluster_resource_cont(event::ObContinuation &cont, const common::ObString &cluster_name, const int64_t cluster_id, diff --git a/src/obproxy/obutils/ob_server_state_processor.cpp b/src/obproxy/obutils/ob_server_state_processor.cpp index d0f6c60d3bc232a9f80923b2ba2ebb0f270e20f7..ba71c193f63b0089911530f147842555a8cf6267 100644 --- a/src/obproxy/obutils/ob_server_state_processor.cpp +++ b/src/obproxy/obutils/ob_server_state_processor.cpp @@ -16,7 +16,6 @@ #include "utils/ob_proxy_utils.h" #include "utils/ob_proxy_hot_upgrader.h" #include "obutils/ob_congestion_manager.h" -#include "obutils/ob_proxy_json_config_info.h" #include "obutils/ob_config_server_processor.h" #include "obutils/ob_resource_pool_processor.h" #include "proxy/mysqllib/ob_resultset_fetcher.h" @@ -288,23 +287,40 @@ int ObServerStateRefreshCont::main_handler(int event, void *data) if (OB_FAIL(ret)) { bool imm_reschedule = false; + bool has_slave_clusters = get_global_config_server_processor().has_slave_clusters(cluster_name_); // primary-slave cluser mode, if primary cluster refresh failed count greater than three, need reschedule now if (ss_refresh_failure_ >= MIN_REFRESH_FAILURE && OB_DEFAULT_CLUSTER_ID == cluster_id_ - && get_global_config_server_processor().has_slave_clusters(cluster_name_)) { + && has_slave_clusters) { imm_reschedule = true; LOG_DEBUG("primary cluster server state task has failed more than MIN_REFRESH_FAILURE times, will schedule immediately", K(ss_refresh_failure_), K(static_cast(MIN_REFRESH_FAILURE)), K(cluster_name_), K(cluster_id_)); } - if (ss_refresh_failure_ >= MAX_REFRESH_FAILURE) { + if (-OB_CLUSTER_NO_MATCH == ret) { imm_reschedule = false; - LOG_INFO("refresh server state has failed too many times, should refresh rslist", - K_(ss_refresh_failure), K(static_cast(MAX_REFRESH_FAILURE)), K(cluster_name_), K(cluster_id_)); - bool need_update_dummy_entry = true; - if (OB_FAIL(add_refresh_rslist_task(need_update_dummy_entry))) { - LOG_WARN("fail to add refresh rslist task", K(ret)); + if (OB_FAIL(handle_delete_cluster_resource(OB_DEFAULT_CLUSTER_ID))) { + LOG_WARN("fail to delete cluster resource", K(ret)); + } + } else if (ss_refresh_failure_ >= MAX_REFRESH_FAILURE) { + bool need_refresh_cluster_role = get_global_proxy_config().with_config_server_ + && get_global_proxy_config().enable_standby + && OB_DEFAULT_CLUSTER_ID == cluster_id_ + && has_slave_clusters; + imm_reschedule = false; + LOG_INFO("refresh server state has failed too many times, should refresh rslist or delete cluster resource", + K_(ss_refresh_failure), K(static_cast(MAX_REFRESH_FAILURE)), K(cluster_name_), K(cluster_id_), K(need_refresh_cluster_role)); + + if (need_refresh_cluster_role) { + if (OB_FAIL(handle_delete_cluster_resource(OB_DEFAULT_CLUSTER_ID))) { + LOG_WARN("fail to delete cluster resource", K(ret)); + } } else { - ss_refresh_failure_ = 0; + bool need_update_dummy_entry = true; + if (OB_FAIL(add_refresh_rslist_task(need_update_dummy_entry))) { + LOG_WARN("fail to add refresh rslist task", K(ret)); + } else { + ss_refresh_failure_ = 0; + } } } // ignore ret, schedule next; @@ -469,6 +485,14 @@ int ObServerStateRefreshCont::handle_ldg_info(void *data) } if (ret != OB_ITER_END) { // some cluster do not support LDG + ret = OB_SUCCESS; + if (ER_TABLEACCESS_DENIED_ERROR == resp->get_err_code()) { + LOG_DEBUG("access denied for ldg_standby_status"); + } else if (ER_NO_SUCH_TABLE == resp->get_err_code()) { + LOG_DEBUG("table ldg_standby_status not exist"); + } else { + LOG_WARN("fail to get all ldg info", K(ret), K(resp->get_err_code())); + } LOG_DEBUG("fail to get ldg info", K(ret), K_(cluster_name)); } else { ret = OB_SUCCESS; @@ -529,6 +553,33 @@ int ObServerStateRefreshCont::handle_all_tenant(void *data) LOG_WARN("fail to schedule refresh server state", K(ret)); } } + + return ret; +} + +int ObServerStateRefreshCont::handle_delete_cluster_resource(int64_t master_cluster_id) +{ + int ret = OB_SUCCESS; + + //1. if master cluster id changed, update master cluster id + ObConfigServerProcessor &csp = get_global_config_server_processor(); + if (OB_DEFAULT_CLUSTER_ID == master_cluster_id) { + LOG_INFO("current cluster has been switched to STANDBY or FailOver, but ob does not return new primary cluster id", K_(cluster_name)); + } else if (OB_FAIL(csp.set_master_cluster_id(cluster_name_, master_cluster_id))) { + LOG_WARN("fail to set master cluster id", K_(cluster_name), K(master_cluster_id), K(ret)); + ret = OB_SUCCESS; + } + //2. delete cluster resource + ObResourcePoolProcessor &rpp = get_global_resource_pool_processor(); + if (cluster_name_ == OB_META_DB_CLUSTER_NAME) { + const bool ignore_cluster_not_exist = true; + if (OB_FAIL(rpp.rebuild_metadb(ignore_cluster_not_exist))) { + PROXY_CS_LOG(WARN, "fail to rebuild metadb cluster resource", K(ret)); + } + } else if (OB_FAIL(rpp.delete_cluster_resource(cluster_name_, cluster_id_))) { + LOG_WARN("fail to delete cluster resource", K_(cluster_name), K(OB_DEFAULT_CLUSTER_ID), K(ret)); + } + return ret; } @@ -549,23 +600,8 @@ int ObServerStateRefreshCont::handle_cluster_role(void *data) LOG_WARN("fail to check cluster role, will reschedule", K(ret)); if (OB_NOT_MASTER == ret) { int tmp_ret = OB_SUCCESS; - //1. if master cluster id changed, update master cluster id - ObConfigServerProcessor &csp = get_global_config_server_processor(); - if (OB_DEFAULT_CLUSTER_ID == master_cluster_id) { - LOG_INFO("current cluster has been switched to STANDBY, but ob does not return new primary cluster id", K_(cluster_name)); - } else if (OB_SUCCESS != (tmp_ret = csp.set_master_cluster_id(cluster_name_, master_cluster_id))) { - LOG_WARN("fail to set master cluster id", K_(cluster_name), K(master_cluster_id), K(tmp_ret)); - tmp_ret = OB_SUCCESS; - } - //2. delete cluster resource - ObResourcePoolProcessor &rpp = get_global_resource_pool_processor(); - if (cluster_name_ == OB_META_DB_CLUSTER_NAME) { - const bool ignore_cluster_not_exist = true; - if (OB_SUCCESS != (tmp_ret = rpp.rebuild_metadb(ignore_cluster_not_exist))) { - PROXY_CS_LOG(WARN, "fail to rebuild metadb cluster resource", K(tmp_ret)); - } - } else if (OB_SUCCESS != (tmp_ret = rpp.delete_cluster_resource(cluster_name_, OB_DEFAULT_CLUSTER_ID))) { - LOG_WARN("fail to delete cluster resource", K_(cluster_name), K(OB_DEFAULT_CLUSTER_ID), K(tmp_ret)); + if (OB_SUCCESS != (tmp_ret = handle_delete_cluster_resource(master_cluster_id))) { + LOG_WARN("fail to delete cluster resource", K_(cluster_name), K(master_cluster_id), K(tmp_ret)); } // cluster resouce delete succes, no need reschedule refresh task ret = OB_SUCC(tmp_ret) ? tmp_ret : ret; @@ -1229,89 +1265,13 @@ int ObServerStateRefreshCont::check_add_refresh_idc_list_task() int ObServerStateRefreshCont::update_all_dummy_entry(const ObIArray &servers_state) { int ret = OB_SUCCESS; - const int64_t server_count = servers_state.count(); - ObSEArray first_idx_in_zone;//save first svr_idx_idx in zone LocationList server_list; //save the ordered servers - ObString last_zone_name; - const ObServerStateInfo *server_info = NULL; - // 1. fill first_idx_in_zone - for (int64_t i = 0; i < server_count && OB_SUCC(ret); ++i) { - server_info = &servers_state.at(i); - if (last_zone_name != server_info->zone_state_->zone_name_) { - // save first idx in zone - if (OB_FAIL(first_idx_in_zone.push_back(i))) { - LOG_WARN("fail to push back first_idx_in_zone", K(i), K(ret)); - } else { - last_zone_name = server_info->zone_state_->zone_name_; - } - }//end of different zone - }//end of for - - if (OB_SUCC(ret)) { - //virtual invalid idx, just used for compute svr_count_in_zone - if (OB_FAIL(first_idx_in_zone.push_back(server_count))) { - LOG_WARN("fail to push back first_idx_in_zone", K(server_count), K(ret)); - } + if (OB_FAIL(ObServerStateRefreshUtils::order_servers_state(servers_state, server_list))) { + LOG_WARN("fail to order servers state", K(ret)); } - LOG_DEBUG("current info", K(servers_state), K(server_count), K(first_idx_in_zone), K(ret)); - - // 2. order servers - if (OB_SUCC(ret) - && OB_LIKELY(server_count > 0) - && OB_LIKELY(first_idx_in_zone.count() > 1)) { - const int64_t zone_count = first_idx_in_zone.count() - 1;//NOTE:: the last one is virtual invalid; - const int64_t replica_count = zone_count; - const int64_t partition_count = (server_count / replica_count + (0 == server_count % replica_count ? 0 : 1)); - ObSEArray unused_server_count; - int64_t svr_count_in_zone = 0; - - for (int64_t i = 0; i < zone_count && OB_SUCC(ret); ++i) { - if (OB_UNLIKELY(0 >= (svr_count_in_zone = first_idx_in_zone.at(i + 1) - first_idx_in_zone.at(i)))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("it should not happened", K(first_idx_in_zone), K(svr_count_in_zone), K(ret)); - } else if (OB_FAIL(unused_server_count.push_back(svr_count_in_zone))) { - LOG_WARN("fail to push back unused_server_count", K(i), K(svr_count_in_zone), K(ret)); - } else {/*do nothing*/} - } - if (OB_SUCC(ret)) { - // random to pick zone - int64_t init_idx = (get_hrtime() + *(static_cast(&server_count))) % server_count; - int64_t finish_count = 0; - int64_t svr_idx = 0; - LOG_DEBUG("begin to extract rs list", K(init_idx)); - while (finish_count < server_count && OB_SUCC(ret)) { - ++init_idx; - for (int64_t i = 0; i < zone_count && OB_SUCC(ret); ++i) { - if (unused_server_count.at(i) > 0) { - svr_count_in_zone = first_idx_in_zone.at(i + 1) - first_idx_in_zone.at(i); - svr_idx = first_idx_in_zone.at(i) + (init_idx + i) % svr_count_in_zone; - if (OB_FAIL(server_list.push_back(servers_state.at(svr_idx).replica_))) { - LOG_WARN("fail to push back server_list", "replica", servers_state.at(svr_idx).replica_, - K(i), K(unused_server_count), K(ret)); - } else { - ++finish_count; - --unused_server_count.at(i); - } - } else { - LOG_DEBUG("no need try", K(i), K(unused_server_count)); - } //end of unused_server_count - } //end of for zone_count - } //end of while server_count - - //make the first one leader - if (OB_SUCC(ret)) { - for (int64_t i = 0; i < partition_count && OB_SUCC(ret); ++i) { - svr_idx = i * replica_count; - server_list.at(svr_idx).role_ = LEADER; - } - LOG_DEBUG("succ to get rs list", K(server_list), K(partition_count), K(replica_count)); - } - } - } //end of order servers - - // 3. update rslist + // update rslist bool need_update_dummy_entry = true; bool is_rs_list_changed = false; if (OB_SUCC(ret) && OB_LIKELY(!server_list.empty())) { @@ -1346,7 +1306,7 @@ int ObServerStateRefreshCont::update_all_dummy_entry(const ObIArray &servers_state, LocationList &server_list) +{ + int ret = OB_SUCCESS; + const int64_t server_count = servers_state.count(); + ObSEArray first_idx_in_zone;//save first svr_idx_idx in zone + ObString last_zone_name; + const ObServerStateInfo *server_info = NULL; + + // 1. fill first_idx_in_zone + for (int64_t i = 0; i < server_count && OB_SUCC(ret); ++i) { + server_info = &servers_state.at(i); + if (last_zone_name != server_info->zone_state_->zone_name_) { + // save first idx in zone + if (OB_FAIL(first_idx_in_zone.push_back(i))) { + LOG_WARN("fail to push back first_idx_in_zone", K(i), K(ret)); + } else { + last_zone_name = server_info->zone_state_->zone_name_; + } + }//end of different zone + }//end of for + + if (OB_SUCC(ret)) { + //virtual invalid idx, just used for compute svr_count_in_zone + if (OB_FAIL(first_idx_in_zone.push_back(server_count))) { + LOG_WARN("fail to push back first_idx_in_zone", K(server_count), K(ret)); + } + } + + LOG_DEBUG("current info", K(servers_state), K(server_count), K(first_idx_in_zone), K(ret)); + + // 2. order servers + if (OB_SUCC(ret) + && OB_LIKELY(server_count > 0) + && OB_LIKELY(first_idx_in_zone.count() > 1)) { + const int64_t zone_count = first_idx_in_zone.count() - 1;//NOTE:: the last one is virtual invalid; + const int64_t replica_count = zone_count; + const int64_t partition_count = (server_count / replica_count + (0 == server_count % replica_count ? 0 : 1)); + ObSEArray unused_server_count; + int64_t svr_count_in_zone = 0; + + for (int64_t i = 0; i < zone_count && OB_SUCC(ret); ++i) { + if (OB_UNLIKELY(0 >= (svr_count_in_zone = first_idx_in_zone.at(i + 1) - first_idx_in_zone.at(i)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("it should not happened", K(first_idx_in_zone), K(svr_count_in_zone), K(ret)); + } else if (OB_FAIL(unused_server_count.push_back(svr_count_in_zone))) { + LOG_WARN("fail to push back unused_server_count", K(i), K(svr_count_in_zone), K(ret)); + } else {/*do nothing*/} + } + if (OB_SUCC(ret)) { + // random to pick zone + int64_t init_idx = (get_hrtime() + *(static_cast(&server_count))) % server_count; + int64_t finish_count = 0; + int64_t svr_idx = 0; + LOG_DEBUG("begin to extract rs list", K(init_idx)); + while (finish_count < server_count && OB_SUCC(ret)) { + ++init_idx; + for (int64_t i = 0; i < zone_count && OB_SUCC(ret); ++i) { + if (unused_server_count.at(i) > 0) { + svr_count_in_zone = first_idx_in_zone.at(i + 1) - first_idx_in_zone.at(i); + svr_idx = first_idx_in_zone.at(i) + (init_idx + i) % svr_count_in_zone; + if (OB_FAIL(server_list.push_back(servers_state.at(svr_idx).replica_))) { + LOG_WARN("fail to push back server_list", "replica", servers_state.at(svr_idx).replica_, + K(i), K(unused_server_count), K(ret)); + } else { + ++finish_count; + --unused_server_count.at(i); + } + } else { + LOG_DEBUG("no need try", K(i), K(unused_server_count)); + } //end of unused_server_count + } //end of for zone_count + } //end of while server_count + + //make the first one leader + if (OB_SUCC(ret)) { + for (int64_t i = 0; i < partition_count && OB_SUCC(ret); ++i) { + svr_idx = i * replica_count; + server_list.at(svr_idx).role_ = LEADER; + } + LOG_DEBUG("succ to get rs list", K(server_list), K(partition_count), K(replica_count)); + } + } + } //end of order servers + + return ret; +} + } // end of namespace obutils } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/obutils/ob_server_state_processor.h b/src/obproxy/obutils/ob_server_state_processor.h index 5cc5d78010d6ebb02248041220d61ce805ddacb3..637b5afa4485963b67433a84d1fd4ced5941a684 100644 --- a/src/obproxy/obutils/ob_server_state_processor.h +++ b/src/obproxy/obutils/ob_server_state_processor.h @@ -13,6 +13,7 @@ #ifndef OBPROXY_STATE_PROCESSOR_H #define OBPROXY_STATE_PROCESSOR_H #include "obutils/ob_state_info.h" +#include "obutils/ob_proxy_json_config_info.h" #include "lib/container/ob_se_array.h" #include "iocore/eventsystem/ob_continuation.h" #include "iocore/eventsystem/ob_lock.h" @@ -69,6 +70,7 @@ private: int refresh_ldg_info(); int refresh_all_tenant(); + int handle_delete_cluster_resource(int64_t master_cluster_id); int handle_cluster_role(void *data); int handle_zone_state(void *data); int handle_server_state(void *data); @@ -150,6 +152,7 @@ public: static uint64_t get_zones_state_hash(common::ObIArray &zones_state); static uint64_t get_servers_state_hash(common::ObIArray &servers_state); static uint64_t get_servers_addr_hash(common::ObIArray &servers_state); + static int order_servers_state(const common::ObIArray &servers_state, LocationList &server_list); }; } // end of namespace obutils diff --git a/src/obproxy/omt/Makemodule.am b/src/obproxy/omt/Makemodule.am new file mode 100644 index 0000000000000000000000000000000000000000..0f7b2cef6ffb7cfbc2642c028ceadb68bd83357e --- /dev/null +++ b/src/obproxy/omt/Makemodule.am @@ -0,0 +1,9 @@ +omt_sources:=\ +obproxy/omt/ob_white_list_table_processor.h\ +obproxy/omt/ob_white_list_table_processor.cpp\ +obproxy/omt/ob_resource_unit_table_processor.h\ +obproxy/omt/ob_resource_unit_table_processor.cpp\ +obproxy/omt/ob_vip_tenant_conn.h\ +obproxy/omt/ob_vip_tenant_conn.cpp\ +obproxy/omt/ob_ssl_config_table_processor.h\ +obproxy/omt/ob_ssl_config_table_processor.cpp diff --git a/src/obproxy/omt/ob_resource_unit_table_processor.cpp b/src/obproxy/omt/ob_resource_unit_table_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4603fdb344c708ebafc87cca625a4496a635072b --- /dev/null +++ b/src/obproxy/omt/ob_resource_unit_table_processor.cpp @@ -0,0 +1,622 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include "ob_resource_unit_table_processor.h" +#include "obutils/ob_proxy_json_config_info.h" + +using namespace obsys; +using namespace oceanbase::common; +using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::json; + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +static const char* JSON_OBPROXY_VIP = "vip"; +static const char* JSON_OBPROXY_VALUE = "value"; +static const uint32_t column_num = 4; + +using namespace oceanbase::common; + +ObResourceUnitTableProcessor::ObResourceUnitTableProcessor() + : is_inited_(false), backup_status_(false), rwlock_(), used_conn_rwlock_() +{ +} + +void ObResourceUnitTableProcessor::destroy() +{ + if (OB_LIKELY(is_inited_)) { + is_inited_ = false; + DRWLock::WRLockGuard guard(rwlock_); + vt_conn_cache_.destroy(); + } +} + +int ObResourceUnitTableProcessor::init() +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else { + ObConfigHandler handler; + ObString table_name = ObString::make_string("resource_unit"); + handler.execute_func_ = &execute; + handler.commit_func_ = &commit; + handler.config_type_ = OBPROXY_CONFIG_CLOUD; + if (OB_FAIL(get_global_config_processor().register_callback(table_name, handler))) { + LOG_WARN("register callback info failed", K(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::execute(void* cloud_params) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(cloud_params)) { + LOG_WARN("params should not be null"); + ret = OB_INVALID_ARGUMENT; + } else { + ObCloudFnParams* params = (ObCloudFnParams*)cloud_params; + ObString name_str; + ObString value_str; + ObString cluster_name = params->cluster_name_; + ObString tenant_name = params->tenant_name_; + SqlFieldResult* fields = params->fields_; + ObProxyBasicStmtType stmt_type = params->stmt_type_; + if (OB_UNLIKELY(cluster_name.empty()) || OB_UNLIKELY(tenant_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_name or cluster_name is null", K(ret), K(cluster_name), K(tenant_name)); + } else if (OB_ISNULL(fields)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fields is null unexpected", K(ret)); + } else { + // The storage format is:[cluster|tenant|name|value] + for (int64_t i = 0; i < fields->field_num_; i++) { + SqlField& sql_field = fields->fields_.at(i); + if (0 == sql_field.column_name_.string_.case_compare("name")) { + name_str = sql_field.column_value_.config_string_; + } else if (0 == sql_field.column_name_.string_.case_compare("value")) { + value_str = sql_field.column_value_.config_string_; + } + } + + if (OBPROXY_T_REPLACE == stmt_type) { + if (OB_FAIL(get_global_resource_unit_table_processor().handle_replace_config( + cluster_name, tenant_name, name_str, value_str))) { + LOG_WARN("fail to handle replace cmd"); + } + } else if (OBPROXY_T_DELETE == stmt_type) { + if (OB_FAIL(get_global_resource_unit_table_processor().handle_delete_config( + cluster_name, tenant_name, name_str))) { + LOG_WARN("fail to handle delete cmd"); + } + } else { + LOG_WARN("operation is unexpected"); + ret = OB_NOT_SUPPORTED; + } + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::commit(bool is_success) +{ + int ret = OB_SUCCESS; + + if (!is_success) { + ret = get_global_resource_unit_table_processor().rollback(); + } + get_global_resource_unit_table_processor().set_backup_status(false); + + return ret; +} + +bool ObResourceUnitTableProcessor::check_and_inc_conn( + ObString& cluster_name, ObString& tenant_name, ObString& ip_name) +{ + int ret = OB_SUCCESS; + bool throttle = false; + ObVipTenantConn* vt_conn = NULL; + int64_t cur_used_connections = 0; + + if (OB_FAIL(inc_conn(cluster_name, tenant_name, ip_name, cur_used_connections))) { + throttle = true; + LOG_WARN("fail to get or create used conn", K(cluster_name), K(tenant_name), K(ip_name), K(ret)); + } + + if (OB_SUCC(ret)) { + DRWLock::RDLockGuard guard(rwlock_); + if (OB_FAIL(get_vt_conn_object(cluster_name, tenant_name, ip_name, vt_conn))) { + if (OB_ENTRY_NOT_EXIST == ret) { + // Scheme review comments: Connections without configuration information are allowed to access + LOG_DEBUG("get vip tenant connect failed", K(ret)); + } else { + // Other errors, access denied + dec_conn(cluster_name, tenant_name, ip_name); + throttle = true; + } + } else { + if (cur_used_connections <= vt_conn->max_connections_) { + LOG_DEBUG("vip tenant connect info", K(cur_used_connections), KPC(vt_conn)); + } else { + LOG_WARN("used connections reach throttle", K(cur_used_connections), K(vt_conn->max_connections_), KPC(vt_conn)); + dec_conn(cluster_name, tenant_name, ip_name); + throttle = true; + } + } + } + + return throttle; +} + +int ObResourceUnitTableProcessor::inc_conn(ObString& cluster_name, ObString& tenant_name, ObString& ip_name, + int64_t& cur_used_connections) +{ + int ret = OB_SUCCESS; + ObString key_name; + ObUsedConn* used_conn = NULL; + common::ObFixedLengthString key_string; + if (OB_FAIL(build_tenant_cluster_vip_name(tenant_name, cluster_name, ip_name, key_string))) { + LOG_WARN("build tenant cluser vip name failed", K(tenant_name), K(cluster_name), K(ip_name), K(ret)); + } else { + key_name = ObString::make_string(key_string.ptr()); + if (OB_FAIL(get_or_create_used_conn(key_name, used_conn, cur_used_connections))) { + LOG_WARN("create used conn failed", K(key_name), K(ret)); + } else { + used_conn->dec_ref(); + } + } + return ret; +} + +void ObResourceUnitTableProcessor::dec_conn( + ObString& cluster_name, ObString& tenant_name, ObString& ip_name) +{ + int ret = OB_SUCCESS; + ObUsedConn* used_conn = NULL; + int64_t cur_used_connections = 0; + common::ObFixedLengthString key_string; + + if (OB_FAIL(build_tenant_cluster_vip_name(tenant_name, cluster_name, ip_name, key_string))) { + LOG_WARN("build tenant cluster vip name failed", K(ret), K(tenant_name), K(cluster_name), K(ip_name)); + } else { + ObString key_name = ObString::make_string(key_string.ptr()); + if (OB_FAIL(get_used_conn(key_name, false, used_conn, cur_used_connections))) { + LOG_WARN("fail to get used conn in map", K(key_name), K(ret)); + } else { + if (OB_NOT_NULL(used_conn)) { + if (ATOMIC_FAA(&used_conn->max_used_connections_, -1) > 1) { + } else { + DRWLock::WRLockGuard guard(used_conn_rwlock_); + if (0 == used_conn->max_used_connections_) { + erase_used_conn(key_name); + used_conn->dec_ref(); + LOG_DEBUG("erase used conn", K(key_name)); + } + } + used_conn->dec_ref(); + } + } + } +} + +int ObResourceUnitTableProcessor::get_vt_conn_object( + ObString& cluster_name, ObString& tenant_name, ObString& vip_name, ObVipTenantConn*& vt_conn) +{ + int ret = OB_SUCCESS; + + common::ObFixedLengthString key_string; + if (OB_FAIL(build_tenant_cluster_vip_name(tenant_name, cluster_name, vip_name, key_string))) { + LOG_WARN("build tenant cluser vip name failed", K(ret), K(tenant_name), K(cluster_name), K(vip_name)); + } else { + ObString key_name = ObString::make_string(key_string.ptr()); + if (OB_FAIL(vt_conn_cache_.get(key_name, vt_conn))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_DEBUG("vip tenant connect not in cache", K(cluster_name), K(tenant_name), K(vip_name), K(ret)); + } else { + LOG_WARN("fail to get vip tenant connect in cache", K(cluster_name), K(tenant_name), K(vip_name), K(ret)); + } + } else { + LOG_DEBUG("succ to get vip tenant connect in cache", K(cluster_name), K(tenant_name), K(vip_name), KPC(vt_conn)); + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::build_tenant_cluster_vip_name(const ObString &tenant_name, const ObString &cluster_name, + const ObString &vip_name, ObFixedLengthString &key_string) +{ + int ret = OB_SUCCESS; + char buf[OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF]; + int64_t len = 0; + len = static_cast(snprintf(buf, OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF, "%.*s#%.*s|%.*s", + tenant_name.length(), tenant_name.ptr(), + cluster_name.length(), cluster_name.ptr(), + vip_name.length(), vip_name.ptr())); + if (OB_UNLIKELY(len <= 0) || OB_UNLIKELY(len >= OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to fill buf", K(ret), K(tenant_name), K(cluster_name), K(vip_name)); + } else if (OB_FAIL(key_string.assign(buf))) { + LOG_WARN("assign failed", K(ret)); + } + + return ret; +} + +int ObResourceUnitTableProcessor::alloc_and_init_vt_conn( + ObString& cluster_name, ObString& tenant_name, ObString& vip_name, + uint32_t max_connections, ObVipTenantConn*& vt_conn) +{ + int ret = OB_SUCCESS; + ObVipTenantConn* tmp_vt_conn = NULL; + + if (OB_UNLIKELY(cluster_name.empty()) || OB_UNLIKELY(tenant_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name is empty", K(cluster_name), K(tenant_name), K(vip_name), K(ret)); + } else if (OB_ISNULL(tmp_vt_conn = op_alloc(ObVipTenantConn))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory for ObVipTenantConn", K(ret)); + } else if (OB_FAIL(tmp_vt_conn->set(cluster_name, tenant_name, vip_name, max_connections))) { + LOG_WARN("fail to set vip tenant connect info", K(ret)); + } else { + vt_conn = tmp_vt_conn; + LOG_DEBUG("succ to set vip tenant connect object", K(*vt_conn)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(tmp_vt_conn)) { + ob_free(tmp_vt_conn); + tmp_vt_conn = NULL; + } + + return ret; +} + +// local vip json format: +//[ +// { +// "vip" : "127.0.0.1", +// "value" : "1000" +// }, +// { +// "vip" : "0.0.0.1", +// "value" : "2000" +// } +//] +int ObResourceUnitTableProcessor::fill_local_vt_conn_cache( + ObString& cluster_name, ObString& tenant_name, ObString& vip_name) +{ + int ret = OB_SUCCESS; + Parser parser; + json::Value *info_config = NULL; + ObArenaAllocator json_allocator(ObModIds::OB_JSON_PARSER); + + int max_connections = 0; + ObString vip_list_str; + if (OB_UNLIKELY(cluster_name.empty()) || OB_UNLIKELY(tenant_name.empty()) || OB_UNLIKELY(vip_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name is empty", K(cluster_name), K(tenant_name), K(vip_name), K(ret)); + } else if (OB_FAIL(parser.init(&json_allocator))) { + LOG_WARN("json parser init failed", K(ret)); + } else if (OB_FAIL(parser.parse(vip_name.ptr(), vip_name.length(), info_config))) { + LOG_WARN("parse json failed", K(ret), "vip_json_str", get_print_json(vip_name)); + } else if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(info_config, json::JT_ARRAY))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + DLIST_FOREACH(it, info_config->get_array()) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(it, json::JT_OBJECT))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + DLIST_FOREACH(p, it->get_object()) { + if (p->name_ == JSON_OBPROXY_VIP) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_STRING))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + vip_list_str = p->value_->get_string(); + } + } else if (p->name_ == JSON_OBPROXY_VALUE) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_NUMBER))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + max_connections = (uint32_t)p->value_->get_number(); + } + } + } + } + + ObVipTenantConn* vt_conn = NULL; + if (OB_SUCC(ret)) { + DRWLock::WRLockGuard guard(rwlock_); + if (OB_FAIL(get_vt_conn_object(cluster_name, tenant_name, vip_list_str, vt_conn))) { + if (OB_ENTRY_NOT_EXIST == ret) { + if (OB_FAIL(alloc_and_init_vt_conn(cluster_name, tenant_name, vip_list_str, max_connections, vt_conn))) { + LOG_WARN("fail to alloc and init vip tenant connect", K(ret)); + } else if (OB_FAIL(vt_conn_cache_.set(vt_conn))) { + LOG_WARN("fail to insert one vip tenant conn into conn_map", K(vt_conn), K(ret)); + if (OB_LIKELY(NULL != vt_conn)) { + vt_conn->destroy(); + vt_conn = NULL; + } + } else { + LOG_DEBUG("succ to insert one vip tenant connect into conn_map", K(*vt_conn)); + } + } + } else { + vt_conn->max_connections_ = max_connections; + LOG_DEBUG("update vip tenant connect in cache", K(vt_conn)); + } + } + + if (OB_FAIL(ret)) { + if (OB_LIKELY(NULL != info_config)) { + info_config = NULL; + } + } + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::backup_local_vt_conn_cache() +{ + int ret = OB_SUCCESS; + + DRWLock::WRLockGuard guard(rwlock_); + if (OB_FAIL(vt_conn_cache_.backup())) { + LOG_WARN("backup connect cache failed"); + } else { + backup_status_ = true; + } + + return ret; +} + +int ObResourceUnitTableProcessor::handle_replace_config( + ObString& cluster_name, ObString& tenant_name, ObString& name_str, ObString& value_str) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(name_str.empty()) || OB_UNLIKELY(value_str.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name or value is null", K(ret), K(name_str), K(value_str)); + } else { + if (0 == name_str.case_compare("resource_max_connections")) { + if (OB_FAIL(conn_handle_replace_config(cluster_name, tenant_name, name_str, value_str))) { + LOG_WARN("fail to replace vip tenant connection"); + } + } else if (0 == name_str.case_compare("resource_memory")) { + // TODO:qianyuan.rqy + ret = OB_NOT_SUPPORTED; + LOG_WARN("now not support", K(name_str), K(ret)); + } else if (0 == name_str.case_compare("resource_cpu")) { + // TODO:qianyuan.rqy + ret = OB_NOT_SUPPORTED; + LOG_WARN("now not support", K(name_str), K(ret)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is unexpected", K(name_str)); + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::handle_delete_config( + ObString& cluster_name, ObString& tenant_name, ObString& name_str) +{ + int ret = OB_SUCCESS; + + if (name_str.empty()) { + // Remove all resource isolation configurations + if (OB_FAIL(conn_handle_delete_config(cluster_name, tenant_name))) { + LOG_WARN("fail to execute vip tenant connection"); + } + // TODO:The configuration information of other features needs to be deleted later + } else if (0 == name_str.case_compare("resource_max_connections")) { + if (OB_FAIL(conn_handle_delete_config(cluster_name, tenant_name))) { + LOG_WARN("fail to execute vip tenant connection"); + } + } else if (0 == name_str.case_compare("resource_memory")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("now not support", K(name_str), K(ret)); + } else if (0 == name_str.case_compare("resource_cpu")) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("now not support", K(name_str), K(ret)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is unexpected", K(name_str)); + } + + return ret; +} + +int ObResourceUnitTableProcessor::conn_handle_replace_config( + ObString& cluster_name, ObString& tenant_name, ObString& name_str, ObString& value_str) +{ + UNUSED(name_str); + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(cluster_name.empty()) || OB_UNLIKELY(tenant_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant or cluster is null", K(ret), K(cluster_name), K(tenant_name)); + } else if (OB_FAIL(backup_local_vt_conn_cache())) { + LOG_WARN("backup vip tenant connect cache failed", K(ret)); + } else if (OB_FAIL(fill_local_vt_conn_cache(cluster_name, tenant_name, value_str))) { + LOG_WARN("update vip tenant connect cache failed", K(ret)); + } else { + LOG_DEBUG("update vip tenant connect cache succed", "count", get_conn_map_count()); + } + + return ret; +} + +int ObResourceUnitTableProcessor::conn_handle_delete_config(ObString& cluster_name, ObString& tenant_name) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(cluster_name.empty()) || OB_UNLIKELY(tenant_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_name or cluster_name is null", K(ret), K(cluster_name), K(tenant_name)); + } else { + if (OB_FAIL(backup_local_vt_conn_cache())) { + LOG_WARN("backup vip tenant connect cache failed", K(ret)); + } else { + DRWLock::WRLockGuard guard(rwlock_); + vt_conn_cache_.erase(cluster_name, tenant_name); + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::rollback() +{ + int ret = OB_SUCCESS; + + DRWLock::WRLockGuard guard(rwlock_); + // The backup can be rolled back only if the backup is successful + if (OB_LIKELY(backup_status_)) { + if (OB_FAIL(vt_conn_cache_.recover())) { + LOG_WARN("recover connect cache failed"); + } + } + + return ret; +} + +int ObResourceUnitTableProcessor::create_used_conn(ObString& key_name, + ObUsedConn*& used_conn, int64_t& cur_used_connections) +{ + int ret = OB_SUCCESS; + DRWLock::WRLockGuard guard(used_conn_rwlock_); + ObUsedConn* tmp_used_conn = NULL; + if (OB_FAIL(used_conn_cache_.get(key_name, tmp_used_conn))) { + if (OB_HASH_NOT_EXIST == ret) { + char *buf = NULL; + int64_t alloc_size = sizeof(ObUsedConn); + if (OB_ISNULL(buf = static_cast(op_fixed_mem_alloc(alloc_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory for used conn", K(alloc_size), K(ret)); + } else { + used_conn = new (buf) ObUsedConn(key_name); + used_conn->inc_ref(); + if (OB_FAIL(used_conn_cache_.set(used_conn))) { + LOG_WARN("fail to set used conn map", K(key_name), KPC(used_conn), K(ret)); + } + } + + if (OB_SUCC(ret)) { + used_conn->inc_ref(); + cur_used_connections = ATOMIC_AAF(&used_conn->max_used_connections_, 1); + } else if (OB_NOT_NULL(used_conn)) { + used_conn->dec_ref(); + used_conn = NULL; + } + } + } else { + used_conn = tmp_used_conn; + used_conn->inc_ref(); + cur_used_connections = ATOMIC_AAF(&used_conn->max_used_connections_, 1); + } + + return ret; +} + +int ObResourceUnitTableProcessor::get_used_conn(ObString& key_name, bool is_need_inc_used_connections, + ObUsedConn*& used_conn, int64_t& cur_used_connections) +{ + int ret = OB_SUCCESS; + DRWLock::RDLockGuard guard(used_conn_rwlock_); + ObUsedConn* tmp_used_conn = NULL; + if (OB_FAIL(used_conn_cache_.get(key_name, tmp_used_conn))) { + } else { + used_conn = tmp_used_conn; + used_conn->inc_ref(); + if (is_need_inc_used_connections) { + cur_used_connections = ATOMIC_AAF(&used_conn->max_used_connections_, 1); + } + LOG_DEBUG("succ to get used conn from map", K(key_name), KPC(used_conn), K(cur_used_connections)); + } + + return ret; +} + +int ObResourceUnitTableProcessor::get_or_create_used_conn(ObString& key_name, + ObUsedConn*& used_conn, int64_t& cur_used_connections) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(get_used_conn(key_name, true, used_conn, cur_used_connections))) { + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(create_used_conn(key_name, used_conn, cur_used_connections))) { + LOG_WARN("fail to create used conn", K(key_name), K(ret)); + } else { + LOG_DEBUG("succ to create used conn", K(key_name), K(cur_used_connections), KPC(used_conn)); + } + } else { + LOG_WARN("fail to get used conn", K(key_name), K(ret)); + } + } + return ret; +} + +int ObResourceUnitTableProcessor::erase_used_conn(ObString& key_name) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(used_conn_cache_.erase(key_name))) { + LOG_WARN("erase used conn failed", K(key_name)); + } + + return ret; +} + +ObResourceUnitTableProcessor &get_global_resource_unit_table_processor() +{ + static ObResourceUnitTableProcessor t_processor; + return t_processor; +} + +} // end of namespace omt +} // end of namespace obproxy +} // end of namespace oceanbase diff --git a/src/obproxy/omt/ob_resource_unit_table_processor.h b/src/obproxy/omt/ob_resource_unit_table_processor.h new file mode 100644 index 0000000000000000000000000000000000000000..f5d49c65dd1b17b3982a5db123db0ff090073d09 --- /dev/null +++ b/src/obproxy/omt/ob_resource_unit_table_processor.h @@ -0,0 +1,124 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OBPROXY_TENANT_PROCESSOR_H +#define OBPROXY_TENANT_PROCESSOR_H + +#include "obutils/ob_config_processor.h" +#include "lib/lock/ob_drw_lock.h" +#include "ob_vip_tenant_conn.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +class ObResourceUnitTableProcessor +{ +public: + ObResourceUnitTableProcessor(); + virtual ~ObResourceUnitTableProcessor() { destroy(); } + + int init(); + void destroy(); + static int execute(void* cloud_params); + static int commit(bool is_success); + + // Handling vip tenant connection related + bool check_and_inc_conn(common::ObString& cluster_name, + common::ObString& tenant_name, common::ObString& ip_name); + + int inc_conn(common::ObString& cluster_name, common::ObString& tenant_name, common::ObString& ip_name, + int64_t& cur_used_connections); + + void dec_conn(common::ObString& cluster_name, + common::ObString& tenant_name, common::ObString& ip_name); + + int build_tenant_cluster_vip_name(const common::ObString &tenant_name, + const common::ObString &cluster_name, const common::ObString &vip_name, + common::ObFixedLengthString& key_string); + + int get_vt_conn_object(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& vip_name, ObVipTenantConn*& vt_conn); + + int alloc_and_init_vt_conn(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& vip_name, uint32_t max_connections, ObVipTenantConn*& vt_conn); + + int fill_local_vt_conn_cache(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& vip_name); + + int backup_local_vt_conn_cache(); + + int handle_replace_config(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& name_str, common::ObString& value_str); + int handle_delete_config(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& name_str); + + int conn_handle_replace_config(common::ObString& cluster_name, common::ObString& tenant_name, + common::ObString& name_str, common::ObString& value_str); + int conn_handle_delete_config(common::ObString& cluster_name, common::ObString& tenant_name); + + ObVipTenantConnCache &get_vt_conn_cache() { return vt_conn_cache_; } + int rollback(); + void set_backup_status(bool status) { backup_status_ = status; } + + int64_t get_conn_map_count() const { return vt_conn_cache_.get_conn_map_count(); } + ObVipTenantConnCache::VTHashMap* get_conn_map() { return vt_conn_cache_.get_conn_map(); } + ObVipTenantConnCache::VTHashMap& get_conn_map_replica() { return vt_conn_cache_.get_conn_map_replica(); } + + int create_used_conn(common::ObString& key_name, ObUsedConn*& used_conn, int64_t& cur_used_connections); + int get_used_conn(common::ObString& key_name, bool is_need_inc_used_connections, ObUsedConn*& used_conn, int64_t& cur_used_connections); + int erase_used_conn(common::ObString& key_name); + int get_or_create_used_conn(common::ObString& key_name, ObUsedConn*& used_conn, int64_t& cur_used_connections); + + TO_STRING_KV(K_(is_inited), K_(backup_status)); + +private: + bool is_inited_; + bool backup_status_; // false: backup fail; true: backup success + + common::DRWLock rwlock_; + ObVipTenantConnCache vt_conn_cache_; + + common::DRWLock used_conn_rwlock_; + ObUsedConnCache used_conn_cache_; + + DISALLOW_COPY_AND_ASSIGN(ObResourceUnitTableProcessor); +}; + +ObResourceUnitTableProcessor &get_global_resource_unit_table_processor(); + +} // end of namespace omt +} // end of namespace obproxy +} // end of namespace oceanbase + +#endif /* OBPROXY_TENANT_PROCESSOR_H */ diff --git a/src/obproxy/omt/ob_ssl_config_table_processor.cpp b/src/obproxy/omt/ob_ssl_config_table_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bfc0be6fbd178dfd55f9c85a50b310478ebcd798 --- /dev/null +++ b/src/obproxy/omt/ob_ssl_config_table_processor.cpp @@ -0,0 +1,521 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include +#include "omt/ob_ssl_config_table_processor.h" +#include "obutils/ob_config_processor.h" +#include "obutils/ob_proxy_json_config_info.h" +#include "utils/ob_proxy_utils.h" +#include "iocore/net/ob_ssl_processor.h" +#include "obutils/ob_proxy_config.h" + +using namespace obsys; +using namespace oceanbase::json; +using namespace oceanbase::common; +using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy::net; + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +int ObSSLConfigTableProcessor::execute(void *arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == arg)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is null unexpected", K(ret)); + } else { + ObCloudFnParams *params = reinterpret_cast(arg); + ObString tenant_name = params->tenant_name_; + ObString cluster_name = params->cluster_name_; + ObString name; + ObString value; + if (NULL == params->fields_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fields is null unexpected", K(ret)); + } else if (cluster_name.empty() || cluster_name.length() > OB_PROXY_MAX_CLUSTER_NAME_LENGTH + || tenant_name.empty() || tenant_name.length() > OB_MAX_TENANT_NAME_LENGTH) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("execute failed, tenant_name or cluster_name is null", K(ret), K(cluster_name), K(tenant_name)); + } else { + for (int64_t i = 0; i < params->fields_->field_num_; i++) { + SqlField &sql_field = params->fields_->fields_.at(i); + if (sql_field.column_name_.string_ == "value") { + value = sql_field.column_value_.config_string_; + } else if (sql_field.column_name_.string_ == "name") { + name = sql_field.column_value_.config_string_; + } + } + + if (params->stmt_type_ == OBPROXY_T_REPLACE) { + if (OB_FAIL(get_global_ssl_config_table_processor().set_ssl_config(cluster_name, tenant_name, name, value))) { + LOG_WARN("set ssl config failed", K(ret), K(cluster_name), K(tenant_name), K(name), K(value)); + } + } else if (params->stmt_type_ == OBPROXY_T_DELETE) { + if (!name.empty() || !value.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("delete failed, name or value is not null", K(ret), K(name), K(value)); + } else if (OB_FAIL(get_global_ssl_config_table_processor().delete_ssl_config(cluster_name, tenant_name))) { + LOG_WARN("delete ssl config failed", K(ret), K(cluster_name), K(tenant_name)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected stmt type", K(ret), K(params->stmt_type_)); + } + } + } + + return ret; +} + +int ObSSLConfigTableProcessor::commit(bool is_success) +{ + if (is_success) { + get_global_ssl_config_table_processor().inc_index(); + get_global_ssl_config_table_processor().handle_delete(); + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + get_global_ssl_config_table_processor().print_config(); + } + } else { + LOG_WARN("ssl config commit failed"); + } + + return OB_SUCCESS; +} + +ObSSLConfigTableProcessor &get_global_ssl_config_table_processor() +{ + static ObSSLConfigTableProcessor g_ssl_config_table_processor; + return g_ssl_config_table_processor; +} + +int ObSSLConfigTableProcessor::init() +{ + int ret = OB_SUCCESS; + ObConfigHandler handler; + handler.execute_func_ = &ObSSLConfigTableProcessor::execute; + handler.commit_func_ = &ObSSLConfigTableProcessor::commit; + handler.config_type_ = OBPROXY_CONFIG_CLOUD; + if (OB_FAIL(ssl_config_map_array_[0].create(32, ObModIds::OB_PROXY_SSL_RELATED))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (OB_FAIL(ssl_config_map_array_[1].create(32, ObModIds::OB_PROXY_SSL_RELATED))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (OB_FAIL(get_global_config_processor().register_callback("ssl_config", handler))) { + LOG_WARN("register ssl config table callback failed", K(ret)); + } + + return ret; +} + +int ObSSLConfigTableProcessor::set_ssl_config(const ObString &cluster_name, + const ObString &tenant_name, + const ObString &name, + const ObString &value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cluster_name.empty() || tenant_name.empty() || name.empty() || value.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("set ssl config failed, invalid argument", K(cluster_name), K(tenant_name), K(name), K(value), K(ret)); + } else if (OB_FAIL(backup_hash_map())) { + LOG_WARN("backup hash map failed", K(ret)); + } else { + ObFixedLengthString key_string; + SSLConfigInfo ssl_config_info; + ssl_config_info.reset(); + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paster tenant and cluser name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + DRWLock::WRLockGuard guard(ssl_config_lock_); + SSLConfigHashMap &backup_map = ssl_config_map_array_[(index_ + 1) % 2]; + if (OB_FAIL(backup_map.get_refactored(key_string, ssl_config_info))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("map get refactored failed", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (0 == name.compare("enable_client_ssl")) { + if (0 == value.case_compare("true")) { + ssl_config_info.enable_client_ssl_ = true; + } else if (0 == value.case_compare("false")) { + ssl_config_info.enable_client_ssl_ = false; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set enable_client_ssl failed", K(ret), K(name), K(value)); + } + + if (OB_SUCC(ret) && cluster_name == "*" && tenant_name == "*") { + if (OB_FAIL(alter_ssl_config(name, value))) { + LOG_WARN("alter ssl config faield", K(ret), K(name), K(value)); + } + } + } else if (0 == name.compare("enable_server_ssl")) { + if (0 == value.case_compare("true")) { + ssl_config_info.enable_server_ssl_ = true; + } else if (0 == value.case_compare("false")) { + ssl_config_info.enable_server_ssl_ = false; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("set enable_server_ssl failed", K(ret), K(name), K(value)); + } + + if (OB_SUCC(ret) && cluster_name == "*" && tenant_name == "*") { + if (OB_FAIL(alter_ssl_config(name, value))) { + LOG_WARN("alter ssl config faield", K(ret), K(name), K(value)); + } + } + } else if (0 == name.compare("key_info")) { + Parser parser; + json::Value *json_value = NULL; + ObArenaAllocator json_allocator(ObModIds::OB_JSON_PARSER); + if (OB_FAIL(parser.init(&json_allocator))) { + LOG_WARN("json parser init failed", K(ret)); + } else if (OB_FAIL(parser.parse(value.ptr(), value.length(), json_value))) { + LOG_WARN("json parse failed", K(ret), K(value)); + } else if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(json_value, json::JT_OBJECT))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + ssl_config_info.is_key_info_valid_ = false; + ObProxyVariantString source_type; + ObProxyVariantString ca_info; + ObProxyVariantString public_key; + ObProxyVariantString private_key; + DLIST_FOREACH(p, json_value->get_object()) { + if (0 == p->name_.case_compare("sourceType")) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_STRING))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + source_type.set_value(p->value_->get_string()); + } + } else if (0 == p->name_.case_compare("CA")) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_STRING))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + ca_info.set_value(p->value_->get_string()); + } + } else if (0 == p->name_.case_compare("publicKey")) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_STRING))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + public_key.set_value(p->value_->get_string()); + } + } else if (0 == p->name_.case_compare("privateKey")) { + if (OB_FAIL(ObProxyJsonUtils::check_config_info_type(p->value_, json::JT_STRING))) { + LOG_WARN("check config info type failed", K(ret)); + } else { + private_key.set_value(p->value_->get_string()); + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(g_ssl_processor.update_key(cluster_name, tenant_name, source_type, + ca_info, public_key, private_key))) { + LOG_WARN("update key failed", K(cluster_name), K(tenant_name), K(source_type), + K(ca_info), K(public_key), K(private_key), K(ret)); + } else { + ssl_config_info.is_key_info_valid_ = true; + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid name for ssl config", K(name), K(ret)); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(backup_map.set_refactored(key_string, ssl_config_info, 1))) { + LOG_WARN("set refactored failed", K(ret)); + } + } + } + + return ret; +} + +int ObSSLConfigTableProcessor::delete_ssl_config(ObString &cluster_name, ObString &tenant_name) +{ + int ret = OB_SUCCESS; + if (cluster_name.empty() || tenant_name.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("cluster name or tenant name empty unexpected", K(ret)); + } else if (OB_FAIL(backup_hash_map())) { + LOG_WARN("backup hash map failed", K(ret)); + } else { + LOG_DEBUG("delete ssl config", K(cluster_name), K(tenant_name)); + ObFixedLengthString key_string; + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + DRWLock::WRLockGuard guard(ssl_config_lock_); + SSLConfigHashMap &backup_map = ssl_config_map_array_[(index_ + 1) % 2]; + if (OB_FAIL(backup_map.erase_refactored(key_string))) { + LOG_WARN("addr hash map erase failed", K(ret)); + } else { + delete_info_ = key_string; + + if (cluster_name == "*" && tenant_name == "*") { + if (OB_FAIL(alter_ssl_config("enable_client_ssl", "false"))) { + LOG_WARN("alter client ssl config failed", K(ret)); + } else if (OB_FAIL(alter_ssl_config("enable_server_ssl", "false"))) { + LOG_WARN("alter server ssl config failed", K(ret)); + } + } + } + } + } + + return ret; +} + +void SSLConfigInfo::reset() +{ + enable_client_ssl_ = false; + enable_server_ssl_ = false; + is_key_info_valid_ = false; +} + +int64_t SSLConfigInfo::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(enable_client_ssl), K_(enable_server_ssl), K_(is_key_info_valid)); + J_OBJ_END(); + return pos; +} + +int ObSSLConfigTableProcessor::backup_hash_map() +{ + int ret = OB_SUCCESS; + int64_t backup_index = (index_ + 1) % 2; + SSLConfigHashMap &backup_map = ssl_config_map_array_[backup_index]; + SSLConfigHashMap ¤t_map = ssl_config_map_array_[index_]; + if (OB_FAIL(backup_map.reuse())) { + LOG_WARN("backup map failed", K(ret)); + } else { + SSLConfigHashMap::iterator iter = current_map.begin(); + for(; OB_SUCC(ret) && iter != current_map.end(); ++iter) { + if (OB_FAIL(backup_map.set_refactored(iter->first, iter->second))) { + LOG_WARN("backup map set refactored failed", K(ret)); + } + } + } + return ret; +} + +void ObSSLConfigTableProcessor::inc_index() +{ + DRWLock::WRLockGuard guard(ssl_config_lock_); + index_ = (index_ + 1) % 2; +} + +void ObSSLConfigTableProcessor::handle_delete() +{ + DRWLock::WRLockGuard guard(ssl_config_lock_); + if (!delete_info_.is_empty()) { + g_ssl_processor.release_ssl_ctx(delete_info_); + delete_info_.reset(); + } +} + +bool ObSSLConfigTableProcessor::is_ssl_supported(const common::ObString &cluster_name, + const common::ObString &tenant_name, + bool is_client) +{ + bool bret = false; + int ret = OB_SUCCESS; + LOG_DEBUG("check ssl support", K(cluster_name), K(tenant_name), K(is_client)); + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + get_global_ssl_config_table_processor().print_config(); + } + SSLConfigInfo ssl_config_info; + ssl_config_info.reset(); + ObFixedLengthString key_string; + DRWLock::RDLockGuard guard(ssl_config_lock_); + SSLConfigHashMap ¤t_map = ssl_config_map_array_[index_]; + // Get tenant configuration + if (OB_SUCC(ret)) { + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else if (OB_FAIL(current_map.get_refactored(key_string, ssl_config_info))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } else { + LOG_DEBUG("get ssl config from tenant succ", K(ssl_config_info)); + } + } + // Get the cluster configuration + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(paste_tenant_and_cluster_name("*", cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else if (OB_FAIL(current_map.get_refactored(key_string, ssl_config_info))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } else { + LOG_DEBUG("get ssl config from cluster succ", K(ssl_config_info)); + } + } + // Get global configuration + if (OB_HASH_NOT_EXIST == ret) { + if (OB_FAIL(paste_tenant_and_cluster_name("*", "*", key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else if (OB_FAIL(current_map.get_refactored(key_string, ssl_config_info))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } + } else { + LOG_DEBUG("get ssl config from global succ", K(ssl_config_info)); + } + } + + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } + + if (OB_SUCC(ret)) { + bool enable_ssl = is_client ? ssl_config_info.enable_client_ssl_ : ssl_config_info.enable_server_ssl_; + bret = enable_ssl && ssl_config_info.is_key_info_valid_; + LOG_DEBUG("get ssl config", K(enable_ssl), K(bret), K(cluster_name), K(tenant_name), K(is_client)); + } + return bret; +} + +void ObSSLConfigTableProcessor::print_config() +{ + SSLConfigHashMap ¤t_map = ssl_config_map_array_[index_]; + SSLConfigHashMap::iterator iter = current_map.begin(); + for(; iter != current_map.end(); ++iter) { + LOG_DEBUG("ssl config map info", K(iter->first), K(iter->second)); + } +} + +bool ObSSLConfigTableProcessor::is_ssl_key_info_valid(const ObString &cluster_name, + const ObString &tenant_name) +{ + bool bret = false; + int ret = OB_SUCCESS; + if (OB_UNLIKELY(cluster_name.empty() || tenant_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(cluster_name), K(tenant_name), K(ret)); + } else { + SSLConfigInfo ssl_config_info; + ssl_config_info.reset(); + ObFixedLengthString key_string; + DRWLock::RDLockGuard guard(ssl_config_lock_); + SSLConfigHashMap ¤t_map = ssl_config_map_array_[index_]; + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster_name failed", K(ret), K(tenant_name), K(cluster_name)); + } else if (OB_FAIL(current_map.get_refactored(key_string, ssl_config_info))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("ssl ctx map get refactored failed", K(ret), K(cluster_name), K(tenant_name)); + } else { + ret = OB_SUCCESS; + } + } else { + LOG_DEBUG("get ssl config from tenant succ", K(ssl_config_info)); + // Currently, the settings ensure that source_type, ca, public_key and private_key are set at the same time + if (ssl_config_info.is_key_info_valid_) { + bret = true; + } + } + } + return bret; +} + +int ObSSLConfigTableProcessor::alter_ssl_config(const common::ObString &key_string, const common::ObString &value_string) +{ + int ret = OB_SUCCESS; + char *old_value = NULL; + bool has_update_config = false; + bool has_dump_config = false; + if (OB_UNLIKELY(key_string.empty() || value_string.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + if (OB_ISNULL(old_value = static_cast(op_fixed_mem_alloc(OB_MAX_CONFIG_VALUE_LEN)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem for old_value", "size", OB_MAX_CONFIG_VALUE_LEN, K(ret)); + } else if (OB_FAIL(get_global_proxy_config().get_old_config_value(key_string, old_value, OB_MAX_CONFIG_VALUE_LEN))) { + LOG_WARN("fail to get old config value", K(key_string), K(ret)); + } else if (OB_FAIL(get_global_proxy_config().update_config_item(key_string, value_string))) { + LOG_WARN("fail to update config", K(key_string), K(value_string), K(ret)); + } else { + has_update_config = true; + LOG_DEBUG("succ to update config", K(key_string), K(value_string), K(old_value)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(get_global_proxy_config().check_proxy_serviceable())) { + LOG_WARN("fail to check proxy serviceable", K(ret)); + } else if (OB_FAIL(get_global_proxy_config().dump_config_to_local())) { + LOG_WARN("fail to dump_config_to_local", K(ret)); + } else { + has_dump_config = true; + } + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + if (has_update_config) { + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = get_global_proxy_config().update_config_item( + key_string, ObString::make_string(old_value))))) { + LOG_WARN("fail to back to old config", K(key_string), K(old_value), K(tmp_ret)); + } else { + LOG_DEBUG("succ to back to old config", K(key_string), K(old_value)); + } + } + if (has_dump_config && OB_LIKELY(OB_SUCCESS == tmp_ret)) { + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = get_global_proxy_config().dump_config_to_local()))) { + LOG_WARN("fail to dump old config", K(tmp_ret)); + } else { + LOG_DEBUG("succ to dump old config"); + } + } + } + } + + return ret; +} + +} // end of omt +} // end of obproxy +} // end of oceanbase diff --git a/src/obproxy/omt/ob_ssl_config_table_processor.h b/src/obproxy/omt/ob_ssl_config_table_processor.h new file mode 100644 index 0000000000000000000000000000000000000000..a8c0311aaa24c86d36289ae99b202552eb6c21fa --- /dev/null +++ b/src/obproxy/omt/ob_ssl_config_table_processor.h @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_SSL_CONFIG_TABLE_PROCESSOR_H_ +#define OB_SSL_CONFIG_TABLE_PROCESSOR_H_ + +#include "obutils/ob_proxy_string_utils.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/lock/ob_drw_lock.h" +#include "utils/ob_proxy_lib.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +struct SSLConfigInfo +{ + SSLConfigInfo() { reset(); } + bool enable_client_ssl_; + bool enable_server_ssl_; + bool is_key_info_valid_; + + void reset(); + int64_t to_string(char *buf, const int64_t buf_len) const; +}; + +class ObSSLConfigTableProcessor +{ +public: + ObSSLConfigTableProcessor() : index_(0), ssl_config_lock_(obsys::WRITE_PRIORITY), delete_info_() {} + ~ObSSLConfigTableProcessor() {} + + int init(); + void inc_index(); + void handle_delete(); + int set_ssl_config(const common::ObString &cluster_name, const common::ObString &tenant_name, + const common::ObString &name, const common::ObString &value); + int delete_ssl_config(common::ObString &cluster_name, common::ObString &tenant_name); + bool is_ssl_supported(const common::ObString &cluster_name, const common::ObString &tenant_name, bool is_client); + bool is_ssl_key_info_valid(const common::ObString &cluster_name, const common::ObString &tenant_name); + void print_config(); +private: + int backup_hash_map(); + int alter_ssl_config(const common::ObString &key, const common::ObString &value); + static int execute(void *arg); + static int commit(bool is_success); + +private: + typedef common::hash::ObHashMap, SSLConfigInfo> SSLConfigHashMap; + SSLConfigHashMap ssl_config_map_array_[2]; + int64_t index_; + common::DRWLock ssl_config_lock_; + common::ObFixedLengthString delete_info_; + +private: +DISALLOW_COPY_AND_ASSIGN(ObSSLConfigTableProcessor); +}; + +ObSSLConfigTableProcessor &get_global_ssl_config_table_processor(); + +} // end of omt +} // end of obproxy +} // end of oceanbase + +#endif diff --git a/src/obproxy/omt/ob_vip_tenant_conn.cpp b/src/obproxy/omt/ob_vip_tenant_conn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..972aca39b9e420b88b9028a2d40cc50e9e707b47 --- /dev/null +++ b/src/obproxy/omt/ob_vip_tenant_conn.cpp @@ -0,0 +1,338 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include "ob_vip_tenant_conn.h" +#include "lib/oblog/ob_log.h" +#include "omt/ob_resource_unit_table_processor.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +using namespace obsys; +using namespace oceanbase::common; + +int ObVipTenantConn::set_tenant_cluster(const ObString &tenant_name, const ObString &cluster_name) +{ + int ret = OB_SUCCESS; + if (tenant_name.empty() || tenant_name.length() > OB_MAX_TENANT_NAME_LENGTH + || cluster_name.empty() || cluster_name.length() > OB_PROXY_MAX_CLUSTER_NAME_LENGTH) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name invalid", K(tenant_name), K(cluster_name), K(ret)); + } else { + MEMCPY(tenant_name_str_, tenant_name.ptr(), tenant_name.length()); + tenant_name_.assign_ptr(tenant_name_str_, tenant_name.length()); + MEMCPY(cluster_name_str_, cluster_name.ptr(), cluster_name.length()); + cluster_name_.assign_ptr(cluster_name_str_, cluster_name.length()); + } + return ret; +} + +int ObVipTenantConn::set_addr(const common::ObString addr) +{ + int ret = OB_SUCCESS; + if (addr.empty() || addr.length() > OB_IP_STR_BUFF) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name invalid", K(addr), K(ret)); + } else { + MEMCPY(vip_name_str_, addr.ptr(), addr.length()); + vip_name_.assign_ptr(vip_name_str_, addr.length()); + } + return ret; +} + +int ObVipTenantConn::set_full_name() +{ + int ret = OB_SUCCESS; + + ObFixedLengthString full_name; + if (OB_FAIL(get_global_resource_unit_table_processor().build_tenant_cluster_vip_name( + tenant_name_, cluster_name_, vip_name_, full_name))) { + LOG_WARN("build tenant cluser and vip name failed", K(ret), K_(tenant_name), K_(cluster_name), K_(vip_name)); + } else { + MEMCPY(full_name_str_, full_name.ptr(), full_name.size()); + full_name_.assign_ptr(full_name_str_, (int32_t)full_name.size()); + } + + return ret; +} + +int ObVipTenantConn::set(const ObString &cluster_name, + const ObString &tenant_name, const ObString &vip_name, uint64_t max_connections) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(set_tenant_cluster(tenant_name, cluster_name))) { + LOG_WARN("fail to set tenant cluster name", K(tenant_name), K(cluster_name), K(ret)); + } else { + if (!vip_name.empty()) { + if (OB_FAIL(set_addr(vip_name))) { + LOG_WARN("fail to set vip name", K(vip_name), K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(set_full_name())) { + LOG_WARN("fail to set full name", K(ret)); + } else { + max_connections_ = max_connections; + } + } + } + + return ret; +} + +void ObVipTenantConn::reset() +{ + tenant_name_.reset(); + cluster_name_.reset(); + vip_name_.reset(); + full_name_.reset(); + max_connections_ = 0; +} + +void ObVipTenantConnCache::destroy() +{ + //CWLockGuard guard(rwlock_); + vt_conn_map_ = NULL; + clear_conn_map(vt_conn_map_array[0]); + clear_conn_map(vt_conn_map_array[1]); +} + +int64_t ObVipTenantConnCache::get_conn_map_count() const +{ + int64_t ret = -1; + //CRLockGuard guard(rwlock_); + if (OB_LIKELY(NULL != vt_conn_map_)) { + ret = vt_conn_map_->count(); + } + + return ret; +} + +int ObVipTenantConnCache::get(ObString& key_name, ObVipTenantConn*& vt_conn) +{ + int ret = OB_SUCCESS; + ObVipTenantConn *tmp_vt_conn = NULL; + + if (key_name.empty() || key_name.length() > OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name invalid", K(key_name), K(ret)); + } else { + //CRLockGuard guard(rwlock_); + if (OB_ISNULL(vt_conn_map_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("vt_conn_map_ should not be null", K(ret)); + } else if (OB_FAIL(vt_conn_map_->get_refactored(key_name, tmp_vt_conn))) { + ret = OB_ENTRY_NOT_EXIST; + LOG_INFO("vip tenant connect is not in vip_tenant_conn_cache", K(key_name), K(ret)); + } else if (OB_ISNULL(tmp_vt_conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("vt get from vt_conn_map is null", K(key_name), K(ret)); + } else { + LOG_DEBUG("vip tenant connect hit in vip_tenant_conn_cache", K(key_name), K(*tmp_vt_conn)); + vt_conn = tmp_vt_conn; + } + } + + return ret; +} + +int ObVipTenantConnCache::set(ObVipTenantConn* vt) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!vt->is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input value", K(vt), K(ret)); + } else { + //CWLockGuard guard(rwlock_); + if (OB_ISNULL(vt_conn_map_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("vt_conn_map_ should not be null", K(ret)); + } else { + ret = vt_conn_map_->unique_set(vt); + if (OB_LIKELY(OB_SUCCESS == ret)) { + LOG_DEBUG("succ to insert vip_tenant into vt_conn_map"); + } else if (OB_HASH_EXIST == ret) { + LOG_DEBUG("vip_tenant alreay exist in the cache map"); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to insert vip_tenant into vt_conn_map", K(ret)); + } + } + } + + return ret; +} + +int ObVipTenantConnCache::erase(ObString& cluster_name, ObString& tenant_name) +{ + int ret = OB_SUCCESS; + + if (cluster_name.empty() || tenant_name.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid input value", K(ret), K(cluster_name), K(tenant_name)); + } else { + //CWLockGuard guard(rwlock_); + VTHashMap::iterator last = vt_conn_map_->end(); + for (VTHashMap::iterator it = vt_conn_map_->begin(); it != last;) { + if (it->cluster_name_ == cluster_name && it->tenant_name_ == tenant_name) { + VTHashMap::iterator tmp = it; + ++it; + vt_conn_map_->erase_refactored(tmp->full_name_); + tmp->destroy(); + } else { + ++it; + } + } + } + + return ret; +} + +int ObVipTenantConnCache::backup() +{ + int ret = OB_SUCCESS; + + //CWLockGuard guard(rwlock_); + // Initialization state: + // vt_conn_map_ = vt_conn_map_array[0]; + // replica_vt_conn_map = vt_conn_map_array[1]; + VTHashMap& replica_vt_conn_map = get_conn_map_replica(); + clear_conn_map(replica_vt_conn_map); + // Even if an error occurs in the backup process and an error is returned to the cloud platform, vt_conn_map_ is still the previous configuration information + VTHashMap::iterator last = vt_conn_map_->end(); + for (VTHashMap::iterator it = vt_conn_map_->begin(); it != last; ++it) { + ObVipTenantConn* tmp_vt_conn = NULL; + if (OB_ISNULL(tmp_vt_conn = op_alloc(ObVipTenantConn))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory for ObVipTenantConn", K(ret)); + } else if (OB_FAIL(tmp_vt_conn->set( + it->cluster_name_, it->tenant_name_, it->vip_name_, it->max_connections_))) { + LOG_WARN("fail to set vip tenant connect info", K(ret)); + } else if (OB_FAIL(replica_vt_conn_map.unique_set(tmp_vt_conn))) { + LOG_WARN("insert vip tenant connection failed", K(ret)); + break; + } + } + + if (OB_FAIL(ret)) { + clear_conn_map(replica_vt_conn_map); + } + + return ret; +} + +int ObVipTenantConnCache::recover() +{ + int ret = OB_SUCCESS; + + //CWLockGuard guard(rwlock_); + if (OB_ISNULL(vt_conn_map_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("vt_conn_map should not be null", K(ret)); + } else { + clear_conn_map(*vt_conn_map_); + if (&vt_conn_map_array[0] != vt_conn_map_) { + vt_conn_map_ = &vt_conn_map_array[0]; + } else { + vt_conn_map_ = &vt_conn_map_array[1]; + } + } + + return ret; +} + +void ObVipTenantConnCache::clear_conn_map(VTHashMap &cache_map) +{ + VTHashMap::iterator last = cache_map.end(); + VTHashMap::iterator tmp; + for (VTHashMap::iterator it = cache_map.begin(); it != last;) { + tmp = it; + ++it; + tmp->destroy(); + } + cache_map.reset(); +} + +void ObVipTenantConnCache::dump_conn_map(VTHashMap &cache_map) +{ + VTHashMap::iterator last = cache_map.end(); + for (VTHashMap::iterator it = cache_map.begin(); it != last; ++it) { + LOG_DEBUG("key: ", K(it->full_name_)); + LOG_DEBUG("value: ", K(it->max_connections_)); + } +} + +int ObUsedConnCache::get(ObString& key_name, ObUsedConn*& used_conn) +{ + int ret = OB_SUCCESS; + if (key_name.empty() || key_name.length() > OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name invalid", K(key_name), K(ret)); + } else if (OB_FAIL(used_conn_map_.get_refactored(key_name, used_conn))) { + LOG_DEBUG("fail to get used conn", K(key_name), K(ret)); + } + return ret; +} + +int ObUsedConnCache::set(ObUsedConn* used_conn) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(used_conn)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("used conn is null", K(ret)); + } else if (OB_FAIL(used_conn_map_.unique_set(used_conn))) { + LOG_WARN("fail to set used conn", K(used_conn), K(ret)); + } + return ret; +} + +int ObUsedConnCache::erase(ObString& key_name) +{ + int ret = OB_SUCCESS; + if (key_name.empty() || key_name.length() > OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + OB_IP_STR_BUFF) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("name invalid", K(key_name), K(ret)); + } else if (OB_FAIL(used_conn_map_.erase_refactored(key_name))) { + LOG_WARN("erase used conn failed", K(key_name), K(ret)); + } + return ret; +} + +} // end of namespace omt +} // end of namespace obproxy +} // end of namespace oceanbase diff --git a/src/obproxy/omt/ob_vip_tenant_conn.h b/src/obproxy/omt/ob_vip_tenant_conn.h new file mode 100644 index 0000000000000000000000000000000000000000..fb5cd864309f63f1ece70b4be3af6c29cbf90e7d --- /dev/null +++ b/src/obproxy/omt/ob_vip_tenant_conn.h @@ -0,0 +1,197 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_VIP_TENANT_CONN_H +#define OB_VIP_TENANT_CONN_H + +#include "lib/net/ob_addr.h" +#include "lib/hash/ob_build_in_hashmap.h" +#include "utils/ob_proxy_lib.h" +#include "iocore/eventsystem/ob_buf_allocator.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +class ObVipTenantConn { +public: + ObVipTenantConn() : cluster_name_(), tenant_name_(), vip_name_(), + full_name_(), max_connections_(0) + { + tenant_name_str_[0] = '\0'; + cluster_name_str_[0] = '\0'; + vip_name_str_[0] = '\0'; + full_name_str_[0] = '\0'; + memset(&vt_link_, 0, sizeof(vt_link_)); + } + virtual ~ObVipTenantConn() { reset(); }; + void destroy() { op_free(this); } + void reset(); + + int set_tenant_cluster(const common::ObString &tenant_name, const common::ObString &cluster_name); + int set_addr(const common::ObString addr); + int set_full_name(); + int set(const common::ObString &cluster_name, const common::ObString &tenant_name, + const common::ObString &vip_name, uint64_t max_connections); + bool is_valid() const { return (!tenant_name_.empty() && !cluster_name_.empty()); } + TO_STRING_KV(K_(cluster_name), K_(tenant_name), K_(vip_name), K_(full_name), + K_(max_connections)); + +public: + bool is_inited_; + common::ObString cluster_name_; + common::ObString tenant_name_; + common::ObString vip_name_; + common::ObString full_name_; + uint64_t max_connections_; + + LINK(ObVipTenantConn, vt_link_); + +private: + char tenant_name_str_[common::OB_MAX_TENANT_NAME_LENGTH]; + char cluster_name_str_[OB_PROXY_MAX_CLUSTER_NAME_LENGTH]; + char vip_name_str_[common::OB_IP_STR_BUFF]; + char full_name_str_[OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + common::OB_IP_STR_BUFF]; + DISALLOW_COPY_AND_ASSIGN(ObVipTenantConn); +}; + +class ObVipTenantConnCache +{ +public: + ObVipTenantConnCache() : vt_conn_map_(&vt_conn_map_array[0]) { } + virtual ~ObVipTenantConnCache() { } + + void destroy(); + +public: + static const int64_t HASH_BUCKET_SIZE = 64; + + struct VTCacheHashing + { + typedef const common::ObString &Key; + typedef ObVipTenantConn Value; + typedef ObDLList(ObVipTenantConn, vt_link_) ListHead; + + static uint64_t hash(Key key) { return key.hash(); } + static Key key(Value *value) { return value->full_name_; } + static bool equal(Key lhs, Key rhs) { return lhs == rhs; } + }; + typedef common::hash::ObBuildInHashMap VTHashMap; + +public: + + int set(ObVipTenantConn* vt); + int get(common::ObString& key_name, ObVipTenantConn*& vt_conn); + int erase(common::ObString& cluster_name, common::ObString& tenant_name); + int backup(); + int recover(); + int64_t get_conn_map_count() const; + void dump_conn_map(); + VTHashMap* get_conn_map() { return vt_conn_map_; }; + VTHashMap& get_conn_map_replica() { return (&vt_conn_map_array[0] != vt_conn_map_ ? vt_conn_map_array[0] : vt_conn_map_array[1]); }; + static void clear_conn_map(VTHashMap &cache_map); + static void dump_conn_map(VTHashMap &cache_map); + + //mutable obsys::CRWLock rwlock_; +private: + VTHashMap* vt_conn_map_; // Always point to the latest configuration + VTHashMap vt_conn_map_array[2]; + DISALLOW_COPY_AND_ASSIGN(ObVipTenantConnCache); +}; + +class ObUsedConn : public common::ObSharedRefCount { +public: + ObUsedConn(common::ObString& full_name) : max_used_connections_(0) { + if (full_name.length() < OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + common::OB_IP_STR_BUFF) { + MEMCPY(full_name_str_, full_name.ptr(), full_name.length()); + full_name_.assign_ptr(full_name_str_, (int32_t)full_name.length()); + } + } + virtual ~ObUsedConn() { reset(); }; + virtual void free() { destroy(); } + void destroy() { + int64_t total_len = sizeof(ObUsedConn); + op_fixed_mem_free(this, total_len); + } + void reset() { + full_name_.reset(); + max_used_connections_ = 0; + } + TO_STRING_KV(K_(full_name), K_(max_used_connections)); + LINK(ObUsedConn, used_conn_link_); + +public: + common::ObString full_name_; + volatile int64_t max_used_connections_; + +private: + char full_name_str_[OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH + common::OB_IP_STR_BUFF]; + DISALLOW_COPY_AND_ASSIGN(ObUsedConn); +}; + +class ObUsedConnCache +{ +public: + ObUsedConnCache() : used_conn_map_() { } + virtual ~ObUsedConnCache() { } + +public: + static const int64_t HASH_BUCKET_SIZE = 64; + + struct UsedConnCacheHashing + { + typedef const common::ObString &Key; + typedef ObUsedConn Value; + typedef ObDLList(ObUsedConn, used_conn_link_) ListHead; + + static uint64_t hash(Key key) { return key.hash(); } + static Key key(Value *value) { return value->full_name_; } + static bool equal(Key lhs, Key rhs) { return lhs == rhs; } + }; + typedef common::hash::ObBuildInHashMap UsedConnHashMap; + +public: + int set(ObUsedConn* used_conn); + int get(common::ObString& key_name, ObUsedConn*& used_conn); + int erase(common::ObString& key_name); + +private: + UsedConnHashMap used_conn_map_; + DISALLOW_COPY_AND_ASSIGN(ObUsedConnCache); +}; + +} // namespace omt +} // namespace obproxy +} // namespace oceanbase + +#endif // OB_VIP_TENANT_CONN_H diff --git a/src/obproxy/omt/ob_white_list_table_processor.cpp b/src/obproxy/omt/ob_white_list_table_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c434d8504b8513e6e03220cbd777347b810ad1b6 --- /dev/null +++ b/src/obproxy/omt/ob_white_list_table_processor.cpp @@ -0,0 +1,364 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include +#include "omt/ob_white_list_table_processor.h" +#include "obutils/ob_config_processor.h" +#include "utils/ob_proxy_utils.h" + +using namespace oceanbase::common; +using namespace oceanbase::obproxy::obutils; +using namespace obsys; +using namespace oceanbase::obproxy::net; + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +int ObWhiteListTableProcessor::execute(void *arg) +{ + int ret = OB_SUCCESS; + ObCloudFnParams *params = reinterpret_cast(arg); + ObString tenant_name = params->tenant_name_; + ObString cluster_name = params->cluster_name_; + ObString ip_list; + if (NULL == params->fields_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fileds is null unexpected", K(ret)); + } else if (cluster_name.empty() || cluster_name.length() > OB_PROXY_MAX_CLUSTER_NAME_LENGTH + || tenant_name.empty() || tenant_name.length() > OB_MAX_TENANT_NAME_LENGTH) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("execute failed, tenant_name or cluster_name is null", K(ret), K(cluster_name), K(tenant_name)); + } else { + for (int64_t i = 0; i < params->fields_->field_num_; i++) { + SqlField &sql_field = params->fields_->fields_.at(i); + if (sql_field.column_name_.string_ == "value") { + ip_list = sql_field.column_value_.config_string_; + } + } + + if (params->stmt_type_ == OBPROXY_T_REPLACE) { + if (OB_FAIL(get_global_white_list_table_processor().set_ip_list(cluster_name, tenant_name, ip_list))) { + LOG_WARN("set ip list failed", K(ret), K(cluster_name), K(tenant_name), K(ip_list)); + } + } else if (params->stmt_type_ == OBPROXY_T_DELETE) { + if (OB_FAIL(get_global_white_list_table_processor().delete_ip_list(cluster_name, tenant_name))) { + LOG_WARN("delete ip list failed", K(ret), K(cluster_name), K(tenant_name)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected stmt type", K(ret), K(params->stmt_type_)); + } + } + + return ret; +} + +int ObWhiteListTableProcessor::commit(bool is_success) +{ + if (is_success) { + get_global_white_list_table_processor().inc_index(); + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + get_global_white_list_table_processor().print_config(); + } + } else { + LOG_WARN("white list commit failed"); + } + + return OB_SUCCESS; +} + +ObWhiteListTableProcessor &get_global_white_list_table_processor() +{ + static ObWhiteListTableProcessor g_white_list_table_processor; + return g_white_list_table_processor; +} + +int ObWhiteListTableProcessor::init() +{ + int ret = OB_SUCCESS; + ObConfigHandler handler; + handler.execute_func_ = &ObWhiteListTableProcessor::execute; + handler.commit_func_ = &ObWhiteListTableProcessor::commit; + handler.config_type_ = OBPROXY_CONFIG_CLOUD; + if (OB_FAIL(addr_hash_map_array_[0].create(32, ObModIds::OB_HASH_BUCKET))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (OB_FAIL(addr_hash_map_array_[1].create(32, ObModIds::OB_HASH_BUCKET))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (OB_FAIL(get_global_config_processor().register_callback("white_list", handler))) { + LOG_WARN("register white_list table callback failed", K(ret)); + } + + return ret; +} + +int ObWhiteListTableProcessor::ip_to_int(char *ip_addr, uint32_t& value) +{ + int ret = OB_SUCCESS; + in_addr addr4; + if (1 == inet_aton(ip_addr, &addr4)) { + value = to_little_endian(addr4.s_addr); + } else { + ret = ob_get_sys_errno(); + } + return ret; +} + +int ObWhiteListTableProcessor::set_ip_list(ObString &cluster_name, ObString &tenant_name, ObString &ip_list) +{ + int ret = OB_SUCCESS; + + if (cluster_name.empty() || tenant_name.empty() || ip_list.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("argument is unexpected", K(ret), K(cluster_name), K(tenant_name), K(ip_list)); + } else if (OB_FAIL(backup_hash_map())) { + LOG_WARN("backup hash map failed", K(ret)); + } else { + ObSEArray addr_array; + char *start = ip_list.ptr(); + int pos = 0; + AddrStruct ip_addr; + memset(&ip_addr, 0, sizeof(AddrStruct)); + while (OB_SUCC(ret) && pos < ip_list.length()) { + if (ip_list[pos] == ',' || pos == ip_list.length() - 1 || ip_list[pos] == '/') { + char buf[64]; + int64_t len = static_cast(ip_list.ptr() + pos - start); + // to the end + if (pos == ip_list.length() - 1) { + len++; + } + memcpy(buf, start, len); + buf[len] = '\0'; + if (ip_addr.ip_ == 0) { + if (OB_FAIL(ip_to_int(buf, ip_addr.ip_))) { + LOG_WARN("ip to int failed", K(ret)); + break; + } + } else { + ip_addr.net_ = atoi(buf); + if (ip_addr.net_ <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("atoi failed", K(ret)); + break; + } + } + + if (ip_list[pos] != '/') { + if (OB_FAIL(addr_array.push_back(ip_addr))) { + LOG_WARN("ip addr push back failed", K(ret)); + } else { + memset(&ip_addr, 0, sizeof(AddrStruct)); + } + } + pos++; + start = ip_list.ptr() + pos; + } else if (ip_list[pos] >= '0' || ip_list[pos] <= '9' || ip_list[pos] == '.') { + pos++; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid char in ip_list", K(ret), K(ip_list)); + } + } + if (OB_SUCC(ret)) { + ObFixedLengthString key_string; + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paster tenant and cluser name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + DRWLock::WRLockGuard guard(white_list_lock_); + WhiteListHashMap &backup_map = addr_hash_map_array_[(index_ + 1) % 2]; + if (OB_FAIL(backup_map.set_refactored(key_string, addr_array, 1))) { + LOG_WARN("addr hash map set failed", K(ret)); + } + } + } + } + return ret; +} + +int ObWhiteListTableProcessor::delete_ip_list(ObString &cluster_name, ObString &tenant_name) +{ + int ret = OB_SUCCESS; + if (cluster_name.empty() || tenant_name.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("cluster name or tenant name empty unexpected", K(ret)); + } else if (OB_FAIL(backup_hash_map())) { + LOG_WARN("backup hash map failed", K(ret)); + } else { + LOG_DEBUG("delete ip list", K(cluster_name), K(tenant_name)); + ObFixedLengthString key_string; + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + DRWLock::WRLockGuard guard(white_list_lock_); + WhiteListHashMap &backup_map = addr_hash_map_array_[(index_ + 1) % 2]; + if (OB_FAIL(backup_map.erase_refactored(key_string))) { + LOG_WARN("addr hash map erase failed", K(ret)); + } + } + } + + return ret; +} + +bool ObWhiteListTableProcessor::can_ip_pass(ObString &cluster_name, ObString &tenant_name, ObString &user_name, char* ip) +{ + uint32_t ip_value = 0; + int ret = OB_SUCCESS; + bool can_pass = false; + if (OB_FAIL(ip_to_int(ip, ip_value))) { + LOG_WARN("ip to in failed", K(ret)); + } else { + can_pass = can_ip_pass(cluster_name, tenant_name, user_name, ip_value, false); + } + return can_pass; +} + +bool ObWhiteListTableProcessor::can_ip_pass(ObString &cluster_name, ObString &tenant_name, + ObString &user_name, uint32_t src_ip, bool is_big_endian) +{ + int ret = OB_SUCCESS; + bool can_pass = false; + uint32_t ip = is_big_endian ? to_little_endian(src_ip) : src_ip; + LOG_DEBUG("check can ip pass", K(cluster_name), K(tenant_name), K(ip)); + if (!cluster_name.empty() && !tenant_name.empty()) { + ObFixedLengthString key_string; + if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster name failed", K(ret), K(tenant_name), K(cluster_name)); + } else { + ObSEArray ip_array; + DRWLock::RDLockGuard guard(white_list_lock_); + WhiteListHashMap ¤t_map = addr_hash_map_array_[index_]; + if (OB_SUCC(current_map.get_refactored(key_string, ip_array))) { + for (int64_t i = 0; !can_pass && i < ip_array.count(); i++) { + AddrStruct addr = ip_array.at(i); + if (0 == addr.ip_) { + can_pass = true; + } else if (0 == addr.net_) { + can_pass = (addr.ip_ == ip); + } else { + uint32_t mask = 0xffffffff; + can_pass = (addr.ip_ == (ip & (mask << (32 - addr.net_)))); + } + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + ObIpEndpoint client_info; + ObIpEndpoint white_list_info; + client_info.sin_.sin_addr.s_addr = __bswap_32(ip); + client_info.sin_.sin_family = AF_INET; + white_list_info.sin_.sin_addr.s_addr = __bswap_32(addr.ip_); + white_list_info.sin_.sin_family = AF_INET; + LOG_DEBUG("ip check can pass", K(client_info), K(white_list_info), "mask", addr.net_, K(can_pass)); + } + } + } else if (OB_HASH_NOT_EXIST == ret) { + can_pass = true; + } else { + LOG_WARN("unexpected error", K(ret)); + } + } + } else { + LOG_WARN("cluster_name or tenant_name is empty", K(cluster_name), K(tenant_name), K(ip)); + } + + if (!can_pass) { + struct sockaddr_in address; + address.sin_addr.s_addr = __bswap_32(ip); + LOG_WARN("can not pass white_list", K(cluster_name), K(tenant_name), K(user_name), "client_ip", inet_ntoa(address.sin_addr)); + } + + return can_pass; +} + +int64_t AddrStruct::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + in_addr addr; + addr.s_addr = static_cast(ip_); + J_OBJ_START(); + J_KV("ip", inet_ntoa(addr), K_(net)); + J_OBJ_END(); + return pos; +} + +int ObWhiteListTableProcessor::backup_hash_map() +{ + int ret = OB_SUCCESS; + int64_t backup_index = (index_ + 1) % 2; + WhiteListHashMap &backup_map = addr_hash_map_array_[backup_index]; + WhiteListHashMap ¤t_map = addr_hash_map_array_[index_]; + if (OB_FAIL(backup_map.reuse())) { + LOG_WARN("backup map failed", K(ret)); + } else { + WhiteListHashMap::iterator iter = current_map.begin(); + for(; OB_SUCC(ret) && iter != current_map.end(); ++iter) { + if (OB_FAIL(backup_map.set_refactored(iter->first, iter->second))) { + LOG_WARN("backup map set refactored failed", K(ret)); + } + } + } + return ret; +} + +void ObWhiteListTableProcessor::inc_index() +{ + DRWLock::WRLockGuard guard(white_list_lock_); + index_ = (index_ + 1) % 2; +} + +void ObWhiteListTableProcessor::print_config() +{ + DRWLock::RDLockGuard guard(white_list_lock_); + WhiteListHashMap ¤t_map = addr_hash_map_array_[index_]; + WhiteListHashMap::iterator iter = current_map.begin(); + for(; iter != current_map.end(); ++iter) { + LOG_DEBUG("white list map info", K(iter->first)); + for (int64_t i = 0; i < iter->second.count(); i++) { + AddrStruct addr_info = iter->second.at(i); + struct sockaddr_in address; + address.sin_addr.s_addr = __bswap_32(addr_info.ip_); + LOG_DEBUG("ip/net info", "ip", inet_ntoa(address.sin_addr), "mask", addr_info.net_); + } + } +} + +uint32_t ObWhiteListTableProcessor::to_little_endian(uint32_t value) { + // Regardless of whether the current host is big endian or little endian, + // the storage format of uint32_t is changed to the same as the little endian type + // Because the network sequence is a big-endian type, the conversion is performed directly + return __bswap_32 (value); +} + +} // end of omt +} // end of obproxy +} // end of oceanbase diff --git a/src/obproxy/omt/ob_white_list_table_processor.h b/src/obproxy/omt/ob_white_list_table_processor.h new file mode 100644 index 0000000000000000000000000000000000000000..d1260e1a95b231c652911394e383a672f07f32ed --- /dev/null +++ b/src/obproxy/omt/ob_white_list_table_processor.h @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_WHITE_LIST_TABLE_PROCESSOR_H_ +#define OB_WHITE_LIST_TABLE_PROCESSOR_H_ + +#include "lib/hash/ob_hashmap.h" +#include "lib/container/ob_se_array.h" +#include "lib/lock/ob_drw_lock.h" +#include "utils/ob_proxy_lib.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace omt +{ + +struct AddrStruct +{ + uint32_t ip_; + uint32_t net_; + + int64_t to_string(char *buf, const int64_t buf_len) const; +}; + +class ObWhiteListTableProcessor +{ +public: + ObWhiteListTableProcessor() : index_(0), white_list_lock_(obsys::WRITE_PRIORITY) {} + ~ObWhiteListTableProcessor() {} + + int init(); + bool can_ip_pass(common::ObString &cluster_name, common::ObString &tenant_name, + common::ObString &user_name, uint32_t src_ip, bool is_big_endian); + bool can_ip_pass(common::ObString &cluster_name, common::ObString &tenant_name, common::ObString &user_name, char *ip); + int set_ip_list(common::ObString &cluster_name, common::ObString &tenant_name, common::ObString &ip_list); + int delete_ip_list(common::ObString &cluster_name, common::ObString &tenant_name); + void inc_index(); + void print_config(); +private: + int32_t ip_to_int(char *addr, uint32_t &value); + int backup_hash_map(); + uint32_t to_little_endian(uint32_t value); +private: + static int execute(void *arg); + static int commit(bool is_success); + +private: + typedef common::hash::ObHashMap, common::ObSEArray > WhiteListHashMap; + WhiteListHashMap addr_hash_map_array_[2]; + int64_t index_; +private: + common::DRWLock white_list_lock_; +}; + +ObWhiteListTableProcessor &get_global_white_list_table_processor(); + +} // end of omt +} // end of obproxy +} // end of oceanbase +#endif diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parse_result.h b/src/obproxy/opsql/expr_parser/ob_expr_parse_result.h index c634cb97cf1a63ca01cc2fe3fde66896d2d79740..a9a5cb8e4cdac29065bed92858bf54e590ae7fdc 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parse_result.h +++ b/src/obproxy/opsql/expr_parser/ob_expr_parse_result.h @@ -43,7 +43,7 @@ typedef enum ObExprParseMode { - INVLIAD_PARSE_MODE = 0, + INVALID_PARSE_MODE = 0, SELECT_STMT_PARSE_MODE, // select/delete stmt INSERT_STMT_PARSE_MODE, // insert/replace/update stmt } ObExprParseMode; @@ -131,6 +131,20 @@ typedef struct _ObProxyRelationExpr ObProxyPartKeyLevel level_; } ObProxyRelationExpr; +/* + * according to observer rs colleague, we use the triple to describe all type of the accuracy + * which is necessary to describe the accuracy of each part key type + * table: oceanbase.__all_virtual_proxy_partition_info + * -> column spare5: VARCHAR, format: "int32_t,int16_t,int16_t" + * which means "length,precision/length_semantics,scale" + * init -1 to each elements for invalid status. + */ +typedef struct _ObProxyPartKeyAccuracy { + int32_t length_; + int16_t precision_; // the same as length_semantics + int16_t scale_; +} ObProxyPartKeyAccuracy; + typedef struct _ObProxyPartKey { ObProxyParseString name_; @@ -145,6 +159,7 @@ typedef struct _ObProxyPartKey ObProxyExprType func_type_; ObProxyParamNode *params_[OBPROXY_MAX_PARAM_NUM]; // used to store generated func param int64_t idx_in_rowid_; + ObProxyPartKeyAccuracy accuracy_; } ObProxyPartKey; typedef struct _ObProxyPartKeyInfo diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser.h b/src/obproxy/opsql/expr_parser/ob_expr_parser.h index 3a374aaf8f20a9b5158b55c62158848f44bed72d..aa6ba7882bce8c4b06a012caa59afa6d79c912a8 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser.h +++ b/src/obproxy/opsql/expr_parser/ob_expr_parser.h @@ -87,7 +87,7 @@ inline int ObExprParser::init_result(ObExprParseResult &parse_result, const char parse_result.all_relation_info_.right_value_num_ = 0; if (0 == parse_result.target_mask_ - || INVLIAD_PARSE_MODE == parse_result.parse_mode_) { + || INVALID_PARSE_MODE == parse_result.parse_mode_) { ret = common::OB_INVALID_ARGUMENT; PROXY_LOG(DEBUG, "failed to initialized parser, maybe parse sql for shard user", K(parse_result.target_mask_), diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser.l b/src/obproxy/opsql/expr_parser/ob_expr_parser.l index 1d8151e190916ac42958f6c8748b4866701f3882..eccb3bcd030c4eabd1875d7dccc0193072c48f4f 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser.l +++ b/src/obproxy/opsql/expr_parser/ob_expr_parser.l @@ -57,8 +57,8 @@ multi_byte_left_parenthesis [\uff08] multi_byte_right_parenthesis [\uff09] space [ \t\n\r\f] identifer ([A-Za-z0-9$_]*) -int_num [0-9]+ -number ([0-9]+E[-+]?[0-9]+)|([0-9]+"."[0-9]*E[-+]?[0-9]+)|("."[0-9]+E[-+]?[0-9]+)|([0-9]+"."[0-9]*)|("."[0-9]+) +int_num [\-\+]?[0-9]+ +number ([\-]?[0-9]+E[-+]?[0-9]+)|([\-]?[0-9]+"."[0-9]*E[-+]?[0-9]+)|([\-]?"."[0-9]+E[-+]?[0-9]+)|([\-]?[0-9]+"."[0-9]*)|([\-]?"."[0-9]+) non_newline [^\n\r] sql_comment ("--"{space}+{non_newline}*)|(#{non_newline}*) @@ -95,6 +95,7 @@ rightbracket \) WHERE { return WHERE; } AS { return AS; } VALUES { return VALUES; } +VALUE { return VALUES; } SET { return SET; } FOR { return END_WHERE; } LIMIT { return END_WHERE; } @@ -119,7 +120,6 @@ IN { return IN; } "!="|"<>" { return COMP_NE; } "?" { return PLACE_HOLDER; } ":"{int_num} { store_pos_place_holder(yytext + 1, yyscanner); return POS_PLACE_HOLDER; } -":"[+-]{int_num} { store_pos_place_holder(yytext + 1, yyscanner); return POS_PLACE_HOLDER; } {int_num} { RETURN_INT_VAL(); } {number} { RETURN_NUMBER_VAL(); } diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser.output b/src/obproxy/opsql/expr_parser/ob_expr_parser.output deleted file mode 100644 index 9e3cc9b6a34123765873267478ee3db0d9e25eea..0000000000000000000000000000000000000000 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser.output +++ /dev/null @@ -1,1932 +0,0 @@ -Terminals unused in grammar - - ERROR - IGNORED_WORD - - -Grammar - - 0 $accept: start $end - - 1 start: DUMMY_SELECT_CLAUSE select_root - 2 | DUMMY_INSERT_CLAUSE insert_root - - 3 select_root: WHERE cond_expr end_flag - 4 | join_expr_list end_flag - 5 | join_expr_list WHERE cond_expr end_flag - 6 | error - - 7 end_flag: END_WHERE - 8 | ')' - 9 | ';' - 10 | END_P - - 11 join_expr_list: join_on_expr - 12 | join_expr_list join_on_expr - - 13 join_on_expr: join_expr ON cond_expr - - 14 join_expr: JOIN NAME_OB - 15 | JOIN NAME_OB '.' NAME_OB - 16 | JOIN NAME_OB NAME_OB - 17 | JOIN NAME_OB AS NAME_OB - 18 | JOIN NAME_OB '.' NAME_OB NAME_OB - 19 | JOIN NAME_OB '.' NAME_OB AS NAME_OB - - 20 cond_expr: bool_pri - 21 | cond_expr AND_OP bool_pri - 22 | '(' cond_expr AND_OP bool_pri ')' - 23 | cond_expr OR_OP bool_pri - 24 | '(' cond_expr OR_OP bool_pri ')' - - 25 bool_pri: expr comp expr - 26 | '(' expr comp expr ')' - 27 | expr IN '(' in_expr_list ')' - 28 | expr BETWEEN expr AND_OP expr - 29 | ROWID COMP_EQ STR_VAL - - 30 comp: COMP_EQ - 31 | COMP_NSEQ - 32 | COMP_GE - 33 | COMP_GT - 34 | COMP_LE - 35 | COMP_LT - 36 | COMP_NE - - 37 in_expr_list: expr - 38 | in_expr_list ',' expr - - 39 expr: token_list - - 40 token_list: token - 41 | token_list token - - 42 func_param_list: /* empty */ - 43 | token_list ',' token_list - 44 | func_param_list ',' token_list - - 45 token: NAME_OB - 46 | NAME_OB '.' NAME_OB - 47 | NAME_OB '.' NAME_OB '.' NAME_OB - 48 | NAME_OB '(' token_list ')' - 49 | NAME_OB '(' func_param_list ')' - 50 | INT_VAL - 51 | STR_VAL - 52 | operator - 53 | PLACE_HOLDER - 54 | POS_PLACE_HOLDER - - 55 operator: '+' - 56 | '-' - 57 | '*' - 58 | '/' - 59 | '%' - 60 | '&' - 61 | '!' - - 62 insert_root: opt_column_list VALUES values_expr_lists end_flag - 63 | SET set_expr opt_where_clause end_flag - 64 | ON cond_expr end_flag - 65 | select_root - - 66 values_expr_lists: '(' values_expr_list ')' - 67 | values_expr_lists ',' '(' values_expr_list ')' - - 68 opt_column_list: /* empty */ - 69 | '(' column_list ')' - - 70 column_list: opt_column - 71 | column_list ',' opt_column - - 72 opt_column: NAME_OB - - 73 values_expr_list: expr - 74 | values_expr_list ',' expr - - 75 set_expr: bool_pri - 76 | set_expr ',' bool_pri - - 77 opt_where_clause: /* empty */ - 78 | WHERE cond_expr - - -Terminals, with rules where they appear - -$end (0) 0 -'!' (33) 61 -'%' (37) 59 -'&' (38) 60 -'(' (40) 22 24 26 27 48 49 66 67 69 -')' (41) 8 22 24 26 27 48 49 66 67 69 -'*' (42) 57 -'+' (43) 55 -',' (44) 38 43 44 67 71 74 76 -'-' (45) 56 -'.' (46) 15 18 19 46 47 -'/' (47) 58 -';' (59) 9 -error (256) 6 -DUMMY_SELECT_CLAUSE (258) 1 -DUMMY_INSERT_CLAUSE (259) 2 -WHERE (260) 3 5 78 -AS (261) 17 19 -VALUES (262) 62 -SET (263) 63 -END_WHERE (264) 7 -JOIN (265) 14 15 16 17 18 19 -AND_OP (266) 21 22 28 -OR_OP (267) 23 24 -IN (268) 27 -ON (269) 13 64 -BETWEEN (270) 28 -ROWID (271) 29 -COMP_EQ (272) 29 30 -COMP_NSEQ (273) 31 -COMP_GE (274) 32 -COMP_GT (275) 33 -COMP_LE (276) 34 -COMP_LT (277) 35 -COMP_NE (278) 36 -PLACE_HOLDER (279) 53 -END_P (280) 10 -ERROR (281) -IGNORED_WORD (282) -NAME_OB (283) 14 15 16 17 18 19 45 46 47 48 49 72 -STR_VAL (284) 29 51 -INT_VAL (285) 50 -POS_PLACE_HOLDER (286) 54 - - -Nonterminals, with rules where they appear - -$accept (44) - on left: 0 -start (45) - on left: 1 2, on right: 0 -select_root (46) - on left: 3 4 5 6, on right: 1 65 -end_flag (47) - on left: 7 8 9 10, on right: 3 4 5 62 63 64 -join_expr_list (48) - on left: 11 12, on right: 4 5 12 -join_on_expr (49) - on left: 13, on right: 11 12 -join_expr (50) - on left: 14 15 16 17 18 19, on right: 13 -cond_expr (51) - on left: 20 21 22 23 24, on right: 3 5 13 21 22 23 24 64 78 -bool_pri (52) - on left: 25 26 27 28 29, on right: 20 21 22 23 24 75 76 -comp (53) - on left: 30 31 32 33 34 35 36, on right: 25 26 -in_expr_list (54) - on left: 37 38, on right: 27 38 -expr (55) - on left: 39, on right: 25 26 27 28 37 38 73 74 -token_list (56) - on left: 40 41, on right: 39 41 43 44 48 -func_param_list (57) - on left: 42 43 44, on right: 44 49 -token (58) - on left: 45 46 47 48 49 50 51 52 53 54, on right: 40 41 -operator (59) - on left: 55 56 57 58 59 60 61, on right: 52 -insert_root (60) - on left: 62 63 64 65, on right: 2 -values_expr_lists (61) - on left: 66 67, on right: 62 67 -opt_column_list (62) - on left: 68 69, on right: 62 -column_list (63) - on left: 70 71, on right: 69 71 -opt_column (64) - on left: 72, on right: 70 71 -values_expr_list (65) - on left: 73 74, on right: 66 67 74 -set_expr (66) - on left: 75 76, on right: 63 76 -opt_where_clause (67) - on left: 77 78, on right: 63 - - -state 0 - - 0 $accept: . start $end - - DUMMY_SELECT_CLAUSE shift, and go to state 1 - DUMMY_INSERT_CLAUSE shift, and go to state 2 - - start go to state 3 - - -state 1 - - 1 start: DUMMY_SELECT_CLAUSE . select_root - - error shift, and go to state 4 - WHERE shift, and go to state 5 - JOIN shift, and go to state 6 - - select_root go to state 7 - join_expr_list go to state 8 - join_on_expr go to state 9 - join_expr go to state 10 - - -state 2 - - 2 start: DUMMY_INSERT_CLAUSE . insert_root - - error shift, and go to state 4 - WHERE shift, and go to state 5 - SET shift, and go to state 11 - JOIN shift, and go to state 6 - ON shift, and go to state 12 - '(' shift, and go to state 13 - - VALUES reduce using rule 68 (opt_column_list) - - select_root go to state 14 - join_expr_list go to state 8 - join_on_expr go to state 9 - join_expr go to state 10 - insert_root go to state 15 - opt_column_list go to state 16 - - -state 3 - - 0 $accept: start . $end - - $end shift, and go to state 17 - - -state 4 - - 6 select_root: error . - - $default reduce using rule 6 (select_root) - - -state 5 - - 3 select_root: WHERE . cond_expr end_flag - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 32 - bool_pri go to state 33 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 6 - - 14 join_expr: JOIN . NAME_OB - 15 | JOIN . NAME_OB '.' NAME_OB - 16 | JOIN . NAME_OB NAME_OB - 17 | JOIN . NAME_OB AS NAME_OB - 18 | JOIN . NAME_OB '.' NAME_OB NAME_OB - 19 | JOIN . NAME_OB '.' NAME_OB AS NAME_OB - - NAME_OB shift, and go to state 38 - - -state 7 - - 1 start: DUMMY_SELECT_CLAUSE select_root . - - $default reduce using rule 1 (start) - - -state 8 - - 4 select_root: join_expr_list . end_flag - 5 | join_expr_list . WHERE cond_expr end_flag - 12 join_expr_list: join_expr_list . join_on_expr - - WHERE shift, and go to state 39 - END_WHERE shift, and go to state 40 - JOIN shift, and go to state 6 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - - end_flag go to state 44 - join_on_expr go to state 45 - join_expr go to state 10 - - -state 9 - - 11 join_expr_list: join_on_expr . - - $default reduce using rule 11 (join_expr_list) - - -state 10 - - 13 join_on_expr: join_expr . ON cond_expr - - ON shift, and go to state 46 - - -state 11 - - 63 insert_root: SET . set_expr opt_where_clause end_flag - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 48 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - set_expr go to state 49 - - -state 12 - - 64 insert_root: ON . cond_expr end_flag - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 50 - bool_pri go to state 33 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 13 - - 69 opt_column_list: '(' . column_list ')' - - NAME_OB shift, and go to state 51 - - column_list go to state 52 - opt_column go to state 53 - - -state 14 - - 65 insert_root: select_root . - - $default reduce using rule 65 (insert_root) - - -state 15 - - 2 start: DUMMY_INSERT_CLAUSE insert_root . - - $default reduce using rule 2 (start) - - -state 16 - - 62 insert_root: opt_column_list . VALUES values_expr_lists end_flag - - VALUES shift, and go to state 54 - - -state 17 - - 0 $accept: start $end . - - $default accept - - -state 18 - - 29 bool_pri: ROWID . COMP_EQ STR_VAL - - COMP_EQ shift, and go to state 55 - - -state 19 - - 53 token: PLACE_HOLDER . - - $default reduce using rule 53 (token) - - -state 20 - - 45 token: NAME_OB . - 46 | NAME_OB . '.' NAME_OB - 47 | NAME_OB . '.' NAME_OB '.' NAME_OB - 48 | NAME_OB . '(' token_list ')' - 49 | NAME_OB . '(' func_param_list ')' - - '.' shift, and go to state 56 - '(' shift, and go to state 57 - - $default reduce using rule 45 (token) - - -state 21 - - 51 token: STR_VAL . - - $default reduce using rule 51 (token) - - -state 22 - - 50 token: INT_VAL . - - $default reduce using rule 50 (token) - - -state 23 - - 54 token: POS_PLACE_HOLDER . - - $default reduce using rule 54 (token) - - -state 24 - - 22 cond_expr: '(' . cond_expr AND_OP bool_pri ')' - 24 | '(' . cond_expr OR_OP bool_pri ')' - 26 bool_pri: '(' . expr comp expr ')' - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 58 - bool_pri go to state 33 - expr go to state 59 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 25 - - 55 operator: '+' . - - $default reduce using rule 55 (operator) - - -state 26 - - 56 operator: '-' . - - $default reduce using rule 56 (operator) - - -state 27 - - 57 operator: '*' . - - $default reduce using rule 57 (operator) - - -state 28 - - 58 operator: '/' . - - $default reduce using rule 58 (operator) - - -state 29 - - 59 operator: '%' . - - $default reduce using rule 59 (operator) - - -state 30 - - 60 operator: '&' . - - $default reduce using rule 60 (operator) - - -state 31 - - 61 operator: '!' . - - $default reduce using rule 61 (operator) - - -state 32 - - 3 select_root: WHERE cond_expr . end_flag - 21 cond_expr: cond_expr . AND_OP bool_pri - 23 | cond_expr . OR_OP bool_pri - - END_WHERE shift, and go to state 40 - AND_OP shift, and go to state 60 - OR_OP shift, and go to state 61 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - - end_flag go to state 62 - - -state 33 - - 20 cond_expr: bool_pri . - - $default reduce using rule 20 (cond_expr) - - -state 34 - - 25 bool_pri: expr . comp expr - 27 | expr . IN '(' in_expr_list ')' - 28 | expr . BETWEEN expr AND_OP expr - - IN shift, and go to state 63 - BETWEEN shift, and go to state 64 - COMP_EQ shift, and go to state 65 - COMP_NSEQ shift, and go to state 66 - COMP_GE shift, and go to state 67 - COMP_GT shift, and go to state 68 - COMP_LE shift, and go to state 69 - COMP_LT shift, and go to state 70 - COMP_NE shift, and go to state 71 - - comp go to state 72 - - -state 35 - - 39 expr: token_list . - 41 token_list: token_list . token - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - $default reduce using rule 39 (expr) - - token go to state 73 - operator go to state 37 - - -state 36 - - 40 token_list: token . - - $default reduce using rule 40 (token_list) - - -state 37 - - 52 token: operator . - - $default reduce using rule 52 (token) - - -state 38 - - 14 join_expr: JOIN NAME_OB . - 15 | JOIN NAME_OB . '.' NAME_OB - 16 | JOIN NAME_OB . NAME_OB - 17 | JOIN NAME_OB . AS NAME_OB - 18 | JOIN NAME_OB . '.' NAME_OB NAME_OB - 19 | JOIN NAME_OB . '.' NAME_OB AS NAME_OB - - AS shift, and go to state 74 - NAME_OB shift, and go to state 75 - '.' shift, and go to state 76 - - $default reduce using rule 14 (join_expr) - - -state 39 - - 5 select_root: join_expr_list WHERE . cond_expr end_flag - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 77 - bool_pri go to state 33 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 40 - - 7 end_flag: END_WHERE . - - $default reduce using rule 7 (end_flag) - - -state 41 - - 10 end_flag: END_P . - - $default reduce using rule 10 (end_flag) - - -state 42 - - 8 end_flag: ')' . - - $default reduce using rule 8 (end_flag) - - -state 43 - - 9 end_flag: ';' . - - $default reduce using rule 9 (end_flag) - - -state 44 - - 4 select_root: join_expr_list end_flag . - - $default reduce using rule 4 (select_root) - - -state 45 - - 12 join_expr_list: join_expr_list join_on_expr . - - $default reduce using rule 12 (join_expr_list) - - -state 46 - - 13 join_on_expr: join_expr ON . cond_expr - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 78 - bool_pri go to state 33 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 47 - - 26 bool_pri: '(' . expr comp expr ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 79 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 48 - - 75 set_expr: bool_pri . - - $default reduce using rule 75 (set_expr) - - -state 49 - - 63 insert_root: SET set_expr . opt_where_clause end_flag - 76 set_expr: set_expr . ',' bool_pri - - WHERE shift, and go to state 80 - ',' shift, and go to state 81 - - $default reduce using rule 77 (opt_where_clause) - - opt_where_clause go to state 82 - - -state 50 - - 21 cond_expr: cond_expr . AND_OP bool_pri - 23 | cond_expr . OR_OP bool_pri - 64 insert_root: ON cond_expr . end_flag - - END_WHERE shift, and go to state 40 - AND_OP shift, and go to state 60 - OR_OP shift, and go to state 61 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - - end_flag go to state 83 - - -state 51 - - 72 opt_column: NAME_OB . - - $default reduce using rule 72 (opt_column) - - -state 52 - - 69 opt_column_list: '(' column_list . ')' - 71 column_list: column_list . ',' opt_column - - ')' shift, and go to state 84 - ',' shift, and go to state 85 - - -state 53 - - 70 column_list: opt_column . - - $default reduce using rule 70 (column_list) - - -state 54 - - 62 insert_root: opt_column_list VALUES . values_expr_lists end_flag - - '(' shift, and go to state 86 - - values_expr_lists go to state 87 - - -state 55 - - 29 bool_pri: ROWID COMP_EQ . STR_VAL - - STR_VAL shift, and go to state 88 - - -state 56 - - 46 token: NAME_OB '.' . NAME_OB - 47 | NAME_OB '.' . NAME_OB '.' NAME_OB - - NAME_OB shift, and go to state 89 - - -state 57 - - 48 token: NAME_OB '(' . token_list ')' - 49 | NAME_OB '(' . func_param_list ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - $default reduce using rule 42 (func_param_list) - - token_list go to state 90 - func_param_list go to state 91 - token go to state 36 - operator go to state 37 - - -state 58 - - 21 cond_expr: cond_expr . AND_OP bool_pri - 22 | '(' cond_expr . AND_OP bool_pri ')' - 23 | cond_expr . OR_OP bool_pri - 24 | '(' cond_expr . OR_OP bool_pri ')' - - AND_OP shift, and go to state 92 - OR_OP shift, and go to state 93 - - -state 59 - - 25 bool_pri: expr . comp expr - 26 | '(' expr . comp expr ')' - 27 | expr . IN '(' in_expr_list ')' - 28 | expr . BETWEEN expr AND_OP expr - - IN shift, and go to state 63 - BETWEEN shift, and go to state 64 - COMP_EQ shift, and go to state 65 - COMP_NSEQ shift, and go to state 66 - COMP_GE shift, and go to state 67 - COMP_GT shift, and go to state 68 - COMP_LE shift, and go to state 69 - COMP_LT shift, and go to state 70 - COMP_NE shift, and go to state 71 - - comp go to state 94 - - -state 60 - - 21 cond_expr: cond_expr AND_OP . bool_pri - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 95 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 61 - - 23 cond_expr: cond_expr OR_OP . bool_pri - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 96 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 62 - - 3 select_root: WHERE cond_expr end_flag . - - $default reduce using rule 3 (select_root) - - -state 63 - - 27 bool_pri: expr IN . '(' in_expr_list ')' - - '(' shift, and go to state 97 - - -state 64 - - 28 bool_pri: expr BETWEEN . expr AND_OP expr - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 98 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 65 - - 30 comp: COMP_EQ . - - $default reduce using rule 30 (comp) - - -state 66 - - 31 comp: COMP_NSEQ . - - $default reduce using rule 31 (comp) - - -state 67 - - 32 comp: COMP_GE . - - $default reduce using rule 32 (comp) - - -state 68 - - 33 comp: COMP_GT . - - $default reduce using rule 33 (comp) - - -state 69 - - 34 comp: COMP_LE . - - $default reduce using rule 34 (comp) - - -state 70 - - 35 comp: COMP_LT . - - $default reduce using rule 35 (comp) - - -state 71 - - 36 comp: COMP_NE . - - $default reduce using rule 36 (comp) - - -state 72 - - 25 bool_pri: expr comp . expr - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 99 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 73 - - 41 token_list: token_list token . - - $default reduce using rule 41 (token_list) - - -state 74 - - 17 join_expr: JOIN NAME_OB AS . NAME_OB - - NAME_OB shift, and go to state 100 - - -state 75 - - 16 join_expr: JOIN NAME_OB NAME_OB . - - $default reduce using rule 16 (join_expr) - - -state 76 - - 15 join_expr: JOIN NAME_OB '.' . NAME_OB - 18 | JOIN NAME_OB '.' . NAME_OB NAME_OB - 19 | JOIN NAME_OB '.' . NAME_OB AS NAME_OB - - NAME_OB shift, and go to state 101 - - -state 77 - - 5 select_root: join_expr_list WHERE cond_expr . end_flag - 21 cond_expr: cond_expr . AND_OP bool_pri - 23 | cond_expr . OR_OP bool_pri - - END_WHERE shift, and go to state 40 - AND_OP shift, and go to state 60 - OR_OP shift, and go to state 61 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - - end_flag go to state 102 - - -state 78 - - 13 join_on_expr: join_expr ON cond_expr . - 21 cond_expr: cond_expr . AND_OP bool_pri - 23 | cond_expr . OR_OP bool_pri - - AND_OP shift, and go to state 60 - OR_OP shift, and go to state 61 - - $default reduce using rule 13 (join_on_expr) - - -state 79 - - 26 bool_pri: '(' expr . comp expr ')' - - COMP_EQ shift, and go to state 65 - COMP_NSEQ shift, and go to state 66 - COMP_GE shift, and go to state 67 - COMP_GT shift, and go to state 68 - COMP_LE shift, and go to state 69 - COMP_LT shift, and go to state 70 - COMP_NE shift, and go to state 71 - - comp go to state 103 - - -state 80 - - 78 opt_where_clause: WHERE . cond_expr - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 24 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - cond_expr go to state 104 - bool_pri go to state 33 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 81 - - 76 set_expr: set_expr ',' . bool_pri - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 105 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 82 - - 63 insert_root: SET set_expr opt_where_clause . end_flag - - END_WHERE shift, and go to state 40 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - - end_flag go to state 106 - - -state 83 - - 64 insert_root: ON cond_expr end_flag . - - $default reduce using rule 64 (insert_root) - - -state 84 - - 69 opt_column_list: '(' column_list ')' . - - $default reduce using rule 69 (opt_column_list) - - -state 85 - - 71 column_list: column_list ',' . opt_column - - NAME_OB shift, and go to state 51 - - opt_column go to state 107 - - -state 86 - - 66 values_expr_lists: '(' . values_expr_list ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 108 - token_list go to state 35 - token go to state 36 - operator go to state 37 - values_expr_list go to state 109 - - -state 87 - - 62 insert_root: opt_column_list VALUES values_expr_lists . end_flag - 67 values_expr_lists: values_expr_lists . ',' '(' values_expr_list ')' - - END_WHERE shift, and go to state 40 - END_P shift, and go to state 41 - ')' shift, and go to state 42 - ';' shift, and go to state 43 - ',' shift, and go to state 110 - - end_flag go to state 111 - - -state 88 - - 29 bool_pri: ROWID COMP_EQ STR_VAL . - - $default reduce using rule 29 (bool_pri) - - -state 89 - - 46 token: NAME_OB '.' NAME_OB . - 47 | NAME_OB '.' NAME_OB . '.' NAME_OB - - '.' shift, and go to state 112 - - $default reduce using rule 46 (token) - - -state 90 - - 41 token_list: token_list . token - 43 func_param_list: token_list . ',' token_list - 48 token: NAME_OB '(' token_list . ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - ')' shift, and go to state 113 - ',' shift, and go to state 114 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - token go to state 73 - operator go to state 37 - - -state 91 - - 44 func_param_list: func_param_list . ',' token_list - 49 token: NAME_OB '(' func_param_list . ')' - - ')' shift, and go to state 115 - ',' shift, and go to state 116 - - -state 92 - - 21 cond_expr: cond_expr AND_OP . bool_pri - 22 | '(' cond_expr AND_OP . bool_pri ')' - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 117 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 93 - - 23 cond_expr: cond_expr OR_OP . bool_pri - 24 | '(' cond_expr OR_OP . bool_pri ')' - - ROWID shift, and go to state 18 - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '(' shift, and go to state 47 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - bool_pri go to state 118 - expr go to state 34 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 94 - - 25 bool_pri: expr comp . expr - 26 | '(' expr comp . expr ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 119 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 95 - - 21 cond_expr: cond_expr AND_OP bool_pri . - - $default reduce using rule 21 (cond_expr) - - -state 96 - - 23 cond_expr: cond_expr OR_OP bool_pri . - - $default reduce using rule 23 (cond_expr) - - -state 97 - - 27 bool_pri: expr IN '(' . in_expr_list ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - in_expr_list go to state 120 - expr go to state 121 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 98 - - 28 bool_pri: expr BETWEEN expr . AND_OP expr - - AND_OP shift, and go to state 122 - - -state 99 - - 25 bool_pri: expr comp expr . - - $default reduce using rule 25 (bool_pri) - - -state 100 - - 17 join_expr: JOIN NAME_OB AS NAME_OB . - - $default reduce using rule 17 (join_expr) - - -state 101 - - 15 join_expr: JOIN NAME_OB '.' NAME_OB . - 18 | JOIN NAME_OB '.' NAME_OB . NAME_OB - 19 | JOIN NAME_OB '.' NAME_OB . AS NAME_OB - - AS shift, and go to state 123 - NAME_OB shift, and go to state 124 - - $default reduce using rule 15 (join_expr) - - -state 102 - - 5 select_root: join_expr_list WHERE cond_expr end_flag . - - $default reduce using rule 5 (select_root) - - -state 103 - - 26 bool_pri: '(' expr comp . expr ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 125 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 104 - - 21 cond_expr: cond_expr . AND_OP bool_pri - 23 | cond_expr . OR_OP bool_pri - 78 opt_where_clause: WHERE cond_expr . - - AND_OP shift, and go to state 60 - OR_OP shift, and go to state 61 - - $default reduce using rule 78 (opt_where_clause) - - -state 105 - - 76 set_expr: set_expr ',' bool_pri . - - $default reduce using rule 76 (set_expr) - - -state 106 - - 63 insert_root: SET set_expr opt_where_clause end_flag . - - $default reduce using rule 63 (insert_root) - - -state 107 - - 71 column_list: column_list ',' opt_column . - - $default reduce using rule 71 (column_list) - - -state 108 - - 73 values_expr_list: expr . - - $default reduce using rule 73 (values_expr_list) - - -state 109 - - 66 values_expr_lists: '(' values_expr_list . ')' - 74 values_expr_list: values_expr_list . ',' expr - - ')' shift, and go to state 126 - ',' shift, and go to state 127 - - -state 110 - - 67 values_expr_lists: values_expr_lists ',' . '(' values_expr_list ')' - - '(' shift, and go to state 128 - - -state 111 - - 62 insert_root: opt_column_list VALUES values_expr_lists end_flag . - - $default reduce using rule 62 (insert_root) - - -state 112 - - 47 token: NAME_OB '.' NAME_OB '.' . NAME_OB - - NAME_OB shift, and go to state 129 - - -state 113 - - 48 token: NAME_OB '(' token_list ')' . - - $default reduce using rule 48 (token) - - -state 114 - - 43 func_param_list: token_list ',' . token_list - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - token_list go to state 130 - token go to state 36 - operator go to state 37 - - -state 115 - - 49 token: NAME_OB '(' func_param_list ')' . - - $default reduce using rule 49 (token) - - -state 116 - - 44 func_param_list: func_param_list ',' . token_list - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - token_list go to state 131 - token go to state 36 - operator go to state 37 - - -state 117 - - 21 cond_expr: cond_expr AND_OP bool_pri . - 22 | '(' cond_expr AND_OP bool_pri . ')' - - ')' shift, and go to state 132 - - $default reduce using rule 21 (cond_expr) - - -state 118 - - 23 cond_expr: cond_expr OR_OP bool_pri . - 24 | '(' cond_expr OR_OP bool_pri . ')' - - ')' shift, and go to state 133 - - $default reduce using rule 23 (cond_expr) - - -state 119 - - 25 bool_pri: expr comp expr . - 26 | '(' expr comp expr . ')' - - ')' shift, and go to state 134 - - $default reduce using rule 25 (bool_pri) - - -state 120 - - 27 bool_pri: expr IN '(' in_expr_list . ')' - 38 in_expr_list: in_expr_list . ',' expr - - ')' shift, and go to state 135 - ',' shift, and go to state 136 - - -state 121 - - 37 in_expr_list: expr . - - $default reduce using rule 37 (in_expr_list) - - -state 122 - - 28 bool_pri: expr BETWEEN expr AND_OP . expr - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 137 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 123 - - 19 join_expr: JOIN NAME_OB '.' NAME_OB AS . NAME_OB - - NAME_OB shift, and go to state 138 - - -state 124 - - 18 join_expr: JOIN NAME_OB '.' NAME_OB NAME_OB . - - $default reduce using rule 18 (join_expr) - - -state 125 - - 26 bool_pri: '(' expr comp expr . ')' - - ')' shift, and go to state 134 - - -state 126 - - 66 values_expr_lists: '(' values_expr_list ')' . - - $default reduce using rule 66 (values_expr_lists) - - -state 127 - - 74 values_expr_list: values_expr_list ',' . expr - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 139 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 128 - - 67 values_expr_lists: values_expr_lists ',' '(' . values_expr_list ')' - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 108 - token_list go to state 35 - token go to state 36 - operator go to state 37 - values_expr_list go to state 140 - - -state 129 - - 47 token: NAME_OB '.' NAME_OB '.' NAME_OB . - - $default reduce using rule 47 (token) - - -state 130 - - 41 token_list: token_list . token - 43 func_param_list: token_list ',' token_list . - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - $default reduce using rule 43 (func_param_list) - - token go to state 73 - operator go to state 37 - - -state 131 - - 41 token_list: token_list . token - 44 func_param_list: func_param_list ',' token_list . - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - $default reduce using rule 44 (func_param_list) - - token go to state 73 - operator go to state 37 - - -state 132 - - 22 cond_expr: '(' cond_expr AND_OP bool_pri ')' . - - $default reduce using rule 22 (cond_expr) - - -state 133 - - 24 cond_expr: '(' cond_expr OR_OP bool_pri ')' . - - $default reduce using rule 24 (cond_expr) - - -state 134 - - 26 bool_pri: '(' expr comp expr ')' . - - $default reduce using rule 26 (bool_pri) - - -state 135 - - 27 bool_pri: expr IN '(' in_expr_list ')' . - - $default reduce using rule 27 (bool_pri) - - -state 136 - - 38 in_expr_list: in_expr_list ',' . expr - - PLACE_HOLDER shift, and go to state 19 - NAME_OB shift, and go to state 20 - STR_VAL shift, and go to state 21 - INT_VAL shift, and go to state 22 - POS_PLACE_HOLDER shift, and go to state 23 - '+' shift, and go to state 25 - '-' shift, and go to state 26 - '*' shift, and go to state 27 - '/' shift, and go to state 28 - '%' shift, and go to state 29 - '&' shift, and go to state 30 - '!' shift, and go to state 31 - - expr go to state 141 - token_list go to state 35 - token go to state 36 - operator go to state 37 - - -state 137 - - 28 bool_pri: expr BETWEEN expr AND_OP expr . - - $default reduce using rule 28 (bool_pri) - - -state 138 - - 19 join_expr: JOIN NAME_OB '.' NAME_OB AS NAME_OB . - - $default reduce using rule 19 (join_expr) - - -state 139 - - 74 values_expr_list: values_expr_list ',' expr . - - $default reduce using rule 74 (values_expr_list) - - -state 140 - - 67 values_expr_lists: values_expr_lists ',' '(' values_expr_list . ')' - 74 values_expr_list: values_expr_list . ',' expr - - ')' shift, and go to state 142 - ',' shift, and go to state 127 - - -state 141 - - 38 in_expr_list: in_expr_list ',' expr . - - $default reduce using rule 38 (in_expr_list) - - -state 142 - - 67 values_expr_lists: values_expr_lists ',' '(' values_expr_list ')' . - - $default reduce using rule 67 (values_expr_lists) diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser.y b/src/obproxy/opsql/expr_parser/ob_expr_parser.y index 588d3e44657fc0476f60854a03c47117a77439fe..2bf5701be7bb1e715f74dd45c68a006a07debe66 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser.y +++ b/src/obproxy/opsql/expr_parser/ob_expr_parser.y @@ -188,18 +188,20 @@ static int64_t get_part_key_idx(ObProxyParseString *db_name, ObExprParseResult *result) { int64_t part_key_idx = IDX_NO_PART_KEY_COLUMN; - if (NULL != db_name && !is_equal(db_name, &result->table_info_.database_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != table_name - && !is_equal(table_name, &result->table_info_.table_name_) - && !is_equal(table_name, &result->table_info_.alias_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != column_name) { - int64_t i = 0; - for (i = 0; i < result->part_key_info_.key_num_ && part_key_idx < 0; ++i) { - if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { - part_key_idx = i; - break; + if (result->part_key_info_.key_num_ > 0) { + if (NULL != db_name && !is_equal(db_name, &result->table_info_.database_name_)) { + part_key_idx = IDX_NO_PART_KEY_COLUMN; + } else if (NULL != table_name + && !is_equal(table_name, &result->table_info_.table_name_) + && !is_equal(table_name, &result->table_info_.alias_name_)) { + part_key_idx = IDX_NO_PART_KEY_COLUMN; + } else if (NULL != column_name) { + int64_t i = 0; + for (i = 0; i < result->part_key_info_.key_num_ && part_key_idx < 0; ++i) { + if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { + part_key_idx = i; + break; + } } } } @@ -366,7 +368,7 @@ static inline void add_right_relation_value(ObExprParseResult *result, %token INT_VAL POS_PLACE_HOLDER %type comp %type token opt_column -%type expr token_list in_expr_list column_list +%type expr token_list in_expr_list column_list func_param_list %type operator %type bool_pri %start start @@ -403,6 +405,7 @@ cond_expr: bool_pri { check_and_add_relation(result, $1); } | '(' cond_expr OR_OP bool_pri ')' { check_and_add_relation(result, $4); } bool_pri: expr comp expr { add_relation(result, $1, $2,$3); $$ = get_relation(result, $1, $2, $3); } + | expr comp '(' expr ')' { add_relation(result, $1, $2,$4); $$ = get_relation(result, $1, $2, $4); } | '(' expr comp expr ')' { $$ = get_relation(result, $2, $3, $4); add_relation(result, $2, $3,$4); } | expr IN '(' in_expr_list ')' { $$ = get_relation(result, $1, F_COMP_EQ, $4); add_relation(result, $1, F_COMP_EQ,$4); } | expr BETWEEN expr AND_OP expr @@ -438,9 +441,9 @@ expr: token_list { $$ = $1; } token_list: token { malloc_list($$, result, $1); } | token_list token { add_token($1, result, $2); $$ = $1; } -func_param_list: /* empty */ - | token_list ',' token_list {} - | func_param_list ',' token_list {} +func_param_list: { $$ = NULL; } /* empty */ + | token_list ',' token_list { add_token_list($1, $3); $$ = $1; } + | func_param_list ',' token_list { add_token_list($1, $3); $$ = $1; } token: NAME_OB { @@ -470,6 +473,7 @@ token: NAME_OB { malloc_node($$, result, TOKEN_FUNC); $$->str_value_ = $1; + $$->child_ = $3; } | INT_VAL { malloc_node($$, result, TOKEN_INT_VAL); $$->int_value_ = $1; } | STR_VAL { malloc_node($$, result, TOKEN_STR_VAL); $$->str_value_ = $1; } diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.c b/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.c deleted file mode 100644 index 7c1625726c81ca9b6eeeded64af5b5e1aef0fcd9..0000000000000000000000000000000000000000 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.c +++ /dev/null @@ -1,2692 +0,0 @@ -#line 2 "ob_expr_parser_lex.c" - -#line 4 "ob_expr_parser_lex.c" - -#define YY_INT_ALIGNED short int - -/* A lexical scanner generated by flex */ - -#define FLEX_SCANNER -#define YY_FLEX_MAJOR_VERSION 2 -#define YY_FLEX_MINOR_VERSION 5 -#define YY_FLEX_SUBMINOR_VERSION 35 -#if YY_FLEX_SUBMINOR_VERSION > 0 -#define FLEX_BETA -#endif - -/* First, we deal with platform-specific or compiler-specific issues. */ - -/* begin standard C headers. */ -#include -#include -#include -#include - -/* end standard C headers. */ - -/* flex integer type definitions */ - -#ifndef FLEXINT_H -#define FLEXINT_H - -/* C99 systems have . Non-C99 systems may or may not. */ - -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - -/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. - */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS 1 -#endif - -#include -typedef int8_t flex_int8_t; -typedef uint8_t flex_uint8_t; -typedef int16_t flex_int16_t; -typedef uint16_t flex_uint16_t; -typedef int32_t flex_int32_t; -typedef uint32_t flex_uint32_t; -#else -typedef signed char flex_int8_t; -typedef short int flex_int16_t; -typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; -typedef unsigned short int flex_uint16_t; -typedef unsigned int flex_uint32_t; -#endif /* ! C99 */ - -/* Limits of integral types. */ -#ifndef INT8_MIN -#define INT8_MIN (-128) -#endif -#ifndef INT16_MIN -#define INT16_MIN (-32767-1) -#endif -#ifndef INT32_MIN -#define INT32_MIN (-2147483647-1) -#endif -#ifndef INT8_MAX -#define INT8_MAX (127) -#endif -#ifndef INT16_MAX -#define INT16_MAX (32767) -#endif -#ifndef INT32_MAX -#define INT32_MAX (2147483647) -#endif -#ifndef UINT8_MAX -#define UINT8_MAX (255U) -#endif -#ifndef UINT16_MAX -#define UINT16_MAX (65535U) -#endif -#ifndef UINT32_MAX -#define UINT32_MAX (4294967295U) -#endif - -#endif /* ! FLEXINT_H */ - -#ifdef __cplusplus - -/* The "const" storage-class-modifier is valid. */ -#define YY_USE_CONST - -#else /* ! __cplusplus */ - -/* C99 requires __STDC__ to be defined as 1. */ -#if defined (__STDC__) - -#define YY_USE_CONST - -#endif /* defined (__STDC__) */ -#endif /* ! __cplusplus */ - -#ifdef YY_USE_CONST -#define yyconst const -#else -#define yyconst -#endif - -/* Returned upon end-of-file. */ -#define YY_NULL 0 - -/* Promotes a possibly negative, possibly signed char to an unsigned - * integer for use as an array index. If the signed char is negative, - * we want to instead treat it as an 8-bit unsigned char, hence the - * double cast. - */ -#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) - -/* An opaque pointer. */ -#ifndef YY_TYPEDEF_YY_SCANNER_T -#define YY_TYPEDEF_YY_SCANNER_T -typedef void* yyscan_t; -#endif - -/* For convenience, these vars (plus the bison vars far below) - are macros in the reentrant scanner. */ -#define yyin yyg->yyin_r -#define yyout yyg->yyout_r -#define yyextra yyg->yyextra_r -#define yyleng yyg->yyleng_r -#define yytext yyg->yytext_r -#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno) -#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column) -#define yy_flex_debug yyg->yy_flex_debug_r - -/* Enter a start condition. This macro really ought to take a parameter, - * but we do it the disgusting crufty way forced on us by the ()-less - * definition of BEGIN. - */ -#define BEGIN yyg->yy_start = 1 + 2 * - -/* Translate the current start state into a value that can be later handed - * to BEGIN to return to the state. The YYSTATE alias is for lex - * compatibility. - */ -#define YY_START ((yyg->yy_start - 1) / 2) -#define YYSTATE YY_START - -/* Action number for EOF rule of a given start state. */ -#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) - -/* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE obexprrestart(yyin ,yyscanner ) - -#define YY_END_OF_BUFFER_CHAR 0 - -/* Size of default input buffer. */ -#ifndef YY_BUF_SIZE -#define YY_BUF_SIZE 16384 -#endif - -/* The state buf must be large enough to hold one state per character in the main buffer. - */ -#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) - -#ifndef YY_TYPEDEF_YY_BUFFER_STATE -#define YY_TYPEDEF_YY_BUFFER_STATE -typedef struct yy_buffer_state *YY_BUFFER_STATE; -#endif - -#define EOB_ACT_CONTINUE_SCAN 0 -#define EOB_ACT_END_OF_FILE 1 -#define EOB_ACT_LAST_MATCH 2 - - #define YY_LESS_LINENO(n) - -/* Return all but the first "n" matched characters back to the input stream. */ -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - int yyless_macro_arg = (n); \ - YY_LESS_LINENO(yyless_macro_arg);\ - *yy_cp = yyg->yy_hold_char; \ - YY_RESTORE_YY_MORE_OFFSET \ - yyg->yy_c_buf_p = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ - YY_DO_BEFORE_ACTION; /* set up yytext again */ \ - } \ - while ( 0 ) - -#define unput(c) yyunput( c, yyg->yytext_ptr , yyscanner ) - -#ifndef YY_TYPEDEF_YY_SIZE_T -#define YY_TYPEDEF_YY_SIZE_T -typedef size_t yy_size_t; -#endif - -#ifndef YY_STRUCT_YY_BUFFER_STATE -#define YY_STRUCT_YY_BUFFER_STATE -struct yy_buffer_state - { - FILE *yy_input_file; - - char *yy_ch_buf; /* input buffer */ - char *yy_buf_pos; /* current position in input buffer */ - - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - yy_size_t yy_buf_size; - - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - int yy_n_chars; - - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; - - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; - - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; - - int yy_bs_lineno; /**< The line count. */ - int yy_bs_column; /**< The column count. */ - - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; - - int yy_buffer_status; - -#define YY_BUFFER_NEW 0 -#define YY_BUFFER_NORMAL 1 - /* When an EOF's been seen but there's still some text to process - * then we mark the buffer as YY_EOF_PENDING, to indicate that we - * shouldn't try reading from the input source any more. We might - * still have a bunch of tokens to match, though, because of - * possible backing-up. - * - * When we actually see the EOF, we change the status to "new" - * (via obexprrestart()), so that the user can continue scanning by - * just pointing yyin at a new input file. - */ -#define YY_BUFFER_EOF_PENDING 2 - - }; -#endif /* !YY_STRUCT_YY_BUFFER_STATE */ - -/* We provide macros for accessing buffer states in case in the - * future we want to put the buffer states in a more general - * "scanner state". - * - * Returns the top of the stack, or NULL. - */ -#define YY_CURRENT_BUFFER ( yyg->yy_buffer_stack \ - ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \ - : NULL) - -/* Same as previous macro, but useful when we know that the buffer stack is not - * NULL or when we need an lvalue. For internal use only. - */ -#define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] - -void obexprrestart (FILE *input_file ,yyscan_t yyscanner ); -void obexpr_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); -void obexpr_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); -void obexpr_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); -void obexprpush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); -void obexprpop_buffer_state (yyscan_t yyscanner ); - -static void obexprensure_buffer_stack (yyscan_t yyscanner ); -static void obexpr_load_buffer_state (yyscan_t yyscanner ); -static void obexpr_init_buffer (YY_BUFFER_STATE b,FILE *file ,yyscan_t yyscanner ); - -#define YY_FLUSH_BUFFER obexpr_flush_buffer(YY_CURRENT_BUFFER ,yyscanner) - -YY_BUFFER_STATE obexpr_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); - -void *obexpralloc (yy_size_t ,yyscan_t yyscanner ); -void *obexprrealloc (void *,yy_size_t ,yyscan_t yyscanner ); -void obexprfree (void * ,yyscan_t yyscanner ); - -#define yy_new_buffer obexpr_create_buffer - -#define yy_set_interactive(is_interactive) \ - { \ - if ( ! YY_CURRENT_BUFFER ){ \ - obexprensure_buffer_stack (yyscanner); \ - YY_CURRENT_BUFFER_LVALUE = \ - obexpr_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ - } \ - YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ - } - -#define yy_set_bol(at_bol) \ - { \ - if ( ! YY_CURRENT_BUFFER ){\ - obexprensure_buffer_stack (yyscanner); \ - YY_CURRENT_BUFFER_LVALUE = \ - obexpr_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ - } \ - YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ - } - -#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) - -/* Begin user sect3 */ - -#define obexprwrap(n) 1 -#define YY_SKIP_YYWRAP - -typedef unsigned char YY_CHAR; - -typedef int yy_state_type; - -#define yytext_ptr yytext_r - -static yy_state_type yy_get_previous_state (yyscan_t yyscanner ); -static yy_state_type yy_try_NUL_trans (yy_state_type current_state ,yyscan_t yyscanner); -static int yy_get_next_buffer (yyscan_t yyscanner ); -static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ); - -/* Done after the current pattern has been matched and before the - * corresponding action - sets up yytext. - */ -#define YY_DO_BEFORE_ACTION \ - yyg->yytext_ptr = yy_bp; \ - yyleng = (size_t) (yy_cp - yy_bp); \ - yyg->yy_hold_char = *yy_cp; \ - yyg->yy_c_buf_p = yy_cp; - -#define YY_NUM_RULES 54 -#define YY_END_OF_BUFFER 55 -/* This struct is not used in this scanner, - but its presence is necessary. */ -struct yy_trans_info - { - flex_int32_t yy_verify; - flex_int32_t yy_nxt; - }; -static yyconst flex_int16_t yy_accept[166] = - { 0, - 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 53, 31, 31, 32, 43, 31, 30, 32, 32, - 37, 32, 32, 32, 28, 32, 23, 18, 21, 25, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 49, 32, 36, 35, 36, 38, 39, - 54, 44, 45, 54, 51, 52, 31, 24, 31, 30, - 14, 0, 29, 33, 29, 28, 30, 0, 26, 22, - 20, 30, 2, 30, 30, 30, 30, 17, 30, 30, - 11, 15, 30, 30, 30, 30, 16, 0, 34, 38, - 0, 0, 40, 0, 41, 44, 0, 46, 0, 0, - - 47, 51, 50, 31, 0, 29, 0, 0, 29, 27, - 19, 13, 30, 5, 30, 30, 30, 30, 30, 4, - 30, 30, 42, 0, 42, 0, 48, 0, 48, 0, - 31, 31, 0, 29, 0, 29, 29, 30, 30, 30, - 10, 30, 30, 30, 30, 0, 0, 30, 7, 30, - 6, 9, 30, 1, 0, 0, 42, 0, 0, 48, - 30, 8, 3, 12, 0 - } ; - -static yyconst flex_int32_t yy_ec[256] = - { 0, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, - 1, 2, 4, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 5, 6, 7, 8, 9, 10, 11, 9, - 9, 12, 13, 9, 14, 15, 16, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 18, 9, 19, - 20, 21, 22, 9, 23, 24, 8, 25, 26, 27, - 28, 29, 30, 31, 8, 32, 33, 34, 35, 36, - 8, 37, 38, 39, 40, 41, 42, 8, 8, 8, - 1, 43, 1, 9, 8, 44, 45, 46, 8, 47, - - 48, 49, 50, 51, 52, 53, 8, 54, 55, 56, - 57, 58, 8, 59, 60, 61, 62, 63, 64, 8, - 8, 8, 1, 65, 1, 9, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1 - } ; - -static yyconst flex_int32_t yy_meta[66] = - { 0, - 1, 1, 2, 2, 1, 3, 1, 4, 1, 1, - 5, 1, 1, 1, 1, 1, 4, 1, 1, 1, - 1, 1, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 6, 7, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 1 - } ; - -static yyconst flex_int16_t yy_base[182] = - { 0, - 0, 0, 63, 64, 57, 58, 64, 65, 360, 359, - 402, 405, 70, 75, 381, 405, 0, 0, 405, 390, - 405, 385, 381, 385, 65, 70, 65, 405, 376, 405, - 54, 55, 54, 53, 70, 60, 60, 66, 65, 62, - 72, 81, 76, 405, 330, 405, 405, 113, 0, 128, - 0, 0, 134, 0, 0, 350, 141, 405, 0, 0, - 405, 147, 129, 405, 130, 137, 144, 376, 349, 338, - 405, 81, 0, 114, 122, 125, 121, 0, 134, 132, - 0, 0, 124, 128, 136, 143, 405, 158, 405, 0, - 169, 332, 405, 326, 405, 0, 190, 405, 323, 314, - - 405, 0, 405, 195, 187, 185, 192, 301, 266, 261, - 405, 0, 153, 0, 163, 177, 174, 180, 182, 0, - 173, 177, 405, 262, 257, 216, 405, 254, 249, 219, - 119, 224, 166, 162, 159, 117, 116, 189, 180, 190, - 0, 192, 214, 214, 215, 240, 246, 219, 0, 219, - 0, 0, 216, 0, 253, 268, 255, 271, 278, 282, - 231, 0, 0, 0, 405, 288, 295, 302, 309, 316, - 105, 323, 330, 337, 344, 351, 357, 364, 371, 378, - 385 - } ; - -static yyconst flex_int16_t yy_def[182] = - { 0, - 165, 1, 166, 166, 167, 167, 168, 168, 169, 169, - 165, 165, 165, 165, 165, 165, 170, 171, 165, 165, - 165, 165, 165, 165, 171, 165, 165, 165, 165, 165, - 171, 171, 171, 171, 171, 171, 171, 171, 171, 171, - 171, 171, 171, 165, 165, 165, 165, 165, 172, 165, - 173, 174, 165, 175, 176, 165, 165, 165, 170, 171, - 165, 165, 165, 165, 165, 171, 171, 165, 165, 165, - 165, 171, 171, 171, 171, 171, 171, 171, 171, 171, - 171, 171, 171, 171, 171, 171, 165, 165, 165, 172, - 165, 177, 165, 165, 165, 174, 165, 165, 178, 165, - - 165, 176, 165, 179, 165, 165, 165, 165, 171, 165, - 165, 171, 171, 171, 171, 171, 171, 171, 171, 171, - 171, 171, 165, 177, 177, 165, 165, 178, 178, 165, - 179, 179, 165, 165, 165, 165, 165, 171, 171, 171, - 171, 171, 171, 171, 171, 180, 181, 171, 171, 171, - 171, 171, 171, 171, 180, 180, 180, 181, 181, 181, - 171, 171, 171, 171, 0, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165 - } ; - -static yyconst flex_int16_t yy_nxt[471] = - { 0, - 12, 13, 14, 13, 15, 16, 17, 18, 19, 20, - 21, 19, 19, 22, 23, 24, 25, 26, 27, 28, - 29, 30, 31, 32, 18, 18, 33, 34, 35, 36, - 37, 38, 18, 18, 39, 18, 40, 41, 18, 18, - 42, 43, 12, 44, 31, 32, 18, 18, 33, 34, - 35, 36, 37, 38, 18, 18, 39, 18, 40, 41, - 18, 18, 42, 43, 45, 47, 47, 50, 50, 53, - 53, 57, 57, 57, 48, 48, 57, 57, 57, 65, - 74, 66, 68, 68, 70, 58, 69, 72, 75, 76, - 67, 73, 77, 78, 79, 80, 83, 84, 81, 51, - - 51, 82, 74, 85, 86, 112, 54, 54, 60, 72, - 75, 76, 67, 73, 77, 78, 79, 80, 83, 84, - 81, 165, 165, 82, 88, 85, 86, 112, 89, 91, - 91, 91, 137, 136, 92, 97, 97, 97, 93, 98, - 99, 94, 57, 57, 57, 63, 106, 100, 104, 104, - 104, 65, 113, 66, 105, 107, 108, 108, 114, 115, - 109, 116, 67, 117, 118, 119, 120, 121, 122, 88, - 91, 91, 91, 89, 113, 136, 105, 107, 134, 123, - 114, 115, 134, 116, 67, 117, 118, 119, 120, 121, - 122, 97, 97, 97, 138, 127, 132, 104, 104, 133, - - 133, 106, 139, 134, 135, 135, 140, 141, 136, 142, - 107, 143, 144, 145, 148, 149, 138, 146, 146, 146, - 147, 147, 147, 150, 139, 132, 104, 104, 140, 141, - 151, 142, 107, 143, 144, 145, 148, 149, 152, 153, - 154, 156, 146, 146, 161, 150, 162, 159, 147, 147, - 157, 160, 151, 163, 129, 165, 165, 165, 165, 129, - 152, 153, 154, 157, 164, 157, 161, 125, 162, 156, - 146, 146, 125, 165, 165, 163, 160, 110, 157, 159, - 147, 147, 109, 160, 165, 165, 164, 160, 46, 46, - 46, 46, 46, 46, 46, 49, 49, 49, 49, 49, - - 49, 49, 52, 52, 52, 52, 52, 52, 52, 55, - 55, 55, 55, 55, 55, 55, 59, 137, 59, 59, - 59, 59, 59, 90, 90, 90, 90, 130, 129, 90, - 95, 95, 95, 95, 95, 95, 95, 96, 96, 126, - 96, 96, 125, 96, 101, 101, 101, 101, 101, 101, - 101, 102, 102, 102, 102, 102, 102, 124, 111, 124, - 124, 124, 124, 124, 128, 69, 128, 128, 128, 128, - 128, 131, 131, 131, 131, 131, 131, 131, 155, 155, - 155, 155, 155, 155, 155, 158, 158, 158, 158, 158, - 158, 158, 110, 103, 87, 71, 64, 63, 62, 61, - - 58, 165, 56, 56, 11, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165 - } ; - -static yyconst flex_int16_t yy_chk[471] = - { 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 3, 4, 5, 6, 7, - 8, 13, 13, 13, 3, 4, 14, 14, 14, 25, - 32, 25, 26, 26, 27, 27, 26, 31, 33, 34, - 25, 31, 35, 36, 37, 38, 40, 41, 39, 5, - - 6, 39, 32, 42, 43, 72, 7, 8, 171, 31, - 33, 34, 25, 31, 35, 36, 37, 38, 40, 41, - 39, 131, 131, 39, 48, 42, 43, 72, 48, 50, - 50, 50, 137, 136, 50, 53, 53, 53, 50, 53, - 53, 50, 57, 57, 57, 63, 65, 53, 62, 62, - 62, 66, 74, 66, 63, 65, 67, 67, 75, 76, - 67, 77, 66, 79, 80, 83, 84, 85, 86, 88, - 91, 91, 91, 88, 74, 135, 63, 65, 134, 91, - 75, 76, 133, 77, 66, 79, 80, 83, 84, 85, - 86, 97, 97, 97, 113, 97, 104, 104, 104, 105, - - 105, 106, 115, 105, 107, 107, 116, 117, 107, 118, - 106, 119, 121, 122, 138, 139, 113, 126, 126, 126, - 130, 130, 130, 140, 115, 132, 132, 132, 116, 117, - 142, 118, 106, 119, 121, 122, 138, 139, 143, 144, - 145, 146, 146, 146, 148, 140, 150, 147, 147, 147, - 146, 147, 142, 153, 129, 155, 155, 157, 157, 128, - 143, 144, 145, 155, 161, 157, 148, 125, 150, 156, - 156, 156, 124, 158, 158, 153, 158, 110, 156, 159, - 159, 159, 109, 159, 160, 160, 161, 160, 166, 166, - 166, 166, 166, 166, 166, 167, 167, 167, 167, 167, - - 167, 167, 168, 168, 168, 168, 168, 168, 168, 169, - 169, 169, 169, 169, 169, 169, 170, 108, 170, 170, - 170, 170, 170, 172, 172, 172, 172, 100, 99, 172, - 173, 173, 173, 173, 173, 173, 173, 174, 174, 94, - 174, 174, 92, 174, 175, 175, 175, 175, 175, 175, - 175, 176, 176, 176, 176, 176, 176, 177, 70, 177, - 177, 177, 177, 177, 178, 69, 178, 178, 178, 178, - 178, 179, 179, 179, 179, 179, 179, 179, 180, 180, - 180, 180, 180, 180, 180, 181, 181, 181, 181, 181, - 181, 181, 68, 56, 45, 29, 24, 23, 22, 20, - - 15, 11, 10, 9, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165 - } ; - -/* The intent behind this definition is that it'll catch - * any uses of REJECT which flex missed. - */ -#define REJECT reject_used_but_not_detected -#define yymore() yymore_used_but_not_detected -#define YY_MORE_ADJ 0 -#define YY_RESTORE_YY_MORE_OFFSET -#line 1 "ob_expr_parser.l" -#define YY_NO_INPUT 1 -#line 6 "ob_expr_parser.l" -#define YYSTYPE OBEXPRSTYPE -#define YYLTYPE OBEXPRLTYPE -#include -#include "opsql/ob_proxy_parse_define.h" -#include "opsql/ob_proxy_parse_malloc.h" -#include "opsql/expr_parser/ob_expr_parse_result.h" -#include "opsql/expr_parser/ob_expr_parser_tab.h" -extern void ob_expr_parser_fatal_error(yyconst char *msg, yyscan_t yyscanner); - -void store_expr_str(char* str, int64_t str_len, char* end_ptr, void *yyscanner); -void store_pos_place_holder(char* str, void *yyscanner); - -#define YY_FATAL_ERROR(msg) ob_expr_parser_fatal_error(msg, yyscanner) - -#define PUSH_STATE(state) \ - { yy_push_state(state, yyscanner); } - -#define POP_STATE(state) \ - { yy_pop_state(yyscanner); } - -#define RETURN_INT_VAL() \ -do {\ - errno = 0;\ - yylval->num = strtoll(yytext, NULL, 10);\ - if (0 != errno) {\ - yylval->num = 0;\ - }\ - return INT_VAL;\ -} while (0); - -#define RETURN_STR_VAL() \ - { store_expr_str(yytext, yyleng, yytext + yyleng, yyscanner); return STR_VAL; } - -#define RETURN_NUMBER_VAL() \ - { store_expr_str(yytext, yyleng, yytext + yyleng, yyscanner); return STR_VAL; } - -#define RETURN_NAME_OB() \ - { store_expr_str(yytext, yyleng, yytext + yyleng, yyscanner); return NAME_OB; } - - - - - -#line 638 "ob_expr_parser_lex.c" - -#define INITIAL 0 -#define in_c_comment 1 -#define sq 2 -#define dq 3 -#define bt 4 - -#ifndef YY_NO_UNISTD_H -/* Special case for "unistd.h", since it is non-ANSI. We include it way - * down here because we want the user's section 1 to have been scanned first. - * The user has a chance to override it with an option. - */ -#include -#endif - -#ifndef YY_EXTRA_TYPE -#define YY_EXTRA_TYPE void * -#endif - -/* Holds the entire state of the reentrant scanner. */ -struct yyguts_t - { - - /* User-defined. Not touched by flex. */ - YY_EXTRA_TYPE yyextra_r; - - /* The rest are the same as the globals declared in the non-reentrant scanner. */ - FILE *yyin_r, *yyout_r; - size_t yy_buffer_stack_top; /**< index of top of stack. */ - size_t yy_buffer_stack_max; /**< capacity of stack. */ - YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */ - char yy_hold_char; - int yy_n_chars; - int yyleng_r; - char *yy_c_buf_p; - int yy_init; - int yy_start; - int yy_did_buffer_switch_on_eof; - int yy_start_stack_ptr; - int yy_start_stack_depth; - int *yy_start_stack; - yy_state_type yy_last_accepting_state; - char* yy_last_accepting_cpos; - - int yylineno_r; - int yy_flex_debug_r; - - char *yytext_r; - int yy_more_flag; - int yy_more_len; - - YYSTYPE * yylval_r; - - YYLTYPE * yylloc_r; - - }; /* end struct yyguts_t */ - -static int yy_init_globals (yyscan_t yyscanner ); - - /* This must go here because YYSTYPE and YYLTYPE are included - * from bison output in section 1.*/ - # define yylval yyg->yylval_r - - # define yylloc yyg->yylloc_r - -int obexprlex_init (yyscan_t* scanner); - -int obexprlex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); - -/* Accessor methods to globals. - These are made visible to non-reentrant scanners for convenience. */ - -int obexprlex_destroy (yyscan_t yyscanner ); - -int obexprget_debug (yyscan_t yyscanner ); - -void obexprset_debug (int debug_flag ,yyscan_t yyscanner ); - -YY_EXTRA_TYPE obexprget_extra (yyscan_t yyscanner ); - -void obexprset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); - -FILE *obexprget_in (yyscan_t yyscanner ); - -void obexprset_in (FILE * in_str ,yyscan_t yyscanner ); - -FILE *obexprget_out (yyscan_t yyscanner ); - -void obexprset_out (FILE * out_str ,yyscan_t yyscanner ); - -int obexprget_leng (yyscan_t yyscanner ); - -char *obexprget_text (yyscan_t yyscanner ); - -int obexprget_lineno (yyscan_t yyscanner ); - -void obexprset_lineno (int line_number ,yyscan_t yyscanner ); - -YYSTYPE * obexprget_lval (yyscan_t yyscanner ); - -void obexprset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner ); - - YYLTYPE *obexprget_lloc (yyscan_t yyscanner ); - - void obexprset_lloc (YYLTYPE * yylloc_param ,yyscan_t yyscanner ); - -/* Macros after this point can all be overridden by user definitions in - * section 1. - */ - -#ifndef YY_SKIP_YYWRAP -#ifdef __cplusplus -extern "C" int obexprwrap (yyscan_t yyscanner ); -#else -extern int obexprwrap (yyscan_t yyscanner ); -#endif -#endif - -#ifndef yytext_ptr -static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); -#endif - -#ifdef YY_NEED_STRLEN -static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); -#endif - -#ifndef YY_NO_INPUT - -#ifdef __cplusplus -static int yyinput (yyscan_t yyscanner ); -#else -static int input (yyscan_t yyscanner ); -#endif - -#endif - - static void yy_push_state (int new_state ,yyscan_t yyscanner); - - static void yy_pop_state (yyscan_t yyscanner ); - - -/* Amount of stuff to slurp up with each read. */ -#ifndef YY_READ_BUF_SIZE -#define YY_READ_BUF_SIZE 8192 -#endif - -/* Copy whatever the last rule matched to the standard output. */ -#ifndef ECHO -/* This used to be an fputs(), but since the string might contain NUL's, - * we now use fwrite(). - */ -#define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0) -#endif - -/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, - * is returned in "result". - */ -#ifndef YY_INPUT -#define YY_INPUT(buf,result,max_size) \ - if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ - { \ - int c = '*'; \ - unsigned n; \ - for ( n = 0; n < max_size && \ - (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ - buf[n] = (char) c; \ - if ( c == '\n' ) \ - buf[n++] = (char) c; \ - if ( c == EOF && ferror( yyin ) ) \ - YY_FATAL_ERROR( "input in flex scanner failed" ); \ - result = n; \ - } \ - else \ - { \ - errno=0; \ - while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \ - { \ - if( errno != EINTR) \ - { \ - YY_FATAL_ERROR( "input in flex scanner failed" ); \ - break; \ - } \ - errno=0; \ - clearerr(yyin); \ - } \ - }\ -\ - -#endif - -/* No semi-colon after return; correct usage is to write "yyterminate();" - - * we don't want an extra ';' after the "return" because that will cause - * some compilers to complain about unreachable statements. - */ -#ifndef yyterminate -#define yyterminate() return YY_NULL -#endif - -/* Number of entries by which start-condition stack grows. */ -#ifndef YY_START_STACK_INCR -#define YY_START_STACK_INCR 25 -#endif - -/* Report a fatal error. */ -#ifndef YY_FATAL_ERROR -#define YY_FATAL_ERROR(msg) yy_fatal_error( msg , yyscanner) -#endif - -/* end tables serialization structures and prototypes */ - -/* Default declaration of generated scanner - a define so the user can - * easily add parameters. - */ -#ifndef YY_DECL -#define YY_DECL_IS_OURS 1 - -extern int obexprlex \ - (YYSTYPE * yylval_param,YYLTYPE * yylloc_param ,yyscan_t yyscanner); - -#define YY_DECL int obexprlex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param , yyscan_t yyscanner) -#endif /* !YY_DECL */ - -/* Code executed at the beginning of each rule, after yytext and yyleng - * have been set up. - */ -#ifndef YY_USER_ACTION -#define YY_USER_ACTION -#endif - -/* Code executed at the end of each rule. */ -#ifndef YY_BREAK -#define YY_BREAK break; -#endif - -#define YY_RULE_SETUP \ - YY_USER_ACTION - -/** The main scanner function which does all the work. - */ -YY_DECL -{ - register yy_state_type yy_current_state; - register char *yy_cp, *yy_bp; - register int yy_act; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - -#line 87 "ob_expr_parser.l" - -#line 889 "ob_expr_parser_lex.c" - - yylval = yylval_param; - - yylloc = yylloc_param; - - if ( !yyg->yy_init ) - { - yyg->yy_init = 1; - -#ifdef YY_USER_INIT - YY_USER_INIT; -#endif - - if ( ! yyg->yy_start ) - yyg->yy_start = 1; /* first start state */ - - if ( ! yyin ) - yyin = stdin; - - if ( ! yyout ) - yyout = stdout; - - if ( ! YY_CURRENT_BUFFER ) { - obexprensure_buffer_stack (yyscanner); - YY_CURRENT_BUFFER_LVALUE = - obexpr_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); - } - - obexpr_load_buffer_state(yyscanner ); - } - - while ( 1 ) /* loops until end-of-file is reached */ - { - yy_cp = yyg->yy_c_buf_p; - - /* Support of yytext. */ - *yy_cp = yyg->yy_hold_char; - - /* yy_bp points to the position in yy_ch_buf of the start of - * the current run. - */ - yy_bp = yy_cp; - - yy_current_state = yyg->yy_start; -yy_match: - do - { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; - if ( yy_accept[yy_current_state] ) - { - yyg->yy_last_accepting_state = yy_current_state; - yyg->yy_last_accepting_cpos = yy_cp; - } - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - ++yy_cp; - } - while ( yy_base[yy_current_state] != 405 ); - -yy_find_action: - yy_act = yy_accept[yy_current_state]; - if ( yy_act == 0 ) - { /* have to back up */ - yy_cp = yyg->yy_last_accepting_cpos; - yy_current_state = yyg->yy_last_accepting_state; - yy_act = yy_accept[yy_current_state]; - } - - YY_DO_BEFORE_ACTION; - -do_action: /* This label is used only to access EOF actions. */ - - switch ( yy_act ) - { /* beginning of action switch */ - case 0: /* must back up */ - /* undo the effects of YY_DO_BEFORE_ACTION */ - *yy_cp = yyg->yy_hold_char; - yy_cp = yyg->yy_last_accepting_cpos; - yy_current_state = yyg->yy_last_accepting_state; - goto yy_find_action; - -case 1: -YY_RULE_SETUP -#line 88 "ob_expr_parser.l" -{ return WHERE; } - YY_BREAK -case 2: -YY_RULE_SETUP -#line 89 "ob_expr_parser.l" -{ return AS; } - YY_BREAK -case 3: -YY_RULE_SETUP -#line 90 "ob_expr_parser.l" -{ return VALUES; } - YY_BREAK -case 4: -YY_RULE_SETUP -#line 91 "ob_expr_parser.l" -{ return SET; } - YY_BREAK -case 5: -YY_RULE_SETUP -#line 92 "ob_expr_parser.l" -{ return END_WHERE; } - YY_BREAK -case 6: -YY_RULE_SETUP -#line 93 "ob_expr_parser.l" -{ return END_WHERE; } - YY_BREAK -case 7: -YY_RULE_SETUP -#line 94 "ob_expr_parser.l" -{ return END_WHERE; } - YY_BREAK -case 8: -YY_RULE_SETUP -#line 95 "ob_expr_parser.l" -{ return END_WHERE; } - YY_BREAK -case 9: -YY_RULE_SETUP -#line 96 "ob_expr_parser.l" -{ return ROWID; } - YY_BREAK -case 10: -YY_RULE_SETUP -#line 98 "ob_expr_parser.l" -{ return JOIN; } - YY_BREAK -case 11: -YY_RULE_SETUP -#line 99 "ob_expr_parser.l" -{ return ON; } - YY_BREAK -case 12: -YY_RULE_SETUP -#line 100 "ob_expr_parser.l" -{ return BETWEEN; } - YY_BREAK -case 13: -YY_RULE_SETUP -#line 101 "ob_expr_parser.l" -{ return AND_OP; } - YY_BREAK -case 14: -YY_RULE_SETUP -#line 102 "ob_expr_parser.l" -{ return AND_OP; } - YY_BREAK -case 15: -YY_RULE_SETUP -#line 103 "ob_expr_parser.l" -{ return OR_OP; } - YY_BREAK -case 16: -YY_RULE_SETUP -#line 104 "ob_expr_parser.l" -{ return OR_OP; } - YY_BREAK -case 17: -YY_RULE_SETUP -#line 105 "ob_expr_parser.l" -{ return IN; } - YY_BREAK -case 18: -YY_RULE_SETUP -#line 106 "ob_expr_parser.l" -{ return COMP_EQ; } - YY_BREAK -case 19: -YY_RULE_SETUP -#line 107 "ob_expr_parser.l" -{ return COMP_NSEQ; } - YY_BREAK -case 20: -YY_RULE_SETUP -#line 108 "ob_expr_parser.l" -{ return COMP_GE; } - YY_BREAK -case 21: -YY_RULE_SETUP -#line 109 "ob_expr_parser.l" -{ return COMP_GT; } - YY_BREAK -case 22: -YY_RULE_SETUP -#line 110 "ob_expr_parser.l" -{ return COMP_LE; } - YY_BREAK -case 23: -YY_RULE_SETUP -#line 111 "ob_expr_parser.l" -{ return COMP_LT; } - YY_BREAK -case 24: -YY_RULE_SETUP -#line 112 "ob_expr_parser.l" -{ return COMP_NE; } - YY_BREAK -case 25: -YY_RULE_SETUP -#line 113 "ob_expr_parser.l" -{ return PLACE_HOLDER; } - YY_BREAK -case 26: -YY_RULE_SETUP -#line 114 "ob_expr_parser.l" -{ store_pos_place_holder(yytext + 1, yyscanner); return POS_PLACE_HOLDER; } - YY_BREAK -case 27: -YY_RULE_SETUP -#line 115 "ob_expr_parser.l" -{ store_pos_place_holder(yytext + 1, yyscanner); return POS_PLACE_HOLDER; } - YY_BREAK -case 28: -YY_RULE_SETUP -#line 117 "ob_expr_parser.l" -{ RETURN_INT_VAL(); } - YY_BREAK -case 29: -YY_RULE_SETUP -#line 118 "ob_expr_parser.l" -{ RETURN_NUMBER_VAL(); } - YY_BREAK -case 30: -YY_RULE_SETUP -#line 119 "ob_expr_parser.l" -{ RETURN_NAME_OB(); } - YY_BREAK -case 31: -/* rule 31 can match eol */ -YY_RULE_SETUP -#line 120 "ob_expr_parser.l" -{ } - YY_BREAK -case 32: -YY_RULE_SETUP -#line 121 "ob_expr_parser.l" -{ return yytext[0]; } - YY_BREAK -/* comment */ -case 33: -YY_RULE_SETUP -#line 124 "ob_expr_parser.l" -{ PUSH_STATE(in_c_comment); } - YY_BREAK -case 34: -YY_RULE_SETUP -#line 125 "ob_expr_parser.l" -{ POP_STATE(); } - YY_BREAK -case 35: -/* rule 35 can match eol */ -YY_RULE_SETUP -#line 126 "ob_expr_parser.l" -{} - YY_BREAK -case 36: -YY_RULE_SETUP -#line 127 "ob_expr_parser.l" -{} - YY_BREAK -/* quote */ -case 37: -YY_RULE_SETUP -#line 130 "ob_expr_parser.l" -{ - PUSH_STATE(sq); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - p->tmp_buf_ = (yytext + 1); - p->tmp_start_ptr_ = yytext; - p->tmp_len_ = 0; - } -} - YY_BREAK -case 38: -/* rule 38 can match eol */ -YY_RULE_SETUP -#line 140 "ob_expr_parser.l" -{ - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - p->tmp_len_ += yyleng; - } -} - YY_BREAK -case 39: -YY_RULE_SETUP -#line 147 "ob_expr_parser.l" -{ - POP_STATE(); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - store_expr_str(p->tmp_buf_, p->tmp_len_, p->tmp_start_ptr_ + p->tmp_len_ + 2, yyscanner); - } - return STR_VAL; -} - YY_BREAK -case 40: -YY_RULE_SETUP -#line 156 "ob_expr_parser.l" -{} - YY_BREAK -case 41: -/* rule 41 can match eol */ -YY_RULE_SETUP -#line 157 "ob_expr_parser.l" -{ - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - p->tmp_len_ += yyleng; - } -} - YY_BREAK -case 42: -/* rule 42 can match eol */ -YY_RULE_SETUP -#line 163 "ob_expr_parser.l" -{} - YY_BREAK -case YY_STATE_EOF(sq): -#line 164 "ob_expr_parser.l" -{ return ERROR; } - YY_BREAK -/* dquote */ -case 43: -YY_RULE_SETUP -#line 168 "ob_expr_parser.l" -{ - PUSH_STATE(dq); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - p->tmp_buf_ = (yytext + 1); - p->tmp_start_ptr_ = yytext; - p->tmp_len_ = 0; - } -} - YY_BREAK -case 44: -/* rule 44 can match eol */ -YY_RULE_SETUP -#line 178 "ob_expr_parser.l" -{ - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - p->tmp_len_ += yyleng; - } -} - YY_BREAK -case 45: -YY_RULE_SETUP -#line 185 "ob_expr_parser.l" -{ - POP_STATE(); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (OB_NOTNULL(p)) { - store_expr_str(p->tmp_buf_, p->tmp_len_, p->tmp_start_ptr_ + p->tmp_len_ + 2, yyscanner); - if (p->is_oracle_mode_) { - return NAME_OB; - } - } - return STR_VAL; -} - YY_BREAK -case 46: -YY_RULE_SETUP -#line 197 "ob_expr_parser.l" -{} - YY_BREAK -case 47: -/* rule 47 can match eol */ -YY_RULE_SETUP -#line 198 "ob_expr_parser.l" -{} - YY_BREAK -case 48: -/* rule 48 can match eol */ -YY_RULE_SETUP -#line 199 "ob_expr_parser.l" -{} - YY_BREAK -case YY_STATE_EOF(dq): -#line 200 "ob_expr_parser.l" -{ return ERROR; } - YY_BREAK -/* backtick */ -case 49: -YY_RULE_SETUP -#line 204 "ob_expr_parser.l" -{ - PUSH_STATE(bt); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (NULL != p) { - p->tmp_buf_ = (char *)obproxy_parse_malloc(OBPROXY_MAX_NAME_LENGTH, p->malloc_pool_); - p->tmp_start_ptr_ = yytext; - p->tmp_len_ = 0; - } -} - YY_BREAK -case 50: -YY_RULE_SETUP -#line 214 "ob_expr_parser.l" -{ - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (NULL != p && NULL != p->tmp_buf_ && p->tmp_len_ + 1 < OBPROXY_MAX_NAME_LENGTH) { - p->tmp_buf_[p->tmp_len_++] = '`'; - } -} - YY_BREAK -case 51: -/* rule 51 can match eol */ -YY_RULE_SETUP -#line 221 "ob_expr_parser.l" -{ - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (NULL != p && NULL != p->tmp_buf_ && p->tmp_len_ + yyleng < OBPROXY_MAX_NAME_LENGTH) { - memcpy(p->tmp_buf_ + p->tmp_len_, yytext, yyleng); - p->tmp_len_ += yyleng; - } -} - YY_BREAK -case 52: -YY_RULE_SETUP -#line 229 "ob_expr_parser.l" -{ - POP_STATE(); - ObExprParseResult *p = (ObExprParseResult *)yyextra; - if (NULL != p && NULL != p->tmp_buf_) { - yylval->str.str_ = obproxy_parse_strndup(p->tmp_buf_, p->tmp_len_, p->malloc_pool_); - yylval->str.str_len_ = p->tmp_len_; - yylval->str.end_ptr_ = p->tmp_start_ptr_ + p->tmp_len_ + 2; - } - return NAME_OB; -} - YY_BREAK -case YY_STATE_EOF(bt): -#line 240 "ob_expr_parser.l" -{ - return ERROR; -} - YY_BREAK -case YY_STATE_EOF(INITIAL): -case YY_STATE_EOF(in_c_comment): -#line 244 "ob_expr_parser.l" -{ return END_P; } - YY_BREAK -case 53: -YY_RULE_SETUP -#line 245 "ob_expr_parser.l" -{ return IGNORED_WORD; } - YY_BREAK -case 54: -YY_RULE_SETUP -#line 246 "ob_expr_parser.l" -ECHO; - YY_BREAK -#line 1354 "ob_expr_parser_lex.c" - - case YY_END_OF_BUFFER: - { - /* Amount of text matched not including the EOB char. */ - int yy_amount_of_matched_text = (int) (yy_cp - yyg->yytext_ptr) - 1; - - /* Undo the effects of YY_DO_BEFORE_ACTION. */ - *yy_cp = yyg->yy_hold_char; - YY_RESTORE_YY_MORE_OFFSET - - if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) - { - /* We're scanning a new file or input source. It's - * possible that this happened because the user - * just pointed yyin at a new source and called - * obexprlex(). If so, then we have to assure - * consistency between YY_CURRENT_BUFFER and our - * globals. Here is the right place to do so, because - * this is the first action (other than possibly a - * back-up) that will match for the new input source. - */ - yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; - YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; - YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; - } - - /* Note that here we test for yy_c_buf_p "<=" to the position - * of the first EOB in the buffer, since yy_c_buf_p will - * already have been incremented past the NUL character - * (since all states make transitions on EOB to the - * end-of-buffer state). Contrast this with the test - * in input(). - */ - if ( yyg->yy_c_buf_p <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] ) - { /* This was really a NUL. */ - yy_state_type yy_next_state; - - yyg->yy_c_buf_p = yyg->yytext_ptr + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state( yyscanner ); - - /* Okay, we're now positioned to make the NUL - * transition. We couldn't have - * yy_get_previous_state() go ahead and do it - * for us because it doesn't know how to deal - * with the possibility of jamming (and we don't - * want to build jamming into it because then it - * will run more slowly). - */ - - yy_next_state = yy_try_NUL_trans( yy_current_state , yyscanner); - - yy_bp = yyg->yytext_ptr + YY_MORE_ADJ; - - if ( yy_next_state ) - { - /* Consume the NUL. */ - yy_cp = ++yyg->yy_c_buf_p; - yy_current_state = yy_next_state; - goto yy_match; - } - - else - { - yy_cp = yyg->yy_c_buf_p; - goto yy_find_action; - } - } - - else switch ( yy_get_next_buffer( yyscanner ) ) - { - case EOB_ACT_END_OF_FILE: - { - yyg->yy_did_buffer_switch_on_eof = 0; - - if ( obexprwrap(yyscanner ) ) - { - /* Note: because we've taken care in - * yy_get_next_buffer() to have set up - * yytext, we can now set up - * yy_c_buf_p so that if some total - * hoser (like flex itself) wants to - * call the scanner after we return the - * YY_NULL, it'll still work - another - * YY_NULL will get returned. - */ - yyg->yy_c_buf_p = yyg->yytext_ptr + YY_MORE_ADJ; - - yy_act = YY_STATE_EOF(YY_START); - goto do_action; - } - - else - { - if ( ! yyg->yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; - } - break; - } - - case EOB_ACT_CONTINUE_SCAN: - yyg->yy_c_buf_p = - yyg->yytext_ptr + yy_amount_of_matched_text; - - yy_current_state = yy_get_previous_state( yyscanner ); - - yy_cp = yyg->yy_c_buf_p; - yy_bp = yyg->yytext_ptr + YY_MORE_ADJ; - goto yy_match; - - case EOB_ACT_LAST_MATCH: - yyg->yy_c_buf_p = - &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars]; - - yy_current_state = yy_get_previous_state( yyscanner ); - - yy_cp = yyg->yy_c_buf_p; - yy_bp = yyg->yytext_ptr + YY_MORE_ADJ; - goto yy_find_action; - } - break; - } - - default: - YY_FATAL_ERROR( - "fatal flex scanner internal error--no action found" ); - } /* end of action switch */ - } /* end of scanning one token */ -} /* end of obexprlex */ - -/* yy_get_next_buffer - try to read in a new buffer - * - * Returns a code representing an action: - * EOB_ACT_LAST_MATCH - - * EOB_ACT_CONTINUE_SCAN - continue scanning from current position - * EOB_ACT_END_OF_FILE - end of file - */ -static int yy_get_next_buffer (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - register char *source = yyg->yytext_ptr; - register int number_to_move, i; - int ret_val; - - if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) - YY_FATAL_ERROR( - "fatal flex scanner internal error--end of buffer missed" ); - - if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) - { /* Don't try to fill the buffer, so this is an EOF. */ - if ( yyg->yy_c_buf_p - yyg->yytext_ptr - YY_MORE_ADJ == 1 ) - { - /* We matched a single character, the EOB, so - * treat this as a final EOF. - */ - return EOB_ACT_END_OF_FILE; - } - - else - { - /* We matched some text prior to the EOB, first - * process it. - */ - return EOB_ACT_LAST_MATCH; - } - } - - /* Try to read more data. */ - - /* First move last chars to start of buffer. */ - number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr) - 1; - - for ( i = 0; i < number_to_move; ++i ) - *(dest++) = *(source++); - - if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) - /* don't do the read, it's not guaranteed to return an EOF, - * just force an EOF - */ - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars = 0; - - else - { - int num_to_read = - YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; - - while ( num_to_read <= 0 ) - { /* Not enough room in the buffer - grow it. */ - - /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = YY_CURRENT_BUFFER; - - int yy_c_buf_p_offset = - (int) (yyg->yy_c_buf_p - b->yy_ch_buf); - - if ( b->yy_is_our_buffer ) - { - int new_size = b->yy_buf_size * 2; - - if ( new_size <= 0 ) - b->yy_buf_size += b->yy_buf_size / 8; - else - b->yy_buf_size *= 2; - - b->yy_ch_buf = (char *) - /* Include room in for 2 EOB chars. */ - obexprrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ,yyscanner ); - } - else - /* Can't grow it, we don't own it. */ - b->yy_ch_buf = 0; - - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( - "fatal error - scanner input buffer overflow" ); - - yyg->yy_c_buf_p = &b->yy_ch_buf[yy_c_buf_p_offset]; - - num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - - number_to_move - 1; - - } - - if ( num_to_read > YY_READ_BUF_SIZE ) - num_to_read = YY_READ_BUF_SIZE; - - /* Read in more data. */ - YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), - yyg->yy_n_chars, (size_t) num_to_read ); - - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars; - } - - if ( yyg->yy_n_chars == 0 ) - { - if ( number_to_move == YY_MORE_ADJ ) - { - ret_val = EOB_ACT_END_OF_FILE; - obexprrestart(yyin ,yyscanner); - } - - else - { - ret_val = EOB_ACT_LAST_MATCH; - YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = - YY_BUFFER_EOF_PENDING; - } - } - - else - ret_val = EOB_ACT_CONTINUE_SCAN; - - if ((yy_size_t) (yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { - /* Extend the array by 50%, plus the number we really need. */ - yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) obexprrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ,yyscanner ); - if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); - } - - yyg->yy_n_chars += number_to_move; - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] = YY_END_OF_BUFFER_CHAR; - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; - - yyg->yytext_ptr = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; - - return ret_val; -} - -/* yy_get_previous_state - get the state just before the EOB char was reached */ - - static yy_state_type yy_get_previous_state (yyscan_t yyscanner) -{ - register yy_state_type yy_current_state; - register char *yy_cp; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - yy_current_state = yyg->yy_start; - - for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp ) - { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); - if ( yy_accept[yy_current_state] ) - { - yyg->yy_last_accepting_state = yy_current_state; - yyg->yy_last_accepting_cpos = yy_cp; - } - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - } - - return yy_current_state; -} - -/* yy_try_NUL_trans - try to make a transition on the NUL character - * - * synopsis - * next_state = yy_try_NUL_trans( current_state ); - */ - static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner) -{ - register int yy_is_jam; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */ - register char *yy_cp = yyg->yy_c_buf_p; - - register YY_CHAR yy_c = 1; - if ( yy_accept[yy_current_state] ) - { - yyg->yy_last_accepting_state = yy_current_state; - yyg->yy_last_accepting_cpos = yy_cp; - } - while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) - { - yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 166 ) - yy_c = yy_meta[(unsigned int) yy_c]; - } - yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 165); - - return yy_is_jam ? 0 : yy_current_state; -} - -#ifndef YY_NO_INPUT -#ifdef __cplusplus - static int yyinput (yyscan_t yyscanner) -#else - static int input (yyscan_t yyscanner) -#endif - -{ - int c; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - *yyg->yy_c_buf_p = yyg->yy_hold_char; - - if ( *yyg->yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) - { - /* yy_c_buf_p now points to the character we want to return. - * If this occurs *before* the EOB characters, then it's a - * valid NUL; if not, then we've hit the end of the buffer. - */ - if ( yyg->yy_c_buf_p < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars] ) - /* This was really a NUL. */ - *yyg->yy_c_buf_p = '\0'; - - else - { /* need more input */ - int offset = yyg->yy_c_buf_p - yyg->yytext_ptr; - ++yyg->yy_c_buf_p; - - switch ( yy_get_next_buffer( yyscanner ) ) - { - case EOB_ACT_LAST_MATCH: - /* This happens because yy_g_n_b() - * sees that we've accumulated a - * token and flags that we need to - * try matching the token before - * proceeding. But for input(), - * there's no matching to consider. - * So convert the EOB_ACT_LAST_MATCH - * to EOB_ACT_END_OF_FILE. - */ - - /* Reset buffer status. */ - obexprrestart(yyin ,yyscanner); - - /*FALLTHROUGH*/ - - case EOB_ACT_END_OF_FILE: - { - if ( obexprwrap(yyscanner ) ) - return EOF; - - if ( ! yyg->yy_did_buffer_switch_on_eof ) - YY_NEW_FILE; -#ifdef __cplusplus - return yyinput(yyscanner); -#else - return input(yyscanner); -#endif - } - - case EOB_ACT_CONTINUE_SCAN: - yyg->yy_c_buf_p = yyg->yytext_ptr + offset; - break; - } - } - } - - c = *(unsigned char *) yyg->yy_c_buf_p; /* cast for 8-bit char's */ - *yyg->yy_c_buf_p = '\0'; /* preserve yytext */ - yyg->yy_hold_char = *++yyg->yy_c_buf_p; - - return c; -} -#endif /* ifndef YY_NO_INPUT */ - -/** Immediately switch to a different input stream. - * @param input_file A readable stream. - * @param yyscanner The scanner object. - * @note This function does not reset the start condition to @c INITIAL . - */ - void obexprrestart (FILE * input_file , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - if ( ! YY_CURRENT_BUFFER ){ - obexprensure_buffer_stack (yyscanner); - YY_CURRENT_BUFFER_LVALUE = - obexpr_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); - } - - obexpr_init_buffer(YY_CURRENT_BUFFER,input_file ,yyscanner); - obexpr_load_buffer_state(yyscanner ); -} - -/** Switch to a different input buffer. - * @param new_buffer The new input buffer. - * @param yyscanner The scanner object. - */ - void obexpr_switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - /* TODO. We should be able to replace this entire function body - * with - * obexprpop_buffer_state(); - * obexprpush_buffer_state(new_buffer); - */ - obexprensure_buffer_stack (yyscanner); - if ( YY_CURRENT_BUFFER == new_buffer ) - return; - - if ( YY_CURRENT_BUFFER ) - { - /* Flush out information for old buffer. */ - *yyg->yy_c_buf_p = yyg->yy_hold_char; - YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = yyg->yy_c_buf_p; - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars; - } - - YY_CURRENT_BUFFER_LVALUE = new_buffer; - obexpr_load_buffer_state(yyscanner ); - - /* We don't actually know whether we did this switch during - * EOF (obexprwrap()) processing, but the only time this flag - * is looked at is after obexprwrap() is called, so it's safe - * to go ahead and always set it. - */ - yyg->yy_did_buffer_switch_on_eof = 1; -} - -static void obexpr_load_buffer_state (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; - yyg->yytext_ptr = yyg->yy_c_buf_p = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; - yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; - yyg->yy_hold_char = *yyg->yy_c_buf_p; -} - -/** Allocate and initialize an input buffer state. - * @param file A readable stream. - * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. - * @param yyscanner The scanner object. - * @return the allocated buffer state. - */ - YY_BUFFER_STATE obexpr_create_buffer (FILE * file, int size , yyscan_t yyscanner) -{ - YY_BUFFER_STATE b; - - b = (YY_BUFFER_STATE) obexpralloc(sizeof( struct yy_buffer_state ) ,yyscanner ); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in obexpr_create_buffer()" ); - - b->yy_buf_size = size; - - /* yy_ch_buf has to be 2 characters longer than the size given because - * we need to put in 2 end-of-buffer characters. - */ - b->yy_ch_buf = (char *) obexpralloc(b->yy_buf_size + 2 ,yyscanner ); - if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in obexpr_create_buffer()" ); - - b->yy_is_our_buffer = 1; - - obexpr_init_buffer(b,file ,yyscanner); - - return b; -} - -/** Destroy the buffer. - * @param b a buffer created with obexpr_create_buffer() - * @param yyscanner The scanner object. - */ - void obexpr_delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - if ( ! b ) - return; - - if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ - YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; - - if ( b->yy_is_our_buffer ) - obexprfree((void *) b->yy_ch_buf ,yyscanner ); - - obexprfree((void *) b ,yyscanner ); -} - -#ifndef __cplusplus -extern int isatty (int ); -#endif /* __cplusplus */ - -/* Initializes or reinitializes a buffer. - * This function is sometimes called more than once on the same buffer, - * such as during a obexprrestart() or at EOF. - */ - static void obexpr_init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner) - -{ - int oerrno = errno; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - obexpr_flush_buffer(b ,yyscanner); - - b->yy_input_file = file; - b->yy_fill_buffer = 1; - - /* If b is the current buffer, then obexpr_init_buffer was _probably_ - * called from obexprrestart() or through yy_get_next_buffer. - * In that case, we don't want to reset the lineno or column. - */ - if (b != YY_CURRENT_BUFFER){ - b->yy_bs_lineno = 1; - b->yy_bs_column = 0; - } - - b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; - - errno = oerrno; -} - -/** Discard all buffered characters. On the next scan, YY_INPUT will be called. - * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. - * @param yyscanner The scanner object. - */ - void obexpr_flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - if ( ! b ) - return; - - b->yy_n_chars = 0; - - /* We always need two end-of-buffer characters. The first causes - * a transition to the end-of-buffer state. The second causes - * a jam in that state. - */ - b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; - b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; - - b->yy_buf_pos = &b->yy_ch_buf[0]; - - b->yy_at_bol = 1; - b->yy_buffer_status = YY_BUFFER_NEW; - - if ( b == YY_CURRENT_BUFFER ) - obexpr_load_buffer_state(yyscanner ); -} - -/** Pushes the new state onto the stack. The new state becomes - * the current state. This function will allocate the stack - * if necessary. - * @param new_buffer The new state. - * @param yyscanner The scanner object. - */ -void obexprpush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - if (new_buffer == NULL) - return; - - obexprensure_buffer_stack(yyscanner); - - /* This block is copied from obexpr_switch_to_buffer. */ - if ( YY_CURRENT_BUFFER ) - { - /* Flush out information for old buffer. */ - *yyg->yy_c_buf_p = yyg->yy_hold_char; - YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = yyg->yy_c_buf_p; - YY_CURRENT_BUFFER_LVALUE->yy_n_chars = yyg->yy_n_chars; - } - - /* Only push if top exists. Otherwise, replace top. */ - if (YY_CURRENT_BUFFER) - yyg->yy_buffer_stack_top++; - YY_CURRENT_BUFFER_LVALUE = new_buffer; - - /* copied from obexpr_switch_to_buffer. */ - obexpr_load_buffer_state(yyscanner ); - yyg->yy_did_buffer_switch_on_eof = 1; -} - -/** Removes and deletes the top of the stack, if present. - * The next element becomes the new top. - * @param yyscanner The scanner object. - */ -void obexprpop_buffer_state (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - if (!YY_CURRENT_BUFFER) - return; - - obexpr_delete_buffer(YY_CURRENT_BUFFER ,yyscanner); - YY_CURRENT_BUFFER_LVALUE = NULL; - if (yyg->yy_buffer_stack_top > 0) - --yyg->yy_buffer_stack_top; - - if (YY_CURRENT_BUFFER) { - obexpr_load_buffer_state(yyscanner ); - yyg->yy_did_buffer_switch_on_eof = 1; - } -} - -/* Allocates the stack if it does not exist. - * Guarantees space for at least one push. - */ -static void obexprensure_buffer_stack (yyscan_t yyscanner) -{ - int num_to_alloc; - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - if (!yyg->yy_buffer_stack) { - - /* First allocation is just for 2 elements, since we don't know if this - * scanner will even need a stack. We use 2 instead of 1 to avoid an - * immediate realloc on the next call. - */ - num_to_alloc = 1; - yyg->yy_buffer_stack = (struct yy_buffer_state**)obexpralloc - (num_to_alloc * sizeof(struct yy_buffer_state*) - , yyscanner); - if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in obexprensure_buffer_stack()" ); - - memset(yyg->yy_buffer_stack, 0, num_to_alloc * sizeof(struct yy_buffer_state*)); - - yyg->yy_buffer_stack_max = num_to_alloc; - yyg->yy_buffer_stack_top = 0; - return; - } - - if (yyg->yy_buffer_stack_top >= (yyg->yy_buffer_stack_max) - 1){ - - /* Increase the buffer to prepare for a possible push. */ - int grow_size = 8 /* arbitrary grow size */; - - num_to_alloc = yyg->yy_buffer_stack_max + grow_size; - yyg->yy_buffer_stack = (struct yy_buffer_state**)obexprrealloc - (yyg->yy_buffer_stack, - num_to_alloc * sizeof(struct yy_buffer_state*) - , yyscanner); - if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in obexprensure_buffer_stack()" ); - - /* zero only the new slots.*/ - memset(yyg->yy_buffer_stack + yyg->yy_buffer_stack_max, 0, grow_size * sizeof(struct yy_buffer_state*)); - yyg->yy_buffer_stack_max = num_to_alloc; - } -} - -/** Setup the input buffer state to scan directly from a user-specified character buffer. - * @param base the character buffer - * @param size the size in bytes of the character buffer - * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. - */ -YY_BUFFER_STATE obexpr_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) -{ - YY_BUFFER_STATE b; - - if ( size < 2 || - base[size-2] != YY_END_OF_BUFFER_CHAR || - base[size-1] != YY_END_OF_BUFFER_CHAR ) - /* They forgot to leave room for the EOB's. */ - return 0; - - b = (YY_BUFFER_STATE) obexpralloc(sizeof( struct yy_buffer_state ) ,yyscanner ); - if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in obexpr_scan_buffer()" ); - - b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ - b->yy_buf_pos = b->yy_ch_buf = base; - b->yy_is_our_buffer = 0; - b->yy_input_file = 0; - b->yy_n_chars = b->yy_buf_size; - b->yy_is_interactive = 0; - b->yy_at_bol = 1; - b->yy_fill_buffer = 0; - b->yy_buffer_status = YY_BUFFER_NEW; - - obexpr_switch_to_buffer(b ,yyscanner ); - - return b; -} - -/** Setup the input buffer state to scan a string. The next call to obexprlex() will - * scan from a @e copy of @a str. - * @param yystr a NUL-terminated string to scan - * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. - * @note If you want to scan bytes that may contain NUL values, then use - * obexpr_scan_bytes() instead. - */ -YY_BUFFER_STATE obexpr_scan_string (yyconst char * yystr , yyscan_t yyscanner) -{ - - return obexpr_scan_bytes(yystr,strlen(yystr) ,yyscanner); -} - -/** Setup the input buffer state to scan the given bytes. The next call to obexprlex() will - * scan from a @e copy of @a bytes. - * @param bytes the byte buffer to scan - * @param len the number of bytes in the buffer pointed to by @a bytes. - * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. - */ -YY_BUFFER_STATE obexpr_scan_bytes (yyconst char * yybytes, int _yybytes_len , yyscan_t yyscanner) -{ - YY_BUFFER_STATE b; - char *buf; - yy_size_t n; - - /* Get memory for full buffer, including space for trailing EOB's. */ - n = _yybytes_len + 2; - buf = (char *) obexpralloc(n ,yyscanner ); - if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in obexpr_scan_bytes()" ); - - memcpy(buf, yybytes, _yybytes_len); - - buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; - - b = obexpr_scan_buffer(buf,n ,yyscanner); - if ( ! b ) - YY_FATAL_ERROR( "bad buffer in obexpr_scan_bytes()" ); - - /* It's okay to grow etc. this buffer, and we should throw it - * away when we're done. - */ - b->yy_is_our_buffer = 1; - - return b; -} - - static void yy_push_state (int new_state , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - if ( yyg->yy_start_stack_ptr >= yyg->yy_start_stack_depth ) - { - yy_size_t new_size; - - yyg->yy_start_stack_depth += YY_START_STACK_INCR; - new_size = yyg->yy_start_stack_depth * sizeof( int ); - - if ( ! yyg->yy_start_stack ) - yyg->yy_start_stack = (int *) obexpralloc(new_size ,yyscanner ); - - else - yyg->yy_start_stack = (int *) obexprrealloc((void *) yyg->yy_start_stack,new_size ,yyscanner ); - - if ( ! yyg->yy_start_stack ) - YY_FATAL_ERROR( "out of memory expanding start-condition stack" ); - } - - yyg->yy_start_stack[yyg->yy_start_stack_ptr++] = YY_START; - - BEGIN(new_state); -} - - static void yy_pop_state (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - if ( --yyg->yy_start_stack_ptr < 0 ) - YY_FATAL_ERROR( "start-condition stack underflow" ); - - BEGIN(yyg->yy_start_stack[yyg->yy_start_stack_ptr]); -} - - -#ifndef YY_EXIT_FAILURE -#define YY_EXIT_FAILURE 2 -#endif - -static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner) -{ - (void) fprintf( stderr, "%s\n", msg ); - exit( YY_EXIT_FAILURE ); -} - -/* Redefine yyless() so it works in section 3 code. */ - -#undef yyless -#define yyless(n) \ - do \ - { \ - /* Undo effects of setting up yytext. */ \ - int yyless_macro_arg = (n); \ - YY_LESS_LINENO(yyless_macro_arg);\ - yytext[yyleng] = yyg->yy_hold_char; \ - yyg->yy_c_buf_p = yytext + yyless_macro_arg; \ - yyg->yy_hold_char = *yyg->yy_c_buf_p; \ - *yyg->yy_c_buf_p = '\0'; \ - yyleng = yyless_macro_arg; \ - } \ - while ( 0 ) - -/* Accessor methods (get/set functions) to struct members. */ - -/** Get the user-defined data for this scanner. - * @param yyscanner The scanner object. - */ -YY_EXTRA_TYPE obexprget_extra (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yyextra; -} - -/** Get the current line number. - * @param yyscanner The scanner object. - */ -int obexprget_lineno (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - if (! YY_CURRENT_BUFFER) - return 0; - - return yylineno; -} - -/** Get the current column number. - * @param yyscanner The scanner object. - */ -int obexprget_column (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - if (! YY_CURRENT_BUFFER) - return 0; - - return yycolumn; -} - -/** Get the input stream. - * @param yyscanner The scanner object. - */ -FILE *obexprget_in (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yyin; -} - -/** Get the output stream. - * @param yyscanner The scanner object. - */ -FILE *obexprget_out (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yyout; -} - -/** Get the length of the current token. - * @param yyscanner The scanner object. - */ -int obexprget_leng (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yyleng; -} - -/** Get the current token. - * @param yyscanner The scanner object. - */ - -char *obexprget_text (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yytext; -} - -/** Set the user-defined data. This data is never touched by the scanner. - * @param user_defined The data to be associated with this scanner. - * @param yyscanner The scanner object. - */ -void obexprset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyextra = user_defined ; -} - -/** Set the current line number. - * @param line_number - * @param yyscanner The scanner object. - */ -void obexprset_lineno (int line_number , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - /* lineno is only valid if an input buffer exists. */ - if (! YY_CURRENT_BUFFER ) - yy_fatal_error( "obexprset_lineno called with no buffer" , yyscanner); - - yylineno = line_number; -} - -/** Set the current column. - * @param line_number - * @param yyscanner The scanner object. - */ -void obexprset_column (int column_no , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - /* column is only valid if an input buffer exists. */ - if (! YY_CURRENT_BUFFER ) - yy_fatal_error( "obexprset_column called with no buffer" , yyscanner); - - yycolumn = column_no; -} - -/** Set the input stream. This does not discard the current - * input buffer. - * @param in_str A readable stream. - * @param yyscanner The scanner object. - * @see obexpr_switch_to_buffer - */ -void obexprset_in (FILE * in_str , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyin = in_str ; -} - -void obexprset_out (FILE * out_str , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yyout = out_str ; -} - -int obexprget_debug (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yy_flex_debug; -} - -void obexprset_debug (int bdebug , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yy_flex_debug = bdebug ; -} - -/* Accessor methods for yylval and yylloc */ - -YYSTYPE * obexprget_lval (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yylval; -} - -void obexprset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yylval = yylval_param; -} - -YYLTYPE *obexprget_lloc (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - return yylloc; -} - -void obexprset_lloc (YYLTYPE * yylloc_param , yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yylloc = yylloc_param; -} - -/* User-visible API */ - -/* obexprlex_init is special because it creates the scanner itself, so it is - * the ONLY reentrant function that doesn't take the scanner as the last argument. - * That's why we explicitly handle the declaration, instead of using our macros. - */ - -int obexprlex_init(yyscan_t* ptr_yy_globals) - -{ - if (ptr_yy_globals == NULL){ - errno = EINVAL; - return 1; - } - - *ptr_yy_globals = (yyscan_t) obexpralloc ( sizeof( struct yyguts_t ), NULL ); - - if (*ptr_yy_globals == NULL){ - errno = ENOMEM; - return 1; - } - - /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */ - memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t)); - - return yy_init_globals ( *ptr_yy_globals ); -} - -/* obexprlex_init_extra has the same functionality as obexprlex_init, but follows the - * convention of taking the scanner as the last argument. Note however, that - * this is a *pointer* to a scanner, as it will be allocated by this call (and - * is the reason, too, why this function also must handle its own declaration). - * The user defined value in the first argument will be available to obexpralloc in - * the yyextra field. - */ - -int obexprlex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ) - -{ - struct yyguts_t dummy_yyguts; - - obexprset_extra (yy_user_defined, &dummy_yyguts); - - if (ptr_yy_globals == NULL){ - errno = EINVAL; - return 1; - } - - *ptr_yy_globals = (yyscan_t) obexpralloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); - - if (*ptr_yy_globals == NULL){ - errno = ENOMEM; - return 1; - } - - /* By setting to 0xAA, we expose bugs in - yy_init_globals. Leave at 0x00 for releases. */ - memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t)); - - obexprset_extra (yy_user_defined, *ptr_yy_globals); - - return yy_init_globals ( *ptr_yy_globals ); -} - -static int yy_init_globals (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - /* Initialization is the same as for the non-reentrant scanner. - * This function is called from obexprlex_destroy(), so don't allocate here. - */ - - yyg->yy_buffer_stack = 0; - yyg->yy_buffer_stack_top = 0; - yyg->yy_buffer_stack_max = 0; - yyg->yy_c_buf_p = (char *) 0; - yyg->yy_init = 0; - yyg->yy_start = 0; - - yyg->yy_start_stack_ptr = 0; - yyg->yy_start_stack_depth = 0; - yyg->yy_start_stack = NULL; - -/* Defined in main.c */ -#ifdef YY_STDINIT - yyin = stdin; - yyout = stdout; -#else - yyin = (FILE *) 0; - yyout = (FILE *) 0; -#endif - - /* For future reference: Set errno on error, since we are called by - * obexprlex_init() - */ - return 0; -} - -/* obexprlex_destroy is for both reentrant and non-reentrant scanners. */ -int obexprlex_destroy (yyscan_t yyscanner) -{ - struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - - /* Pop the buffer stack, destroying each element. */ - while(YY_CURRENT_BUFFER){ - obexpr_delete_buffer(YY_CURRENT_BUFFER ,yyscanner ); - YY_CURRENT_BUFFER_LVALUE = NULL; - obexprpop_buffer_state(yyscanner); - } - - /* Destroy the stack itself. */ - obexprfree(yyg->yy_buffer_stack ,yyscanner); - yyg->yy_buffer_stack = NULL; - - /* Destroy the start condition stack. */ - obexprfree(yyg->yy_start_stack ,yyscanner ); - yyg->yy_start_stack = NULL; - - /* Reset the globals. This is important in a non-reentrant scanner so the next time - * obexprlex() is called, initialization will occur. */ - yy_init_globals( yyscanner); - - /* Destroy the main struct (reentrant only). */ - obexprfree ( yyscanner , yyscanner ); - yyscanner = NULL; - return 0; -} - -/* - * Internal utility routines. - */ - -#ifndef yytext_ptr -static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner) -{ - register int i; - for ( i = 0; i < n; ++i ) - s1[i] = s2[i]; -} -#endif - -#ifdef YY_NEED_STRLEN -static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner) -{ - register int n; - for ( n = 0; s[n]; ++n ) - ; - - return n; -} -#endif - -#define YYTABLES_NAME "yytables" - -#line 246 "ob_expr_parser.l" - - - -inline void *obexpralloc(size_t bytes,void *yyscanner) -{ - void *ptr_ret = NULL; - ObExprParseResult *p = obexprget_extra(yyscanner); - if (OB_ISNULL(p)) { - // print err into msg buffer later - } else { - ptr_ret = obproxy_parse_malloc(bytes, p->malloc_pool_); - } - return ptr_ret; -} - -inline void *obexprrealloc(void *ptr,size_t bytes,void *yyscanner) -{ - void *ptr_ret = NULL; - ObExprParseResult *p = obexprget_extra(yyscanner); - if (OB_ISNULL(p)) { - // print err into msg buffer later - } else { - ptr_ret = obproxy_parse_realloc(ptr, bytes, p->malloc_pool_); - } - return ptr_ret; - -} - -inline void obexprfree(void *ptr,void *yyscanner) -{ - // Do nothing -- we leave it to the garbage collector. - obproxy_parse_free(ptr); -} - -inline void store_expr_str(char* str, int64_t str_len, char*end_ptr, void *yyscanner) -{ - YYSTYPE *lval = obexprget_lval(yyscanner); - if (OB_ISNULL(lval)) { - // do nothing - } else { - lval->str.str_ = str; - lval->str.end_ptr_ = end_ptr; - lval->str.str_len_ = str_len; - } -} - -inline void store_pos_place_holder(char *str, void *yyscanner) -{ - YYSTYPE *lval = obexprget_lval(yyscanner); - if (OB_ISNULL(lval)) { - // do nothing - } else { - errno = 0; - lval->num = strtoll(str, NULL, 10); - if (0 != errno) { - lval->num = 0; - } - } -} - -/* A Bison parser, made by GNU Bison 2.4.1. */ - -/* Skeleton interface for Bison's Yacc-like parsers in C - - Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 - Free Software Foundation, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . */ - -/* As a special exception, you may create a larger work that contains - part or all of the Bison parser skeleton and distribute that work - under terms of your choice, so long as that work isn't itself a - parser generator using the skeleton or a modified version thereof - as a parser skeleton. Alternatively, if you modify or redistribute - the parser skeleton itself, you may (at your option) remove this - special exception, which will cause the skeleton and the resulting - Bison output files to be licensed under the GNU General Public - License without this special exception. - - This special exception was added by the Free Software Foundation in - version 2.2 of Bison. */ - - -#ifndef YY_OBEXPR_OB_EXPR_PARSER_TAB_H_INCLUDED -# define YY_OBEXPR_OB_EXPR_PARSER_TAB_H_INCLUDED -/* Debug traces. */ -#ifndef OBEXPRDEBUG -# if defined YYDEBUG -#if YYDEBUG -# define OBEXPRDEBUG 1 -# else -# define OBEXPRDEBUG 0 -# endif -# else /* ! defined YYDEBUG */ -# define OBEXPRDEBUG 0 -# endif /* ! defined YYDEBUG */ -#endif /* ! defined OBEXPRDEBUG */ -#if OBEXPRDEBUG -extern int obexprdebug; -#endif -/* Tokens. */ -#ifndef OBEXPRTOKENTYPE -# define OBEXPRTOKENTYPE - /* Put the tokens into the symbol table, so that GDB and other debuggers - know about them. */ - enum obexprtokentype { - DUMMY_SELECT_CLAUSE = 258, - DUMMY_INSERT_CLAUSE = 259, - WHERE = 260, - AS = 261, - VALUES = 262, - SET = 263, - END_WHERE = 264, - JOIN = 265, - AND_OP = 266, - OR_OP = 267, - IN = 268, - ON = 269, - BETWEEN = 270, - ROWID = 271, - COMP_EQ = 272, - COMP_NSEQ = 273, - COMP_GE = 274, - COMP_GT = 275, - COMP_LE = 276, - COMP_LT = 277, - COMP_NE = 278, - PLACE_HOLDER = 279, - END_P = 280, - ERROR = 281, - IGNORED_WORD = 282, - NAME_OB = 283, - STR_VAL = 284, - INT_VAL = 285, - POS_PLACE_HOLDER = 286 - }; -#endif - - - -#if ! defined OBEXPRSTYPE && ! defined OBEXPRSTYPE_IS_DECLARED -typedef union OBEXPRSTYPE -{ - - - int64_t num; - ObProxyParseString str; - ObProxyFunctionType func; - ObProxyOperatorType operator; - ObProxyTokenNode *node; - ObProxyTokenList *list; - ObProxyRelationExpr *relation; - - - -} OBEXPRSTYPE; -# define OBEXPRSTYPE_IS_TRIVIAL 1 -# define obexprstype OBEXPRSTYPE /* obsolescent; will be withdrawn */ -# define OBEXPRSTYPE_IS_DECLARED 1 -#endif - - - -#if ! defined OBEXPRLTYPE && ! defined OBEXPRLTYPE_IS_DECLARED -typedef struct OBEXPRLTYPE -{ - int first_line; - int first_column; - int last_line; - int last_column; -} OBEXPRLTYPE; -# define obexprltype OBEXPRLTYPE /* obsolescent; will be withdrawn */ -# define OBEXPRLTYPE_IS_DECLARED 1 -# define OBEXPRLTYPE_IS_TRIVIAL 1 -#endif - - - -#endif - diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.h b/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.h deleted file mode 100644 index ba0e627b48da5f03e2518725cf2f4b403d62bcef..0000000000000000000000000000000000000000 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser_lex.h +++ /dev/null @@ -1,349 +0,0 @@ -#ifndef obexprHEADER_H -#define obexprHEADER_H 1 -#define obexprIN_HEADER 1 - -#line 6 "ob_expr_parser_lex.h" - -#line 8 "ob_expr_parser_lex.h" - -#define YY_INT_ALIGNED short int - -/* A lexical scanner generated by flex */ - -#define FLEX_SCANNER -#define YY_FLEX_MAJOR_VERSION 2 -#define YY_FLEX_MINOR_VERSION 5 -#define YY_FLEX_SUBMINOR_VERSION 35 -#if YY_FLEX_SUBMINOR_VERSION > 0 -#define FLEX_BETA -#endif - -/* First, we deal with platform-specific or compiler-specific issues. */ - -/* begin standard C headers. */ -#include -#include -#include -#include - -/* end standard C headers. */ - -/* flex integer type definitions */ - -#ifndef FLEXINT_H -#define FLEXINT_H - -/* C99 systems have . Non-C99 systems may or may not. */ - -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - -/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. - */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS 1 -#endif - -#include -typedef int8_t flex_int8_t; -typedef uint8_t flex_uint8_t; -typedef int16_t flex_int16_t; -typedef uint16_t flex_uint16_t; -typedef int32_t flex_int32_t; -typedef uint32_t flex_uint32_t; -#else -typedef signed char flex_int8_t; -typedef short int flex_int16_t; -typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; -typedef unsigned short int flex_uint16_t; -typedef unsigned int flex_uint32_t; -#endif /* ! C99 */ - -/* Limits of integral types. */ -#ifndef INT8_MIN -#define INT8_MIN (-128) -#endif -#ifndef INT16_MIN -#define INT16_MIN (-32767-1) -#endif -#ifndef INT32_MIN -#define INT32_MIN (-2147483647-1) -#endif -#ifndef INT8_MAX -#define INT8_MAX (127) -#endif -#ifndef INT16_MAX -#define INT16_MAX (32767) -#endif -#ifndef INT32_MAX -#define INT32_MAX (2147483647) -#endif -#ifndef UINT8_MAX -#define UINT8_MAX (255U) -#endif -#ifndef UINT16_MAX -#define UINT16_MAX (65535U) -#endif -#ifndef UINT32_MAX -#define UINT32_MAX (4294967295U) -#endif - -#endif /* ! FLEXINT_H */ - -#ifdef __cplusplus - -/* The "const" storage-class-modifier is valid. */ -#define YY_USE_CONST - -#else /* ! __cplusplus */ - -/* C99 requires __STDC__ to be defined as 1. */ -#if defined (__STDC__) - -#define YY_USE_CONST - -#endif /* defined (__STDC__) */ -#endif /* ! __cplusplus */ - -#ifdef YY_USE_CONST -#define yyconst const -#else -#define yyconst -#endif - -/* An opaque pointer. */ -#ifndef YY_TYPEDEF_YY_SCANNER_T -#define YY_TYPEDEF_YY_SCANNER_T -typedef void* yyscan_t; -#endif - -/* For convenience, these vars (plus the bison vars far below) - are macros in the reentrant scanner. */ -#define yyin yyg->yyin_r -#define yyout yyg->yyout_r -#define yyextra yyg->yyextra_r -#define yyleng yyg->yyleng_r -#define yytext yyg->yytext_r -#define yylineno (YY_CURRENT_BUFFER_LVALUE->yy_bs_lineno) -#define yycolumn (YY_CURRENT_BUFFER_LVALUE->yy_bs_column) -#define yy_flex_debug yyg->yy_flex_debug_r - -/* Size of default input buffer. */ -#ifndef YY_BUF_SIZE -#define YY_BUF_SIZE 16384 -#endif - -#ifndef YY_TYPEDEF_YY_BUFFER_STATE -#define YY_TYPEDEF_YY_BUFFER_STATE -typedef struct yy_buffer_state *YY_BUFFER_STATE; -#endif - -#ifndef YY_TYPEDEF_YY_SIZE_T -#define YY_TYPEDEF_YY_SIZE_T -typedef size_t yy_size_t; -#endif - -#ifndef YY_STRUCT_YY_BUFFER_STATE -#define YY_STRUCT_YY_BUFFER_STATE -struct yy_buffer_state - { - FILE *yy_input_file; - - char *yy_ch_buf; /* input buffer */ - char *yy_buf_pos; /* current position in input buffer */ - - /* Size of input buffer in bytes, not including room for EOB - * characters. - */ - yy_size_t yy_buf_size; - - /* Number of characters read into yy_ch_buf, not including EOB - * characters. - */ - int yy_n_chars; - - /* Whether we "own" the buffer - i.e., we know we created it, - * and can realloc() it to grow it, and should free() it to - * delete it. - */ - int yy_is_our_buffer; - - /* Whether this is an "interactive" input source; if so, and - * if we're using stdio for input, then we want to use getc() - * instead of fread(), to make sure we stop fetching input after - * each newline. - */ - int yy_is_interactive; - - /* Whether we're considered to be at the beginning of a line. - * If so, '^' rules will be active on the next match, otherwise - * not. - */ - int yy_at_bol; - - int yy_bs_lineno; /**< The line count. */ - int yy_bs_column; /**< The column count. */ - - /* Whether to try to fill the input buffer when we reach the - * end of it. - */ - int yy_fill_buffer; - - int yy_buffer_status; - - }; -#endif /* !YY_STRUCT_YY_BUFFER_STATE */ - -void obexprrestart (FILE *input_file ,yyscan_t yyscanner ); -void obexpr_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); -void obexpr_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); -void obexpr_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); -void obexprpush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); -void obexprpop_buffer_state (yyscan_t yyscanner ); - -YY_BUFFER_STATE obexpr_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); -YY_BUFFER_STATE obexpr_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); - -void *obexpralloc (yy_size_t ,yyscan_t yyscanner ); -void *obexprrealloc (void *,yy_size_t ,yyscan_t yyscanner ); -void obexprfree (void * ,yyscan_t yyscanner ); - -/* Begin user sect3 */ - -#define obexprwrap(n) 1 -#define YY_SKIP_YYWRAP - -#define yytext_ptr yytext_r - -#ifdef YY_HEADER_EXPORT_START_CONDITIONS -#define INITIAL 0 -#define in_c_comment 1 -#define sq 2 -#define dq 3 -#define bt 4 - -#endif - -#ifndef YY_NO_UNISTD_H -/* Special case for "unistd.h", since it is non-ANSI. We include it way - * down here because we want the user's section 1 to have been scanned first. - * The user has a chance to override it with an option. - */ -#include -#endif - -#ifndef YY_EXTRA_TYPE -#define YY_EXTRA_TYPE void * -#endif - -int obexprlex_init (yyscan_t* scanner); - -int obexprlex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); - -/* Accessor methods to globals. - These are made visible to non-reentrant scanners for convenience. */ - -int obexprlex_destroy (yyscan_t yyscanner ); - -int obexprget_debug (yyscan_t yyscanner ); - -void obexprset_debug (int debug_flag ,yyscan_t yyscanner ); - -YY_EXTRA_TYPE obexprget_extra (yyscan_t yyscanner ); - -void obexprset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); - -FILE *obexprget_in (yyscan_t yyscanner ); - -void obexprset_in (FILE * in_str ,yyscan_t yyscanner ); - -FILE *obexprget_out (yyscan_t yyscanner ); - -void obexprset_out (FILE * out_str ,yyscan_t yyscanner ); - -int obexprget_leng (yyscan_t yyscanner ); - -char *obexprget_text (yyscan_t yyscanner ); - -int obexprget_lineno (yyscan_t yyscanner ); - -void obexprset_lineno (int line_number ,yyscan_t yyscanner ); - -OBEXPRSTYPE * obexprget_lval (yyscan_t yyscanner ); - -void obexprset_lval (OBEXPRSTYPE * yylval_param ,yyscan_t yyscanner ); - - OBEXPRLTYPE *obexprget_lloc (yyscan_t yyscanner ); - - void obexprset_lloc (OBEXPRLTYPE * yylloc_param ,yyscan_t yyscanner ); - -/* Macros after this point can all be overridden by user definitions in - * section 1. - */ - -#ifndef YY_SKIP_YYWRAP -#ifdef __cplusplus -extern "C" int obexprwrap (yyscan_t yyscanner ); -#else -extern int obexprwrap (yyscan_t yyscanner ); -#endif -#endif - -#ifndef yytext_ptr -static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); -#endif - -#ifdef YY_NEED_STRLEN -static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); -#endif - -#ifndef YY_NO_INPUT - -#endif - -/* Amount of stuff to slurp up with each read. */ -#ifndef YY_READ_BUF_SIZE -#define YY_READ_BUF_SIZE 8192 -#endif - -/* Number of entries by which start-condition stack grows. */ -#ifndef YY_START_STACK_INCR -#define YY_START_STACK_INCR 25 -#endif - -/* Default declaration of generated scanner - a define so the user can - * easily add parameters. - */ -#ifndef YY_DECL -#define YY_DECL_IS_OURS 1 - -extern int obexprlex \ - (OBEXPRSTYPE * yylval_param,OBEXPRLTYPE * yylloc_param ,yyscan_t yyscanner); - -#define YY_DECL int obexprlex \ - (OBEXPRSTYPE * yylval_param, OBEXPRLTYPE * yylloc_param , yyscan_t yyscanner) -#endif /* !YY_DECL */ - -/* yy_get_previous_state - get the state just before the EOB char was reached */ - -#undef YY_NEW_FILE -#undef YY_FLUSH_BUFFER -#undef yy_set_bol -#undef yy_new_buffer -#undef yy_set_interactive -#undef YY_DO_BEFORE_ACTION - -#ifdef YY_DECL_IS_OURS -#undef YY_DECL_IS_OURS -#undef YY_DECL -#endif - -#line 246 "ob_expr_parser.l" - - -#line 348 "ob_expr_parser_lex.h" -#undef obexprIN_HEADER -#endif /* obexprHEADER_H */ diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser_tab.c b/src/obproxy/opsql/expr_parser/ob_expr_parser_tab.c deleted file mode 100644 index 978cec1fec3a58a792e7fb7f5c3b191c7d618ee5..0000000000000000000000000000000000000000 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser_tab.c +++ /dev/null @@ -1,2529 +0,0 @@ - -/* A Bison parser, made by GNU Bison 2.4.1. */ - -/* Skeleton implementation for Bison's Yacc-like parsers in C - - Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 - Free Software Foundation, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . */ - -/* As a special exception, you may create a larger work that contains - part or all of the Bison parser skeleton and distribute that work - under terms of your choice, so long as that work isn't itself a - parser generator using the skeleton or a modified version thereof - as a parser skeleton. Alternatively, if you modify or redistribute - the parser skeleton itself, you may (at your option) remove this - special exception, which will cause the skeleton and the resulting - Bison output files to be licensed under the GNU General Public - License without this special exception. - - This special exception was added by the Free Software Foundation in - version 2.2 of Bison. */ - -/* C LALR(1) parser skeleton written by Richard Stallman, by - simplifying the original so-called "semantic" parser. */ - -/* All symbols defined below should begin with yy or YY, to avoid - infringing on user name space. This should be done even for local - variables, as they might otherwise be expanded by user macros. - There are some unavoidable exceptions within include files to - define necessary library symbols; they are noted "INFRINGES ON - USER NAME SPACE" below. */ - -/* Identify Bison output. */ -#define YYBISON 1 - -/* Bison version. */ -#define YYBISON_VERSION "2.4.1" - -/* Skeleton name. */ -#define YYSKELETON_NAME "yacc.c" - -/* Pure parsers. */ -#define YYPURE 1 - -/* Push parsers. */ -#define YYPUSH 0 - -/* Pull parsers. */ -#define YYPULL 1 - -/* Using locations. */ -#define YYLSP_NEEDED 1 - -/* Substitute the variable and function names. */ -#define YYSTYPE OBEXPRSTYPE -#define YYLTYPE OBEXPRLTYPE -#define yyparse obexprparse -#define yylex obexprlex -#define yyerror obexprerror -#define yylval obexprlval -#define yychar obexprchar -#define yydebug obexprdebug -#define yynerrs obexprnerrs -#define yylloc obexprlloc - -/* Copy the first part of user declarations. */ - - -#include -#include "opsql/ob_proxy_parse_define.h" -#include "opsql/expr_parser/ob_expr_parse_result.h" - - - -/* Enabling traces. */ -#ifndef YYDEBUG -# define YYDEBUG 0 -#endif - -/* Enabling verbose error messages. */ -#ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 -#else -# define YYERROR_VERBOSE 0 -#endif - -/* Enabling the token table. */ -#ifndef YYTOKEN_TABLE -# define YYTOKEN_TABLE 0 -#endif - - -#ifndef YY_OBEXPR_OB_EXPR_PARSER_TAB_H_INCLUDED -# define YY_OBEXPR_OB_EXPR_PARSER_TAB_H_INCLUDED -/* Debug traces. */ -#ifndef OBEXPRDEBUG -# if defined YYDEBUG -#if YYDEBUG -# define OBEXPRDEBUG 1 -# else -# define OBEXPRDEBUG 0 -# endif -# else /* ! defined YYDEBUG */ -# define OBEXPRDEBUG 0 -# endif /* ! defined YYDEBUG */ -#endif /* ! defined OBEXPRDEBUG */ -#if OBEXPRDEBUG -extern int obexprdebug; -#endif -/* Tokens. */ -#ifndef OBEXPRTOKENTYPE -# define OBEXPRTOKENTYPE - /* Put the tokens into the symbol table, so that GDB and other debuggers - know about them. */ - enum obexprtokentype { - DUMMY_SELECT_CLAUSE = 258, - DUMMY_INSERT_CLAUSE = 259, - WHERE = 260, - AS = 261, - VALUES = 262, - SET = 263, - END_WHERE = 264, - JOIN = 265, - AND_OP = 266, - OR_OP = 267, - IN = 268, - ON = 269, - BETWEEN = 270, - ROWID = 271, - COMP_EQ = 272, - COMP_NSEQ = 273, - COMP_GE = 274, - COMP_GT = 275, - COMP_LE = 276, - COMP_LT = 277, - COMP_NE = 278, - PLACE_HOLDER = 279, - END_P = 280, - ERROR = 281, - IGNORED_WORD = 282, - NAME_OB = 283, - STR_VAL = 284, - INT_VAL = 285, - POS_PLACE_HOLDER = 286 - }; -#endif - - - -#if ! defined OBEXPRSTYPE && ! defined OBEXPRSTYPE_IS_DECLARED -typedef union OBEXPRSTYPE -{ - - - int64_t num; - ObProxyParseString str; - ObProxyFunctionType func; - ObProxyOperatorType operator; - ObProxyTokenNode *node; - ObProxyTokenList *list; - ObProxyRelationExpr *relation; - - - -} OBEXPRSTYPE; -# define OBEXPRSTYPE_IS_TRIVIAL 1 -# define obexprstype OBEXPRSTYPE /* obsolescent; will be withdrawn */ -# define OBEXPRSTYPE_IS_DECLARED 1 -#endif - -#if ! defined OBEXPRLTYPE && ! defined OBEXPRLTYPE_IS_DECLARED -typedef struct OBEXPRLTYPE -{ - int first_line; - int first_column; - int last_line; - int last_column; -} OBEXPRLTYPE; -# define obexprltype OBEXPRLTYPE /* obsolescent; will be withdrawn */ -# define OBEXPRLTYPE_IS_DECLARED 1 -# define OBEXPRLTYPE_IS_TRIVIAL 1 -#endif - - -#endif -/* Copy the second part of user declarations. */ - - -#include "ob_expr_parser_lex.h" -#define YYLEX_PARAM result->yyscan_info_ -extern void yyerror(YYLTYPE* yylloc, ObExprParseResult* p, char* s,...); -extern void *obproxy_parse_malloc(const size_t nbyte, void *malloc_pool); - -static inline bool is_equal(ObProxyParseString *l, ObProxyParseString *r) -{ - return NULL != l && NULL != r && l->str_len_ == r->str_len_ - && 0 == strncasecmp(l->str_, r->str_, l->str_len_); -} - -static inline void add_token(ObProxyTokenList *list, ObExprParseResult *result, ObProxyTokenNode *node) -{ - UNUSED(result); // use for perf later - if (OB_ISNULL(list) || OB_ISNULL(node)) { - // do nothing - } else { - if (TOKEN_COLUMN == node->type_) { - list->column_node_ = node; - } - if (NULL != list->tail_) { - list->tail_->next_ = node; - list->tail_ = node; - } - } -} - -// expr in (xxx,xxx,xxx) -static inline void add_token_list(ObProxyTokenList *list, ObProxyTokenList *next_list) -{ - if (OB_ISNULL(list) || OB_ISNULL(next_list)) { - } else if (NULL != list->tail_ - && NULL != next_list->head_ - && (TOKEN_INT_VAL == list->tail_->type_ || TOKEN_STR_VAL == list->tail_->type_) - && (TOKEN_INT_VAL == next_list->head_->type_ || TOKEN_STR_VAL == next_list->head_->type_)) { - list->tail_->next_ = next_list->head_; - list->tail_ = next_list->head_; - list->tail_->next_ = NULL; - } -} - -static inline ObProxyFunctionType get_reverse_func(ObProxyFunctionType type) -{ - ObProxyFunctionType ret_type = type; - switch (type) { - case F_COMP_GE: - ret_type = F_COMP_LE; - break; - case F_COMP_GT: - ret_type = F_COMP_LT; - break; - case F_COMP_LE: - ret_type = F_COMP_GE; - break; - case F_COMP_LT: - ret_type = F_COMP_GT; - break; - default: - // do nothing - break; - } - return ret_type; -} - -static inline int64_t get_mask(ObProxyFunctionType type, ObProxyPartKeyLevel level) -{ - int64_t mask = 0; - int64_t flag = NO_BOUND_FLAG; - switch (type) { - case F_COMP_EQ: - case F_COMP_NSEQ: - flag = BOTH_BOUND_FLAG; - break; - case F_COMP_GE: - case F_COMP_GT: - flag = LOW_BOUND_FLAG; - break; - case F_COMP_LE: - case F_COMP_LT: - flag = HIGH_BOUND_FLAG; - break; - default: - break; - } - switch (level) { - case PART_KEY_LEVEL_ONE: - mask = GET_FIRST_PART_MASK(flag); - break; - case PART_KEY_LEVEL_TWO: - mask = GET_SUB_PART_MASK(flag); - break; - case PART_KEY_LEVEL_BOTH: - mask = GET_FIRST_PART_MASK(flag) | GET_SUB_PART_MASK(flag); - break; - default: - break; - } - return mask; -} - -static inline void set_part_key_column_idx(ObExprParseResult *result, ObProxyParseString *column_name) -{ - int64_t i = 0; - for (i = 0; i < result->part_key_info_.key_num_; ++i) { - if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { - result->part_key_info_.part_keys_[i].idx_ = result->column_idx_; - break; - } - } -} - -#define malloc_node(node, result, type) \ - do { \ - if (OB_ISNULL(node = ((ObProxyTokenNode *)obproxy_parse_malloc(sizeof(ObProxyTokenNode), \ - result->malloc_pool_)))) { \ - YYABORT; \ - } else { \ - node->type_ = type; \ - node->child_ = NULL; \ - node->next_ = NULL; \ - } \ - } while(0) \ - -#define malloc_list(list, result, node) \ - do { \ - if (OB_ISNULL(list = ((ObProxyTokenList *)obproxy_parse_malloc(sizeof(ObProxyTokenList), \ - result->malloc_pool_)))) { \ - YYABORT; \ - } else if (OB_ISNULL(node)) { \ - list->column_node_ = NULL; \ - } else { \ - if (TOKEN_COLUMN == node->type_) { \ - list->column_node_ = node; \ - } else { \ - list->column_node_ = NULL; \ - } \ - list->head_ = node; \ - list->tail_ = node; \ - } \ - } while(0) \ - -#define check_and_add_relation(result, relation) \ - do { \ - if (NULL == relation) { \ - } else { \ - int64_t new_mask = get_mask(relation->type_, relation->level_); \ - if ((result->cur_mask_ | new_mask) != result->cur_mask_) { \ - if (result->relation_info_.relation_num_ < OBPROXY_MAX_RELATION_NUM) { \ - result->relation_info_.relations_[result->relation_info_.relation_num_++] = relation; \ - result->cur_mask_ = (result->cur_mask_ | new_mask); \ - } else { \ - /* YYACCEPT; */ \ - } \ - } \ - if ((result->cur_mask_ & result->target_mask_) == result->target_mask_) { \ - /* YYACCEPT; */ \ - } \ - } \ - } while(0) \ - -static int64_t get_part_key_idx(ObProxyParseString *db_name, - ObProxyParseString *table_name, - ObProxyParseString *column_name, - ObExprParseResult *result) -{ - int64_t part_key_idx = IDX_NO_PART_KEY_COLUMN; - if (NULL != db_name && !is_equal(db_name, &result->table_info_.database_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != table_name - && !is_equal(table_name, &result->table_info_.table_name_) - && !is_equal(table_name, &result->table_info_.alias_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != column_name) { - int64_t i = 0; - for (i = 0; i < result->part_key_info_.key_num_ && part_key_idx < 0; ++i) { - if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { - part_key_idx = i; - break; - } - } - } - return part_key_idx; -} -static inline void add_relation(ObExprParseResult *result, - ObProxyTokenList *left_value, - ObProxyFunctionType type, - ObProxyTokenList *right_value) -{ - if (result->all_relation_info_.relation_num_ < OBPROXY_MAX_RELATION_NUM) { - ObProxyRelationExpr *relation = NULL; - ObProxyTokenList *tmp_left = NULL; - ObProxyTokenList *tmp_right = NULL; - ObProxyFunctionType tmp_type = F_NONE; - ObProxyPartKeyLevel tmp_level = PART_KEY_LEVEL_ZERO; - - if (NULL != left_value->column_node_ - && TOKEN_COLUMN == left_value->column_node_->type_) { - tmp_left = left_value; - tmp_right = right_value; - tmp_type = type; - } else if (NULL != right_value->column_node_ - && TOKEN_COLUMN == right_value->column_node_->type_) { - tmp_left = right_value; - tmp_right = left_value; - tmp_type = get_reverse_func(type); - } - - if (NULL == tmp_left || NULL == tmp_right || F_COMP_NE == tmp_type) { - // will return null - } else if (OB_ISNULL(relation = ((ObProxyRelationExpr *)obproxy_parse_malloc( - sizeof(ObProxyRelationExpr), result->malloc_pool_)))) { - // will return null - } else { - relation->left_value_ = tmp_left; - relation->type_ = tmp_type; - relation->right_value_ = tmp_right; - relation->level_ = tmp_level; - - result->all_relation_info_.relations_[result->all_relation_info_.relation_num_++] = relation; - } - } -} - - -static inline ObProxyRelationExpr *get_relation(ObExprParseResult *result, - ObProxyTokenList *left_value, - ObProxyFunctionType type, - ObProxyTokenList *right_value) -{ - ObProxyRelationExpr *relation = NULL; - ObProxyTokenList *tmp_left = NULL; - ObProxyTokenList *tmp_right = NULL; - ObProxyFunctionType tmp_type = F_NONE; - ObProxyPartKeyLevel tmp_level = PART_KEY_LEVEL_ZERO; - int64_t tmp_column_idx_ = -1; - - if (NULL != left_value->column_node_ - && TOKEN_COLUMN == left_value->column_node_->type_ - && left_value->column_node_->part_key_idx_ >= 0) { - tmp_left = left_value; - tmp_level = result->part_key_info_.part_keys_[left_value->column_node_->part_key_idx_].level_; - tmp_right = right_value; - tmp_type = type; - tmp_column_idx_ = left_value->column_node_->part_key_idx_; - } else if (NULL != right_value->column_node_ - && TOKEN_COLUMN == right_value->column_node_->type_ - && right_value->column_node_->part_key_idx_ >= 0) { - tmp_left = right_value; - tmp_level = result->part_key_info_.part_keys_[right_value->column_node_->part_key_idx_].level_; - tmp_right = left_value; - tmp_type = get_reverse_func(type); - tmp_column_idx_ = right_value->column_node_->part_key_idx_; - } - - if (NULL == tmp_left || NULL == tmp_right || F_COMP_NE == tmp_type) { - // will return null - } else if (OB_ISNULL(relation = ((ObProxyRelationExpr *)obproxy_parse_malloc( - sizeof(ObProxyRelationExpr), result->malloc_pool_)))) { - // will return null - } else { - relation->column_idx_ = tmp_column_idx_; - relation->left_value_ = tmp_left; - relation->type_ = tmp_type; - relation->right_value_ = tmp_right; - relation->level_ = tmp_level; - } - return relation; -} - -static inline ObProxyRelationExpr *get_values_relation(ObExprParseResult *result, - ObProxyTokenList *right_value) -{ - ObProxyRelationExpr *relation = NULL; - if (NULL == right_value) { - // will return null - } else { - int64_t i = 0; - for (i = 0; i < result->part_key_info_.key_num_; ++i) { - if (result->values_list_idx_ == result->part_key_info_.part_keys_[i].idx_) { - if (OB_ISNULL(relation = ((ObProxyRelationExpr *)obproxy_parse_malloc( - sizeof(ObProxyRelationExpr), result->malloc_pool_)))) { - } else { - relation->column_idx_ = i; - relation->type_ = F_COMP_EQ; - relation->right_value_ = right_value; - relation->level_ = result->part_key_info_.part_keys_[i].level_; - } - break; - } - } - } - return relation; -} - -static inline void add_left_relation_value(ObExprParseResult *result, - ObProxyTokenList *left_value) -{ - ObProxyRelationExpr *relation = NULL; - if (NULL == left_value) { - // will return - } else if (result->all_relation_info_.relation_num_ >= OBPROXY_MAX_RELATION_NUM) { - // do nothing - } else if (OB_ISNULL(relation = ((ObProxyRelationExpr *)obproxy_parse_malloc( - sizeof(ObProxyRelationExpr), result->malloc_pool_)))) { - } else { - relation->type_ = F_COMP_EQ; - relation->left_value_ = left_value; - relation->right_value_ = NULL; - result->all_relation_info_.relations_[result->all_relation_info_.relation_num_++] = relation; - } -} - -static inline void add_right_relation_value(ObExprParseResult *result, - ObProxyTokenList *right_value) -{ - if (NULL == right_value) { - // will return - } else if (result->all_relation_info_.relation_num_ >= OBPROXY_MAX_RELATION_NUM) { - // do nohting - } else if (result->all_relation_info_.right_value_num_ >= result->all_relation_info_.relation_num_) { - // ignore - } else if (OB_ISNULL(result->all_relation_info_.relations_[result->all_relation_info_.right_value_num_])) { - } else { - ObProxyRelationExpr *relation = result->all_relation_info_.relations_[result->all_relation_info_.right_value_num_++]; - relation->type_ = F_COMP_EQ; - relation->right_value_ = right_value; - } -} - - - - -#ifdef short -# undef short -#endif - -#ifdef YYTYPE_UINT8 -typedef YYTYPE_UINT8 yytype_uint8; -#else -typedef unsigned char yytype_uint8; -#endif - -#ifdef YYTYPE_INT8 -typedef YYTYPE_INT8 yytype_int8; -#elif (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -typedef signed char yytype_int8; -#else -typedef short int yytype_int8; -#endif - -#ifdef YYTYPE_UINT16 -typedef YYTYPE_UINT16 yytype_uint16; -#else -typedef unsigned short int yytype_uint16; -#endif - -#ifdef YYTYPE_INT16 -typedef YYTYPE_INT16 yytype_int16; -#else -typedef short int yytype_int16; -#endif - -#ifndef YYSIZE_T -# ifdef __SIZE_TYPE__ -# define YYSIZE_T __SIZE_TYPE__ -# elif defined size_t -# define YYSIZE_T size_t -# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# define YYSIZE_T size_t -# else -# define YYSIZE_T unsigned int -# endif -#endif - -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) - -#ifndef YY_ -# if YYENABLE_NLS -# if ENABLE_NLS -# include /* INFRINGES ON USER NAME SPACE */ -# define YY_(msgid) dgettext ("bison-runtime", msgid) -# endif -# endif -# ifndef YY_ -# define YY_(msgid) msgid -# endif -#endif - -/* Suppress unused-variable warnings by "using" E. */ -#if ! defined lint || defined __GNUC__ -# define YYUSE(e) ((void) (e)) -#else -# define YYUSE(e) /* empty */ -#endif - -/* Identity function, used to suppress warnings about constant conditions. */ -#ifndef lint -# define YYID(n) (n) -#else -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static int -YYID (int yyi) -#else -static int -YYID (yyi) - int yyi; -#endif -{ - return yyi; -} -#endif - -#if ! defined yyoverflow || YYERROR_VERBOSE - -/* The parser invokes alloca or malloc; define the necessary symbols. */ - -# ifdef YYSTACK_USE_ALLOCA -# if YYSTACK_USE_ALLOCA -# ifdef __GNUC__ -# define YYSTACK_ALLOC __builtin_alloca -# elif defined __BUILTIN_VA_ARG_INCR -# include /* INFRINGES ON USER NAME SPACE */ -# elif defined _AIX -# define YYSTACK_ALLOC __alloca -# elif defined _MSC_VER -# include /* INFRINGES ON USER NAME SPACE */ -# define alloca _alloca -# else -# define YYSTACK_ALLOC alloca -# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# endif -# endif -# endif - -# ifdef YYSTACK_ALLOC - /* Pacify GCC's `empty if-body' warning. */ -# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) -# ifndef YYSTACK_ALLOC_MAXIMUM - /* The OS might guarantee only one guard page at the bottom of the stack, - and a page size can be as small as 4096 bytes. So we cannot safely - invoke alloca (N) if N exceeds 4096. Use a slightly smaller number - to allow for a few compiler-allocated temporary stack slots. */ -# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ -# endif -# else -# define YYSTACK_ALLOC YYMALLOC -# define YYSTACK_FREE YYFREE -# ifndef YYSTACK_ALLOC_MAXIMUM -# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM -# endif -# if (defined __cplusplus && ! defined _STDLIB_H \ - && ! ((defined YYMALLOC || defined malloc) \ - && (defined YYFREE || defined free))) -# include /* INFRINGES ON USER NAME SPACE */ -# ifndef _STDLIB_H -# define _STDLIB_H 1 -# endif -# endif -# ifndef YYMALLOC -# define YYMALLOC malloc -# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# ifndef YYFREE -# define YYFREE free -# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -void free (void *); /* INFRINGES ON USER NAME SPACE */ -# endif -# endif -# endif -#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - - -#if (! defined yyoverflow \ - && (! defined __cplusplus \ - || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ - && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) - -/* A type that is properly aligned for any stack member. */ -union yyalloc -{ - yytype_int16 yyss_alloc; - YYSTYPE yyvs_alloc; - YYLTYPE yyls_alloc; -}; - -/* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) - -/* The size of an array large to enough to hold all stacks, each with - N elements. */ -# define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE) + sizeof (YYLTYPE)) \ - + 2 * YYSTACK_GAP_MAXIMUM) - -/* Copy COUNT objects from FROM to TO. The source and destination do - not overlap. */ -# ifndef YYCOPY -# if defined __GNUC__ && 1 < __GNUC__ -# define YYCOPY(To, From, Count) \ - __builtin_memcpy (To, From, (Count) * sizeof (*(From))) -# else -# define YYCOPY(To, From, Count) \ - do \ - { \ - YYSIZE_T yyi; \ - for (yyi = 0; yyi < (Count); yyi++) \ - (To)[yyi] = (From)[yyi]; \ - } \ - while (YYID (0)) -# endif -# endif - -/* Relocate STACK from its old location to the new one. The - local variables YYSIZE and YYSTACKSIZE give the old and new number of - elements in the stack, and YYPTR gives the new location of the - stack. Advance YYPTR to a properly aligned location for the next - stack. */ -# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ - do \ - { \ - YYSIZE_T yynewbytes; \ - YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ - Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ - } \ - while (YYID (0)) - -#endif - -/* YYFINAL -- State number of the termination state. */ -#define YYFINAL 17 -/* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 207 - -/* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 44 -/* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 24 -/* YYNRULES -- Number of rules. */ -#define YYNRULES 79 -/* YYNRULES -- Number of states. */ -#define YYNSTATES 143 - -/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 286 - -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) - -/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ -static const yytype_uint8 yytranslate[] = -{ - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 43, 2, 2, 2, 41, 42, 2, - 35, 32, 39, 37, 36, 38, 34, 40, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 33, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31 -}; - -#if YYDEBUG -/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in - YYRHS. */ -static const yytype_uint8 yyprhs[] = -{ - 0, 0, 3, 6, 9, 13, 16, 21, 23, 25, - 27, 29, 31, 33, 36, 40, 43, 48, 52, 57, - 63, 70, 72, 76, 82, 86, 92, 96, 102, 108, - 114, 118, 120, 122, 124, 126, 128, 130, 132, 134, - 138, 140, 142, 145, 146, 150, 154, 156, 160, 166, - 171, 176, 178, 180, 182, 184, 186, 188, 190, 192, - 194, 196, 198, 200, 205, 210, 214, 216, 220, 226, - 227, 231, 233, 237, 239, 241, 245, 247, 251, 252 -}; - -/* YYRHS -- A `-1'-separated list of the rules' RHS. */ -static const yytype_int8 yyrhs[] = -{ - 45, 0, -1, 3, 46, -1, 4, 60, -1, 5, - 51, 47, -1, 48, 47, -1, 48, 5, 51, 47, - -1, 1, -1, 9, -1, 32, -1, 33, -1, 25, - -1, 49, -1, 48, 49, -1, 50, 14, 51, -1, - 10, 28, -1, 10, 28, 34, 28, -1, 10, 28, - 28, -1, 10, 28, 6, 28, -1, 10, 28, 34, - 28, 28, -1, 10, 28, 34, 28, 6, 28, -1, - 52, -1, 51, 11, 52, -1, 35, 51, 11, 52, - 32, -1, 51, 12, 52, -1, 35, 51, 12, 52, - 32, -1, 55, 53, 55, -1, 35, 55, 53, 55, - 32, -1, 55, 13, 35, 54, 32, -1, 55, 15, - 55, 11, 55, -1, 16, 17, 29, -1, 17, -1, - 18, -1, 19, -1, 20, -1, 21, -1, 22, -1, - 23, -1, 55, -1, 54, 36, 55, -1, 56, -1, - 58, -1, 56, 58, -1, -1, 56, 36, 56, -1, - 57, 36, 56, -1, 28, -1, 28, 34, 28, -1, - 28, 34, 28, 34, 28, -1, 28, 35, 56, 32, - -1, 28, 35, 57, 32, -1, 30, -1, 29, -1, - 59, -1, 24, -1, 31, -1, 37, -1, 38, -1, - 39, -1, 40, -1, 41, -1, 42, -1, 43, -1, - 62, 7, 61, 47, -1, 8, 66, 67, 47, -1, - 14, 51, 47, -1, 46, -1, 35, 65, 32, -1, - 61, 36, 35, 65, 32, -1, -1, 35, 63, 32, - -1, 64, -1, 63, 36, 64, -1, 28, -1, 55, - -1, 65, 36, 55, -1, 52, -1, 66, 36, 52, - -1, -1, 5, 51, -1 -}; - -/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = -{ - 0, 373, 373, 374, 376, 377, 378, 379, 381, 382, - 383, 384, 386, 387, 389, 391, 392, 393, 394, 395, - 396, 398, 399, 400, 401, 402, 404, 405, 406, 407, - 417, 424, 425, 426, 427, 428, 429, 430, 432, 433, - 435, 437, 438, 440, 441, 442, 444, 450, 456, 462, - 468, 473, 474, 475, 476, 482, 488, 489, 490, 491, - 492, 493, 494, 496, 497, 498, 499, 501, 505, 510, - 511, 516, 520, 525, 531, 541, 551, 552, 554, 555 -}; -#endif - -#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE -/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. - First, the terminals, then, starting at YYNTOKENS, nonterminals. */ -static const char *const yytname[] = -{ - "$end", "error", "$undefined", "DUMMY_SELECT_CLAUSE", - "DUMMY_INSERT_CLAUSE", "WHERE", "AS", "VALUES", "SET", "END_WHERE", - "JOIN", "AND_OP", "OR_OP", "IN", "ON", "BETWEEN", "ROWID", "COMP_EQ", - "COMP_NSEQ", "COMP_GE", "COMP_GT", "COMP_LE", "COMP_LT", "COMP_NE", - "PLACE_HOLDER", "END_P", "ERROR", "IGNORED_WORD", "NAME_OB", "STR_VAL", - "INT_VAL", "POS_PLACE_HOLDER", "')'", "';'", "'.'", "'('", "','", "'+'", - "'-'", "'*'", "'/'", "'%'", "'&'", "'!'", "$accept", "start", - "select_root", "end_flag", "join_expr_list", "join_on_expr", "join_expr", - "cond_expr", "bool_pri", "comp", "in_expr_list", "expr", "token_list", - "func_param_list", "token", "operator", "insert_root", - "values_expr_lists", "opt_column_list", "column_list", "opt_column", - "values_expr_list", "set_expr", "opt_where_clause", 0 -}; -#endif - -# ifdef YYPRINT -/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to - token YYLEX-NUM. */ -static const yytype_uint16 yytoknum[] = -{ - 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, - 285, 286, 41, 59, 46, 40, 44, 43, 45, 42, - 47, 37, 38, 33 -}; -# endif - -/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = -{ - 0, 44, 45, 45, 46, 46, 46, 46, 47, 47, - 47, 47, 48, 48, 49, 50, 50, 50, 50, 50, - 50, 51, 51, 51, 51, 51, 52, 52, 52, 52, - 52, 53, 53, 53, 53, 53, 53, 53, 54, 54, - 55, 56, 56, 57, 57, 57, 58, 58, 58, 58, - 58, 58, 58, 58, 58, 58, 59, 59, 59, 59, - 59, 59, 59, 60, 60, 60, 60, 61, 61, 62, - 62, 63, 63, 64, 65, 65, 66, 66, 67, 67 -}; - -/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = -{ - 0, 2, 2, 2, 3, 2, 4, 1, 1, 1, - 1, 1, 1, 2, 3, 2, 4, 3, 4, 5, - 6, 1, 3, 5, 3, 5, 3, 5, 5, 5, - 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, - 1, 1, 2, 0, 3, 3, 1, 3, 5, 4, - 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 4, 4, 3, 1, 3, 5, 0, - 3, 1, 3, 1, 1, 3, 1, 3, 0, 2 -}; - -/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state - STATE-NUM when YYTABLE doesn't specify something else to do. Zero - means the default is an error. */ -static const yytype_uint8 yydefact[] = -{ - 0, 0, 0, 0, 7, 0, 0, 2, 0, 12, - 0, 0, 0, 0, 66, 3, 0, 1, 0, 54, - 46, 52, 51, 55, 0, 56, 57, 58, 59, 60, - 61, 62, 0, 21, 0, 40, 41, 53, 15, 0, - 8, 11, 9, 10, 5, 13, 0, 0, 76, 78, - 0, 73, 0, 71, 0, 0, 0, 43, 0, 0, - 0, 0, 4, 0, 0, 31, 32, 33, 34, 35, - 36, 37, 0, 42, 0, 17, 0, 0, 14, 0, - 0, 0, 0, 65, 70, 0, 0, 0, 30, 47, - 0, 0, 0, 0, 0, 22, 24, 0, 0, 26, - 18, 16, 6, 0, 79, 77, 64, 72, 74, 0, - 0, 63, 0, 49, 0, 50, 0, 22, 24, 26, - 0, 38, 0, 0, 19, 0, 67, 0, 0, 48, - 44, 45, 23, 25, 27, 28, 0, 29, 20, 75, - 0, 39, 68 -}; - -/* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int8 yydefgoto[] = -{ - -1, 3, 7, 44, 8, 9, 10, 32, 33, 72, - 120, 34, 35, 91, 36, 37, 15, 87, 16, 52, - 53, 109, 49, 82 -}; - -/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing - STATE-NUM. */ -#define YYPACT_NINF -54 -static const yytype_int16 yypact[] = -{ - 73, 19, 7, 6, -54, 86, -17, -54, 76, -54, - 2, 106, 86, -10, -54, -54, 30, -54, 27, -54, - 58, -54, -54, -54, 86, -54, -54, -54, -54, -54, - -54, -54, 57, -54, 177, 146, -54, -54, -3, 86, - -54, -54, -54, -54, -54, -54, 86, 146, -54, 0, - 57, -54, -6, -54, -7, 26, 22, 146, 88, 177, - 106, 106, -54, 23, 146, -54, -54, -54, -54, -54, - -54, -54, 146, -54, 32, -54, 37, 57, 94, 184, - 86, 106, 62, -54, -54, -10, 146, 13, -54, 33, - 126, 3, 106, 106, 146, -54, -54, 146, 69, -54, - -54, 4, -54, 146, 94, -54, -54, -54, -54, 11, - 53, -54, 63, -54, 146, -54, 146, 75, 79, 81, - 21, -54, 146, 90, -54, 81, -54, 146, 146, -54, - 146, 146, -54, -54, -54, -54, 146, -54, -54, -54, - 42, -54, -54 -}; - -/* YYPGOTO[NTERM-NUM]. */ -static const yytype_int8 yypgoto[] = -{ - -54, -54, 117, -23, -54, 112, -54, -5, -9, -46, - -54, -24, -53, -54, -34, -54, -54, -54, -54, -54, - 46, 5, -54, -54 -}; - -/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If - positive, shift that token. If negative, reduce the rule which - number is the opposite. If zero, do what YYDEFACT says. - If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -70 -static const yytype_int16 yytable[] = -{ - 59, 73, 48, 74, 90, 80, 17, 50, 4, 62, - 123, 38, 5, 94, -69, 11, 46, 6, 51, 58, - 4, 12, 40, 79, 5, 75, 84, 83, 86, 6, - 85, 76, 124, 103, 77, 115, 81, 54, 41, 116, - 98, 78, 13, 126, 55, 42, 43, 127, 99, 110, - 89, 95, 96, 135, 102, 88, 73, 136, 97, 106, - 100, 130, 108, 131, 111, 101, 40, 112, 60, 61, - 119, 40, 105, 121, 142, 104, 1, 2, 127, 125, - 122, 39, 41, 117, 118, 40, 6, 41, 128, 42, - 43, 129, 56, 57, 42, 43, 73, 73, 137, 92, - 93, 41, 18, 139, 108, 60, 61, 132, 42, 43, - 19, 133, 141, 134, 20, 21, 22, 23, 138, 14, - 45, 24, 18, 25, 26, 27, 28, 29, 30, 31, - 19, 107, 0, 140, 20, 21, 22, 23, 0, 0, - 0, 47, 0, 25, 26, 27, 28, 29, 30, 31, - 19, 0, 0, 0, 20, 21, 22, 23, 113, 0, - 0, 0, 114, 25, 26, 27, 28, 29, 30, 31, - 19, 0, 0, 0, 20, 21, 22, 23, 0, 0, - 0, 0, 0, 25, 26, 27, 28, 29, 30, 31, - 63, 0, 64, 0, 65, 66, 67, 68, 69, 70, - 71, 65, 66, 67, 68, 69, 70, 71 -}; - -static const yytype_int16 yycheck[] = -{ - 24, 35, 11, 6, 57, 5, 0, 12, 1, 32, - 6, 28, 5, 59, 7, 8, 14, 10, 28, 24, - 1, 14, 9, 47, 5, 28, 32, 50, 35, 10, - 36, 34, 28, 79, 39, 32, 36, 7, 25, 36, - 64, 46, 35, 32, 17, 32, 33, 36, 72, 36, - 28, 60, 61, 32, 77, 29, 90, 36, 35, 82, - 28, 114, 86, 116, 87, 28, 9, 34, 11, 12, - 94, 9, 81, 97, 32, 80, 3, 4, 36, 103, - 11, 5, 25, 92, 93, 9, 10, 25, 35, 32, - 33, 28, 34, 35, 32, 33, 130, 131, 122, 11, - 12, 25, 16, 127, 128, 11, 12, 32, 32, 33, - 24, 32, 136, 32, 28, 29, 30, 31, 28, 2, - 8, 35, 16, 37, 38, 39, 40, 41, 42, 43, - 24, 85, -1, 128, 28, 29, 30, 31, -1, -1, - -1, 35, -1, 37, 38, 39, 40, 41, 42, 43, - 24, -1, -1, -1, 28, 29, 30, 31, 32, -1, - -1, -1, 36, 37, 38, 39, 40, 41, 42, 43, - 24, -1, -1, -1, 28, 29, 30, 31, -1, -1, - -1, -1, -1, 37, 38, 39, 40, 41, 42, 43, - 13, -1, 15, -1, 17, 18, 19, 20, 21, 22, - 23, 17, 18, 19, 20, 21, 22, 23 -}; - -/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing - symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = -{ - 0, 3, 4, 45, 1, 5, 10, 46, 48, 49, - 50, 8, 14, 35, 46, 60, 62, 0, 16, 24, - 28, 29, 30, 31, 35, 37, 38, 39, 40, 41, - 42, 43, 51, 52, 55, 56, 58, 59, 28, 5, - 9, 25, 32, 33, 47, 49, 14, 35, 52, 66, - 51, 28, 63, 64, 7, 17, 34, 35, 51, 55, - 11, 12, 47, 13, 15, 17, 18, 19, 20, 21, - 22, 23, 53, 58, 6, 28, 34, 51, 51, 55, - 5, 36, 67, 47, 32, 36, 35, 61, 29, 28, - 56, 57, 11, 12, 53, 52, 52, 35, 55, 55, - 28, 28, 47, 53, 51, 52, 47, 64, 55, 65, - 36, 47, 34, 32, 36, 32, 36, 52, 52, 55, - 54, 55, 11, 6, 28, 55, 32, 36, 35, 28, - 56, 56, 32, 32, 32, 32, 36, 55, 28, 55, - 65, 55, 32 -}; - -#define yyerrok (yyerrstatus = 0) -#define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 - -#define YYACCEPT goto yyacceptlab -#define YYABORT goto yyabortlab -#define YYERROR goto yyerrorlab - - -/* Like YYERROR except do call yyerror. This remains here temporarily - to ease the transition to the new meaning of YYERROR, for GCC. - Once GCC version 2 has supplanted version 1, this can go. */ - -#define YYFAIL goto yyerrlab - -#define YYRECOVERING() (!!yyerrstatus) - -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY && yylen == 1) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - yytoken = YYTRANSLATE (yychar); \ - YYPOPSTACK (1); \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (&yylloc, result, YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (YYID (0)) - - -#define YYTERROR 1 -#define YYERRCODE 256 - - -/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. - If N is 0, then set CURRENT to the empty location which ends - the previous symbol: RHS[0] (always defined). */ - -#define YYRHSLOC(Rhs, K) ((Rhs)[K]) -#ifndef YYLLOC_DEFAULT -# define YYLLOC_DEFAULT(Current, Rhs, N) \ - do \ - if (YYID (N)) \ - { \ - (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = \ - YYRHSLOC (Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ - } \ - while (YYID (0)) -#endif - - -/* YY_LOCATION_PRINT -- Print the location on the stream. - This macro was not mandated originally: define only if we know - we won't break user code: when these are the locations we know. */ - -#ifndef YY_LOCATION_PRINT -# if YYLTYPE_IS_TRIVIAL -# define YY_LOCATION_PRINT(File, Loc) \ - fprintf (File, "%d.%d-%d.%d", \ - (Loc).first_line, (Loc).first_column, \ - (Loc).last_line, (Loc).last_column) -# else -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -# endif -#endif - - -/* YYLEX -- calling `yylex' with the right arguments. */ - -#ifdef YYLEX_PARAM -# define YYLEX yylex (&yylval, &yylloc, YYLEX_PARAM) -#else -# define YYLEX yylex (&yylval, &yylloc, YYLEX_PARAM) -#endif - -/* Enable debugging if requested. */ -#if YYDEBUG - -# ifndef YYFPRINTF -# include /* INFRINGES ON USER NAME SPACE */ -# define YYFPRINTF fprintf -# endif - -# define YYDPRINTF(Args) \ -do { \ - if (yydebug) \ - YYFPRINTF Args; \ -} while (YYID (0)) - -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ -do { \ - if (yydebug) \ - { \ - YYFPRINTF (stderr, "%s ", Title); \ - yy_symbol_print (stderr, \ - Type, Value, Location, result); \ - YYFPRINTF (stderr, "\n"); \ - } \ -} while (YYID (0)) - - -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ - -/*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp, ObExprParseResult* result) -#else -static void -yy_symbol_value_print (yyoutput, yytype, yyvaluep, yylocationp, result) - FILE *yyoutput; - int yytype; - YYSTYPE const * const yyvaluep; - YYLTYPE const * const yylocationp; - ObExprParseResult* result; -#endif -{ - if (!yyvaluep) - return; - YYUSE (yylocationp); - YYUSE (result); -# ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); -# else - YYUSE (yyoutput); -# endif - switch (yytype) - { - default: - break; - } -} - - -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp, ObExprParseResult* result) -#else -static void -yy_symbol_print (yyoutput, yytype, yyvaluep, yylocationp, result) - FILE *yyoutput; - int yytype; - YYSTYPE const * const yyvaluep; - YYLTYPE const * const yylocationp; - ObExprParseResult* result; -#endif -{ - if (yytype < YYNTOKENS) - YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); - else - YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); - - YY_LOCATION_PRINT (yyoutput, *yylocationp); - YYFPRINTF (yyoutput, ": "); - yy_symbol_value_print (yyoutput, yytype, yyvaluep, yylocationp, result); - YYFPRINTF (yyoutput, ")"); -} - -/*------------------------------------------------------------------. -| yy_stack_print -- Print the state stack from its BOTTOM up to its | -| TOP (included). | -`------------------------------------------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop) -#else -static void -yy_stack_print (yybottom, yytop) - yytype_int16 *yybottom; - yytype_int16 *yytop; -#endif -{ - YYFPRINTF (stderr, "Stack now"); - for (; yybottom <= yytop; yybottom++) - { - int yybot = *yybottom; - YYFPRINTF (stderr, " %d", yybot); - } - YYFPRINTF (stderr, "\n"); -} - -# define YY_STACK_PRINT(Bottom, Top) \ -do { \ - if (yydebug) \ - yy_stack_print ((Bottom), (Top)); \ -} while (YYID (0)) - - -/*------------------------------------------------. -| Report that the YYRULE is going to be reduced. | -`------------------------------------------------*/ - -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yy_reduce_print (YYSTYPE *yyvsp, YYLTYPE *yylsp, int yyrule, ObExprParseResult* result) -#else -static void -yy_reduce_print (yyvsp, yylsp, yyrule, result) - YYSTYPE *yyvsp; - YYLTYPE *yylsp; - int yyrule; - ObExprParseResult* result; -#endif -{ - int yynrhs = yyr2[yyrule]; - int yyi; - unsigned long int yylno = yyrline[yyrule]; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", - yyrule - 1, yylno); - /* The symbols being reduced. */ - for (yyi = 0; yyi < yynrhs; yyi++) - { - YYFPRINTF (stderr, " $%d = ", yyi + 1); - yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], - &(yyvsp[(yyi + 1) - (yynrhs)]) - , &(yylsp[(yyi + 1) - (yynrhs)]) , result); - YYFPRINTF (stderr, "\n"); - } -} - -# define YY_REDUCE_PRINT(Rule) \ -do { \ - if (yydebug) \ - yy_reduce_print (yyvsp, yylsp, Rule, result); \ -} while (YYID (0)) - -/* Nonzero means print parse trace. It is left uninitialized so that - multiple parsers can coexist. */ -int yydebug; -#else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) -# define YY_STACK_PRINT(Bottom, Top) -# define YY_REDUCE_PRINT(Rule) -#endif /* !YYDEBUG */ - - -/* YYINITDEPTH -- initial size of the parser's stacks. */ -#ifndef YYINITDEPTH -# define YYINITDEPTH 200 -#endif - -/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only - if the built-in stack extension method is used). - - Do not make this value too large; the results are undefined if - YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) - evaluated with infinite-precision integer arithmetic. */ - -#ifndef YYMAXDEPTH -# define YYMAXDEPTH 10000 -#endif - - - -#if YYERROR_VERBOSE - -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static YYSIZE_T -yystrlen (const char *yystr) -#else -static YYSIZE_T -yystrlen (yystr) - const char *yystr; -#endif -{ - YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; - return yylen; -} -# endif -# endif - -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else -/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in - YYDEST. */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static char * -yystpcpy (char *yydest, const char *yysrc) -#else -static char * -yystpcpy (yydest, yysrc) - char *yydest; - const char *yysrc; -#endif -{ - char *yyd = yydest; - const char *yys = yysrc; - - while ((*yyd++ = *yys++) != '\0') - continue; - - return yyd - 1; -} -# endif -# endif - -# ifndef yytnamerr -/* Copy to YYRES the contents of YYSTR after stripping away unnecessary - quotes and backslashes, so that it's suitable for yyerror. The - heuristic is that double-quoting is unnecessary unless the string - contains an apostrophe, a comma, or backslash (other than - backslash-backslash). YYSTR is taken from yytname. If YYRES is - null, do not copy; instead, return the length of what the result - would have been. */ -static YYSIZE_T -yytnamerr (char *yyres, const char *yystr) -{ - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const *yyp = yystr; - - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; - - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; - yyn++; - break; - - case '"': - if (yyres) - yyres[yyn] = '\0'; - return yyn; - } - do_not_strip_quotes: ; - } - - if (! yyres) - return yystrlen (yystr); - - return yystpcpy (yyres, yystr) - yyres; -} -# endif - -/* Copy into YYRESULT an error message about the unexpected token - YYCHAR while in state YYSTATE. Return the number of bytes copied, - including the terminating null byte. If YYRESULT is null, do not - copy anything; just return the number of bytes that would be - copied. As a special case, return 0 if an ordinary "syntax error" - message will do. Return YYSIZE_MAXIMUM if overflow occurs during - size calculation. */ -static YYSIZE_T -yysyntax_error (char *yyresult, int yystate, int yychar) -{ - int yyn = yypact[yystate]; - - if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) - return 0; - else - { - int yytype = YYTRANSLATE (yychar); - YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); - YYSIZE_T yysize = yysize0; - YYSIZE_T yysize1; - int yysize_overflow = 0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - int yyx; - -# if 0 - /* This is so xgettext sees the translatable formats that are - constructed on the fly. */ - YY_("syntax error, unexpected %s"); - YY_("syntax error, unexpected %s, expecting %s"); - YY_("syntax error, unexpected %s, expecting %s or %s"); - YY_("syntax error, unexpected %s, expecting %s or %s or %s"); - YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); -# endif - char *yyfmt; - char const *yyf; - static char const yyunexpected[] = "syntax error, unexpected %s"; - static char const yyexpecting[] = ", expecting %s"; - static char const yyor[] = " or %s"; - char yyformat[sizeof yyunexpected - + sizeof yyexpecting - 1 - + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) - * (sizeof yyor - 1))]; - char const *yyprefix = yyexpecting; - - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. */ - int yyxbegin = yyn < 0 ? -yyn : 0; - - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yycount = 1; - - yyarg[0] = yytname[yytype]; - yyfmt = yystpcpy (yyformat, yyunexpected); - - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - yyformat[sizeof yyunexpected - 1] = '\0'; - break; - } - yyarg[yycount++] = yytname[yyx]; - yysize1 = yysize + yytnamerr (0, yytname[yyx]); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - yyfmt = yystpcpy (yyfmt, yyprefix); - yyprefix = yyor; - } - - yyf = YY_(yyformat); - yysize1 = yysize + yystrlen (yyf); - yysize_overflow |= (yysize1 < yysize); - yysize = yysize1; - - if (yysize_overflow) - return YYSIZE_MAXIMUM; - - if (yyresult) - { - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - char *yyp = yyresult; - int yyi = 0; - while ((*yyp = *yyf) != '\0') - { - if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyf += 2; - } - else - { - yyp++; - yyf++; - } - } - } - return yysize; - } -} -#endif /* YYERROR_VERBOSE */ - - -/*-----------------------------------------------. -| Release the memory associated to this symbol. | -`-----------------------------------------------*/ - -/*ARGSUSED*/ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -static void -yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocationp, ObExprParseResult* result) -#else -static void -yydestruct (yymsg, yytype, yyvaluep, yylocationp, result) - const char *yymsg; - int yytype; - YYSTYPE *yyvaluep; - YYLTYPE *yylocationp; - ObExprParseResult* result; -#endif -{ - YYUSE (yyvaluep); - YYUSE (yylocationp); - YYUSE (result); - - if (!yymsg) - yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); - - switch (yytype) - { - - default: - break; - } -} - -/* Prevent warnings from -Wmissing-prototypes. */ -#ifdef YYPARSE_PARAM -#if defined __STDC__ || defined __cplusplus -int yyparse (void *YYPARSE_PARAM); -#else -int yyparse (); -#endif -#else /* ! YYPARSE_PARAM */ -#if defined __STDC__ || defined __cplusplus -int yyparse (ObExprParseResult* result); -#else -int yyparse (); -#endif -#endif /* ! YYPARSE_PARAM */ - - - - - -/*-------------------------. -| yyparse or yypush_parse. | -`-------------------------*/ - -#ifdef YYPARSE_PARAM -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (void *YYPARSE_PARAM) -#else -int -yyparse (YYPARSE_PARAM) - void *YYPARSE_PARAM; -#endif -#else /* ! YYPARSE_PARAM */ -#if (defined __STDC__ || defined __C99__FUNC__ \ - || defined __cplusplus || defined _MSC_VER) -int -yyparse (ObExprParseResult* result) -#else -int -yyparse (result) - ObExprParseResult* result; -#endif -#endif -{ -/* The lookahead symbol. */ -int yychar; - -/* The semantic value of the lookahead symbol. */ -YYSTYPE yylval; - -/* Location data for the lookahead symbol. */ -YYLTYPE yylloc; - - /* Number of syntax errors so far. */ - int yynerrs; - - int yystate; - /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; - - /* The stacks and their tools: - `yyss': related to states. - `yyvs': related to semantic values. - `yyls': related to locations. - - Refer to the stacks thru separate pointers, to allow yyoverflow - to reallocate them elsewhere. */ - - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16 *yyss; - yytype_int16 *yyssp; - - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE *yyvs; - YYSTYPE *yyvsp; - - /* The location stack. */ - YYLTYPE yylsa[YYINITDEPTH]; - YYLTYPE *yyls; - YYLTYPE *yylsp; - - /* The locations where the error started and ended. */ - YYLTYPE yyerror_range[2]; - - YYSIZE_T yystacksize; - - int yyn; - int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken; - /* The variables used to return semantic value and location from the - action routines. */ - YYSTYPE yyval; - YYLTYPE yyloc; - -#if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char *yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; -#endif - -#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) - - /* The number of symbols on the RHS of the reduced rule. - Keep to zero when no symbol should be popped. */ - int yylen = 0; - - yytoken = 0; - yyss = yyssa; - yyvs = yyvsa; - yyls = yylsa; - yystacksize = YYINITDEPTH; - - YYDPRINTF ((stderr, "Starting parse\n")); - - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; - yychar = YYEMPTY; /* Cause a token to be read. */ - if (SELECT_STMT_PARSE_MODE == result->parse_mode_) { - yychar = DUMMY_SELECT_CLAUSE; - } else if (INSERT_STMT_PARSE_MODE == result->parse_mode_) { - yychar = DUMMY_INSERT_CLAUSE; - } - - - /* Initialize stack pointers. - Waste one element of value and location stack - so that they stay on the same level as the state stack. - The wasted elements are never initialized. */ - yyssp = yyss; - yyvsp = yyvs; - yylsp = yyls; - -#if YYLTYPE_IS_TRIVIAL - /* Initialize the default location before parsing starts. */ - yylloc.first_line = yylloc.last_line = 1; - yylloc.first_column = yylloc.last_column = 1; -#endif - - goto yysetstate; - -/*------------------------------------------------------------. -| yynewstate -- Push a new state, which is found in yystate. | -`------------------------------------------------------------*/ - yynewstate: - /* In all cases, when you get here, the value and location stacks - have just been pushed. So pushing a state here evens the stacks. */ - yyssp++; - - yysetstate: - *yyssp = yystate; - - if (yyss + yystacksize - 1 <= yyssp) - { - /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; - -#ifdef yyoverflow - { - /* Give user a chance to reallocate the stack. Use copies of - these so that the &'s don't force the real ones into - memory. */ - YYSTYPE *yyvs1 = yyvs; - yytype_int16 *yyss1 = yyss; - YYLTYPE *yyls1 = yyls; - - /* Each stack pointer address is followed by the size of the - data in use in that stack, in bytes. This used to be a - conditional around just the two extra args, but that might - be undefined if yyoverflow is a macro. */ - yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), - &yyls1, yysize * sizeof (*yylsp), - &yystacksize); - - yyls = yyls1; - yyss = yyss1; - yyvs = yyvs1; - } -#else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE - goto yyexhaustedlab; -# else - /* Extend the stack our own way. */ - if (YYMAXDEPTH <= yystacksize) - goto yyexhaustedlab; - yystacksize *= 2; - if (YYMAXDEPTH < yystacksize) - yystacksize = YYMAXDEPTH; - - { - yytype_int16 *yyss1 = yyss; - union yyalloc *yyptr = - (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); - if (! yyptr) - goto yyexhaustedlab; - YYSTACK_RELOCATE (yyss_alloc, yyss); - YYSTACK_RELOCATE (yyvs_alloc, yyvs); - YYSTACK_RELOCATE (yyls_alloc, yyls); -# undef YYSTACK_RELOCATE - if (yyss1 != yyssa) - YYSTACK_FREE (yyss1); - } -# endif -#endif /* no yyoverflow */ - - yyssp = yyss + yysize - 1; - yyvsp = yyvs + yysize - 1; - yylsp = yyls + yysize - 1; - - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); - - if (yyss + yystacksize - 1 <= yyssp) - YYABORT; - } - - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); - - if (yystate == YYFINAL) - YYACCEPT; - - goto yybackup; - -/*-----------. -| yybackup. | -`-----------*/ -yybackup: - - /* Do appropriate processing given the current state. Read a - lookahead token if we need one and don't already have one. */ - - /* First try to decide what to do without reference to lookahead token. */ - yyn = yypact[yystate]; - if (yyn == YYPACT_NINF) - goto yydefault; - - /* Not known => get a lookahead token if don't already have one. */ - - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ - if (yychar == YYEMPTY) - { - YYDPRINTF ((stderr, "Reading a token: ")); - yychar = YYLEX; - } - - if (yychar <= YYEOF) - { - yychar = yytoken = YYEOF; - YYDPRINTF ((stderr, "Now at end of input.\n")); - } - else - { - yytoken = YYTRANSLATE (yychar); - YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); - } - - /* If the proper action on seeing token YYTOKEN is to reduce or to - detect an error, take that action. */ - yyn += yytoken; - if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) - goto yydefault; - yyn = yytable[yyn]; - if (yyn <= 0) - { - if (yyn == 0 || yyn == YYTABLE_NINF) - goto yyerrlab; - yyn = -yyn; - goto yyreduce; - } - - /* Count tokens shifted since error; after three, turn off error - status. */ - if (yyerrstatus) - yyerrstatus--; - - /* Shift the lookahead token. */ - YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - - yystate = yyn; - *++yyvsp = yylval; - *++yylsp = yylloc; - goto yynewstate; - - -/*-----------------------------------------------------------. -| yydefault -- do the default action for the current state. | -`-----------------------------------------------------------*/ -yydefault: - yyn = yydefact[yystate]; - if (yyn == 0) - goto yyerrlab; - goto yyreduce; - - -/*-----------------------------. -| yyreduce -- Do a reduction. | -`-----------------------------*/ -yyreduce: - /* yyn is the number of a rule to reduce with. */ - yylen = yyr2[yyn]; - - /* If YYLEN is nonzero, implement the default value of the action: - `$$ = $1'. - - Otherwise, the following line sets YYVAL to garbage. - This behavior is undocumented and Bison - users should not rely upon it. Assigning to YYVAL - unconditionally makes the parser a bit smaller, and it avoids a - GCC warning that YYVAL may be used uninitialized. */ - yyval = yyvsp[1-yylen]; - - /* Default location. */ - YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); - YY_REDUCE_PRINT (yyn); - switch (yyn) - { - case 4: - - { YYACCEPT; ;} - break; - - case 5: - - { YYACCEPT; ;} - break; - - case 6: - - { YYACCEPT; ;} - break; - - case 7: - - { YYACCEPT; ;} - break; - - case 21: - - { check_and_add_relation(result, (yyvsp[(1) - (1)].relation)); ;} - break; - - case 22: - - { check_and_add_relation(result, (yyvsp[(3) - (3)].relation)); ;} - break; - - case 23: - - { check_and_add_relation(result, (yyvsp[(4) - (5)].relation)); ;} - break; - - case 24: - - { check_and_add_relation(result, (yyvsp[(3) - (3)].relation)); ;} - break; - - case 25: - - { check_and_add_relation(result, (yyvsp[(4) - (5)].relation)); ;} - break; - - case 26: - - { add_relation(result, (yyvsp[(1) - (3)].list), (yyvsp[(2) - (3)].func),(yyvsp[(3) - (3)].list)); (yyval.relation) = get_relation(result, (yyvsp[(1) - (3)].list), (yyvsp[(2) - (3)].func), (yyvsp[(3) - (3)].list)); ;} - break; - - case 27: - - { (yyval.relation) = get_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func), (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func),(yyvsp[(4) - (5)].list)); ;} - break; - - case 28: - - { (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ, (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ,(yyvsp[(4) - (5)].list)); ;} - break; - - case 29: - - { - (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_GE, (yyvsp[(3) - (5)].list)); - check_and_add_relation(result, (yyval.relation)); - add_relation(result, (yyvsp[(1) - (5)].list), F_COMP_GE, (yyvsp[(3) - (5)].list)); - (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_LE, (yyvsp[(5) - (5)].list)); - check_and_add_relation(result, (yyval.relation)); - add_relation(result, (yyvsp[(1) - (5)].list), F_COMP_LE, (yyvsp[(5) - (5)].list)); - (yyval.relation) = NULL; - ;} - break; - - case 30: - - { - result->has_rowid_ =true; - result->rowid_str_ = (yyvsp[(3) - (3)].str); - (yyval.relation) = NULL; - ;} - break; - - case 31: - - { (yyval.func) = F_COMP_EQ; ;} - break; - - case 32: - - { (yyval.func) = F_COMP_NSEQ; ;} - break; - - case 33: - - { (yyval.func) = F_COMP_GE; ;} - break; - - case 34: - - { (yyval.func) = F_COMP_GT; ;} - break; - - case 35: - - { (yyval.func) = F_COMP_LE; ;} - break; - - case 36: - - { (yyval.func) = F_COMP_LT; ;} - break; - - case 37: - - { (yyval.func) = F_COMP_NE; ;} - break; - - case 38: - - { (yyval.list) = (yyvsp[(1) - (1)].list); ;} - break; - - case 39: - - { (yyval.list) = (yyvsp[(1) - (3)].list); add_token_list((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} - break; - - case 40: - - { (yyval.list) = (yyvsp[(1) - (1)].list); ;} - break; - - case 41: - - { malloc_list((yyval.list), result, (yyvsp[(1) - (1)].node)); ;} - break; - - case 42: - - { add_token((yyvsp[(1) - (2)].list), result, (yyvsp[(2) - (2)].node)); (yyval.list) = (yyvsp[(1) - (2)].list); ;} - break; - - case 44: - - {;} - break; - - case 45: - - {;} - break; - - case 46: - - { - malloc_node((yyval.node), result, TOKEN_COLUMN); - (yyval.node)->part_key_idx_ = get_part_key_idx(NULL, NULL, &(yyvsp[(1) - (1)].str), result); - (yyval.node)->column_name_ = (yyvsp[(1) - (1)].str); - ;} - break; - - case 47: - - { - malloc_node((yyval.node), result, TOKEN_COLUMN); - (yyval.node)->part_key_idx_ = get_part_key_idx(NULL, &(yyvsp[(1) - (3)].str), &(yyvsp[(3) - (3)].str), result); - (yyval.node)->column_name_ = (yyvsp[(3) - (3)].str); - ;} - break; - - case 48: - - { - malloc_node((yyval.node), result, TOKEN_COLUMN); - (yyval.node)->part_key_idx_ = get_part_key_idx(&(yyvsp[(1) - (5)].str), &(yyvsp[(3) - (5)].str), &(yyvsp[(5) - (5)].str), result); - (yyval.node)->column_name_ = (yyvsp[(5) - (5)].str); - ;} - break; - - case 49: - - { - malloc_node((yyval.node), result, TOKEN_FUNC); - (yyval.node)->str_value_ = (yyvsp[(1) - (4)].str); - (yyval.node)->child_ = (yyvsp[(3) - (4)].list); - ;} - break; - - case 50: - - { - malloc_node((yyval.node), result, TOKEN_FUNC); - (yyval.node)->str_value_ = (yyvsp[(1) - (4)].str); - ;} - break; - - case 51: - - { malloc_node((yyval.node), result, TOKEN_INT_VAL); (yyval.node)->int_value_ = (yyvsp[(1) - (1)].num); ;} - break; - - case 52: - - { malloc_node((yyval.node), result, TOKEN_STR_VAL); (yyval.node)->str_value_ = (yyvsp[(1) - (1)].str); ;} - break; - - case 53: - - { malloc_node((yyval.node), result, TOKEN_OPERATOR); (yyval.node)->operator_ = (yyvsp[(1) - (1)].operator); ;} - break; - - case 54: - - { - result->placeholder_list_idx_++; - malloc_node((yyval.node), result, TOKEN_PLACE_HOLDER); - (yyval.node)->placeholder_idx_ = result->placeholder_list_idx_ - 1; - ;} - break; - - case 55: - - { - malloc_node((yyval.node), result, TOKEN_PLACE_HOLDER); - (yyval.node)->placeholder_idx_ = (yyvsp[(1) - (1)].num); - ;} - break; - - case 56: - - { (yyval.operator) = OPT_ADD; ;} - break; - - case 57: - - { (yyval.operator) = OPT_MINUS; ;} - break; - - case 58: - - { (yyval.operator) = OPT_MUL; ;} - break; - - case 59: - - { (yyval.operator) = OPT_DIV; ;} - break; - - case 60: - - { (yyval.operator) = OPT_MOD; ;} - break; - - case 61: - - { (yyval.operator) = OPT_AND; ;} - break; - - case 62: - - { (yyval.operator) = OPT_NOT; ;} - break; - - case 63: - - { YYACCEPT; ;} - break; - - case 64: - - { YYACCEPT; ;} - break; - - case 65: - - { YYACCEPT; ;} - break; - - case 67: - - { - result->multi_param_values_++; - ;} - break; - - case 68: - - { - result->multi_param_values_++; - ;} - break; - - case 71: - - { - malloc_list((yyval.list), result, (yyvsp[(1) - (1)].node)); - add_left_relation_value(result, (yyval.list)); - ;} - break; - - case 72: - - { - malloc_list((yyval.list), result, (yyvsp[(3) - (3)].node)); - add_left_relation_value(result, (yyval.list)); - ;} - break; - - case 73: - - { - set_part_key_column_idx(result, &(yyvsp[(1) - (1)].str)); - result->column_idx_++; - malloc_node((yyval.node), result, TOKEN_COLUMN); - (yyval.node)->column_name_ = (yyvsp[(1) - (1)].str); - ;} - break; - - case 74: - - { - if (result->multi_param_values_ < 1) { - result->values_list_idx_ = 0; - result->all_relation_info_.right_value_num_ = 0; - ObProxyRelationExpr *relation = get_values_relation(result, (yyvsp[(1) - (1)].list)); - check_and_add_relation(result, relation); - add_right_relation_value(result, (yyvsp[(1) - (1)].list)); - } - ;} - break; - - case 75: - - { - if (result->multi_param_values_ < 1) { - result->values_list_idx_++; - ObProxyRelationExpr *relation = get_values_relation(result, (yyvsp[(3) - (3)].list)); - check_and_add_relation(result, relation); - add_right_relation_value(result, (yyvsp[(3) - (3)].list)); - } - ;} - break; - - case 76: - - { check_and_add_relation(result, (yyvsp[(1) - (1)].relation)); ;} - break; - - case 77: - - { check_and_add_relation(result, (yyvsp[(3) - (3)].relation)); ;} - break; - - case 79: - - {;} - break; - - - - default: break; - } - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); - - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - - *++yyvsp = yyval; - *++yylsp = yyloc; - - /* Now `shift' the result of the reduction. Determine what state - that goes to, based on the state we popped back to and the rule - number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; - - goto yynewstate; - - -/*------------------------------------. -| yyerrlab -- here on detecting error | -`------------------------------------*/ -yyerrlab: - /* If not already recovering from an error, report this error. */ - if (!yyerrstatus) - { - ++yynerrs; -#if ! YYERROR_VERBOSE - yyerror (&yylloc, result, YY_("syntax error")); -#else - { - YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); - if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) - { - YYSIZE_T yyalloc = 2 * yysize; - if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) - yyalloc = YYSTACK_ALLOC_MAXIMUM; - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - yymsg = (char *) YYSTACK_ALLOC (yyalloc); - if (yymsg) - yymsg_alloc = yyalloc; - else - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - } - } - - if (0 < yysize && yysize <= yymsg_alloc) - { - (void) yysyntax_error (yymsg, yystate, yychar); - yyerror (&yylloc, result, yymsg); - } - else - { - yyerror (&yylloc, result, YY_("syntax error")); - if (yysize != 0) - goto yyexhaustedlab; - } - } -#endif - } - - yyerror_range[0] = yylloc; - - if (yyerrstatus == 3) - { - /* If just tried and failed to reuse lookahead token after an - error, discard it. */ - - if (yychar <= YYEOF) - { - /* Return failure if at end of input. */ - if (yychar == YYEOF) - YYABORT; - } - else - { - yydestruct ("Error: discarding", - yytoken, &yylval, &yylloc, result); - yychar = YYEMPTY; - } - } - - /* Else will try to reuse lookahead token after shifting the error - token. */ - goto yyerrlab1; - - -/*---------------------------------------------------. -| yyerrorlab -- error raised explicitly by YYERROR. | -`---------------------------------------------------*/ -yyerrorlab: - - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; - - yyerror_range[0] = yylsp[1-yylen]; - /* Do not reclaim the symbols of the rule which action triggered - this YYERROR. */ - YYPOPSTACK (yylen); - yylen = 0; - YY_STACK_PRINT (yyss, yyssp); - yystate = *yyssp; - goto yyerrlab1; - - -/*-------------------------------------------------------------. -| yyerrlab1 -- common code for both syntax error and YYERROR. | -`-------------------------------------------------------------*/ -yyerrlab1: - yyerrstatus = 3; /* Each real token shifted decrements this. */ - - for (;;) - { - yyn = yypact[yystate]; - if (yyn != YYPACT_NINF) - { - yyn += YYTERROR; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) - { - yyn = yytable[yyn]; - if (0 < yyn) - break; - } - } - - /* Pop the current state because it cannot handle the error token. */ - if (yyssp == yyss) - YYABORT; - - yyerror_range[0] = *yylsp; - yydestruct ("Error: popping", - yystos[yystate], yyvsp, yylsp, result); - YYPOPSTACK (1); - yystate = *yyssp; - YY_STACK_PRINT (yyss, yyssp); - } - - *++yyvsp = yylval; - - yyerror_range[1] = yylloc; - /* Using YYLLOC is tempting, but would change the location of - the lookahead. YYLOC is available though. */ - YYLLOC_DEFAULT (yyloc, (yyerror_range - 1), 2); - *++yylsp = yyloc; - - /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); - - yystate = yyn; - goto yynewstate; - - -/*-------------------------------------. -| yyacceptlab -- YYACCEPT comes here. | -`-------------------------------------*/ -yyacceptlab: - yyresult = 0; - goto yyreturn; - -/*-----------------------------------. -| yyabortlab -- YYABORT comes here. | -`-----------------------------------*/ -yyabortlab: - yyresult = 1; - goto yyreturn; - -#if !defined(yyoverflow) || YYERROR_VERBOSE -/*-------------------------------------------------. -| yyexhaustedlab -- memory exhaustion comes here. | -`-------------------------------------------------*/ -yyexhaustedlab: - yyerror (&yylloc, result, YY_("memory exhausted")); - yyresult = 2; - /* Fall through. */ -#endif - -yyreturn: - if (yychar != YYEMPTY) - yydestruct ("Cleanup: discarding lookahead", - yytoken, &yylval, &yylloc, result); - /* Do not reclaim the symbols of the rule which action triggered - this YYABORT or YYACCEPT. */ - YYPOPSTACK (yylen); - YY_STACK_PRINT (yyss, yyssp); - while (yyssp != yyss) - { - yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp, yylsp, result); - YYPOPSTACK (1); - } -#ifndef yyoverflow - if (yyss != yyssa) - YYSTACK_FREE (yyss); -#endif -#if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); -#endif - /* Make sure YYID is used. */ - return YYID (yyresult); -} - - - - -void yyerror(YYLTYPE* yylloc, ObExprParseResult* p, char* s, ...) -{ - // do nothing -} - -void ob_expr_parser_fatal_error(yyconst char *msg, yyscan_t yyscanner) -{ - fprintf(stderr, "FATAL ERROR:%s\n", msg); - ObExprParseResult *p = obexprget_extra(yyscanner); - if (OB_ISNULL(p)) { - fprintf(stderr, "unexpected null parse result\n"); - } else { - longjmp(p->jmp_buf_, 1);//the secord param must be non-zero value - } -} - -int ob_expr_parse_sql(ObExprParseResult* p, const char* buf, size_t len) -{ - int ret = OB_SUCCESS; - //obexprdebug = 1; - if (OB_ISNULL(p) || OB_ISNULL(buf) || OB_UNLIKELY(len <= 0)) { - ret = OB_INVALID_ARGUMENT; - // print err msg later - } else if (OB_FAIL(obexprlex_init_extra(p, &(p->yyscan_info_)))) { - // print err msg later - } else { - int val = setjmp(p->jmp_buf_); - if (val) { - ret = OB_PARSER_ERR_PARSE_SQL; - } else { - obexpr_scan_buffer((char *)buf, len, p->yyscan_info_); - if (OB_FAIL(obexprparse(p))) { - // print err msg later - } else { - // do nothing - } - } - } - - return ret; -} - diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_lex.c b/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_lex.c index 2e469cf4362e51518ce52f2128b9113714fd05e0..911ee235c003552e9d1f259412473f5892e32c3d 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_lex.c +++ b/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_lex.c @@ -357,29 +357,29 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static yyconst flex_int16_t yy_accept[193] = +static yyconst flex_int16_t yy_accept[197] = { 0, 30, 30, 0, 0, 0, 0, 0, 0, 0, 0, 58, 56, 31, 31, 35, 46, 31, 30, 35, 35, - 40, 33, 34, 32, 35, 35, 35, 28, 35, 23, - 18, 21, 25, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 52, 35, 56, 56, - 39, 38, 39, 41, 42, 57, 47, 48, 57, 54, - 55, 31, 0, 24, 31, 30, 14, 0, 29, 36, - 29, 28, 30, 0, 26, 22, 20, 30, 2, 30, - 30, 30, 30, 17, 30, 30, 11, 15, 30, 30, - 30, 30, 16, 0, 0, 0, 37, 41, 0, 0, - - 43, 0, 0, 44, 47, 0, 49, 0, 0, 0, - 50, 54, 53, 31, 0, 0, 29, 0, 0, 29, - 27, 19, 13, 30, 5, 30, 30, 30, 30, 30, - 4, 30, 30, 33, 34, 32, 45, 0, 45, 0, - 0, 51, 0, 51, 0, 0, 31, 31, 31, 0, - 0, 29, 0, 29, 29, 30, 30, 30, 10, 30, - 30, 30, 30, 0, 0, 0, 0, 31, 30, 7, - 30, 6, 9, 30, 1, 0, 0, 45, 0, 0, - 0, 0, 51, 0, 0, 30, 8, 3, 0, 0, - 12, 0 + 40, 33, 34, 35, 32, 35, 35, 35, 28, 35, + 24, 19, 22, 26, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 52, 35, 56, + 56, 39, 38, 39, 41, 42, 57, 47, 48, 57, + 54, 55, 31, 0, 25, 31, 30, 15, 28, 0, + 0, 28, 29, 36, 29, 28, 30, 0, 27, 23, + 21, 30, 2, 30, 30, 30, 30, 18, 30, 30, + 12, 16, 30, 30, 30, 30, 17, 0, 0, 0, + + 37, 41, 0, 0, 43, 0, 0, 44, 47, 0, + 49, 0, 0, 0, 50, 54, 53, 31, 0, 0, + 0, 29, 0, 0, 29, 20, 14, 30, 6, 30, + 30, 30, 30, 30, 5, 30, 30, 33, 34, 32, + 45, 0, 45, 0, 0, 51, 0, 51, 0, 0, + 31, 31, 31, 0, 29, 0, 29, 0, 29, 30, + 30, 30, 11, 30, 30, 30, 30, 0, 0, 0, + 0, 31, 30, 8, 30, 7, 10, 4, 1, 0, + 0, 45, 0, 0, 0, 0, 51, 0, 0, 30, + 9, 3, 0, 0, 13, 0 } ; @@ -427,128 +427,133 @@ static yyconst flex_int32_t yy_meta[76] = 1, 1, 1, 1, 1 } ; -static yyconst flex_int16_t yy_base[209] = +static yyconst flex_int16_t yy_base[213] = { 0, - 0, 0, 73, 74, 67, 68, 74, 75, 321, 320, - 366, 497, 80, 87, 341, 497, 0, 0, 497, 353, - 497, 497, 497, 497, 345, 340, 345, 74, 78, 62, - 497, 335, 497, 59, 68, 63, 62, 78, 68, 68, - 74, 71, 71, 81, 86, 83, 497, 289, 287, 282, - 497, 497, 121, 0, 139, 0, 0, 145, 0, 0, - 307, 155, 284, 497, 0, 0, 497, 161, 116, 497, - 124, 148, 154, 332, 331, 326, 497, 88, 0, 75, - 120, 132, 128, 0, 140, 140, 0, 0, 133, 137, - 145, 152, 497, 280, 113, 172, 497, 0, 185, 337, - - 497, 330, 277, 497, 0, 202, 497, 339, 327, 273, - 497, 0, 497, 207, 272, 197, 187, 203, 319, 318, - 317, 497, 0, 148, 0, 178, 189, 187, 192, 193, - 0, 184, 188, 497, 497, 497, 497, 325, 324, 228, - 265, 497, 327, 325, 237, 261, 230, 249, 251, 253, - 295, 285, 277, 266, 260, 207, 203, 208, 0, 214, - 229, 231, 232, 266, 168, 269, 128, 259, 236, 0, - 235, 0, 0, 233, 0, 281, 287, 290, 296, 121, - 300, 306, 310, 315, 86, 258, 0, 0, 321, 323, - 0, 497, 392, 399, 406, 413, 420, 95, 427, 434, - - 441, 448, 455, 461, 468, 475, 482, 489 + 0, 0, 73, 74, 67, 68, 74, 75, 330, 329, + 375, 517, 80, 87, 351, 517, 0, 0, 517, 363, + 517, 517, 517, 352, 517, 75, 351, 356, 76, 82, + 62, 517, 346, 517, 63, 72, 65, 66, 81, 71, + 71, 77, 75, 73, 87, 91, 86, 517, 300, 297, + 292, 517, 517, 105, 0, 139, 0, 0, 145, 0, + 0, 317, 155, 294, 517, 0, 0, 517, 342, 161, + 341, 148, 116, 517, 124, 151, 159, 340, 339, 334, + 517, 132, 0, 128, 132, 135, 134, 0, 148, 147, + 0, 0, 139, 143, 151, 158, 517, 288, 118, 177, + + 517, 0, 212, 345, 517, 338, 285, 517, 0, 218, + 517, 347, 334, 281, 517, 0, 517, 208, 273, 210, + 211, 212, 219, 318, 311, 517, 0, 173, 0, 190, + 204, 201, 209, 210, 0, 201, 205, 517, 517, 517, + 517, 312, 306, 244, 247, 517, 296, 294, 247, 229, + 194, 250, 253, 220, 268, 255, 250, 197, 184, 229, + 222, 225, 0, 226, 241, 242, 243, 274, 132, 293, + 124, 270, 250, 0, 250, 0, 0, 246, 0, 301, + 304, 316, 322, 86, 307, 326, 331, 337, 66, 277, + 0, 0, 341, 343, 0, 517, 412, 419, 426, 433, + + 440, 94, 447, 454, 461, 468, 475, 481, 488, 495, + 502, 509 } ; -static yyconst flex_int16_t yy_def[209] = +static yyconst flex_int16_t yy_def[213] = { 0, - 192, 1, 193, 193, 194, 194, 195, 195, 196, 196, - 192, 192, 192, 192, 192, 192, 197, 198, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 198, 192, 192, - 192, 192, 192, 198, 198, 198, 198, 198, 198, 198, - 198, 198, 198, 198, 198, 198, 192, 192, 192, 192, - 192, 192, 192, 199, 192, 200, 201, 192, 202, 203, - 192, 192, 192, 192, 197, 198, 192, 192, 192, 192, - 192, 198, 198, 192, 192, 192, 192, 198, 198, 198, - 198, 198, 198, 198, 198, 198, 198, 198, 198, 198, - 198, 198, 192, 192, 192, 192, 192, 199, 192, 204, - - 192, 192, 192, 192, 201, 192, 192, 205, 192, 192, - 192, 203, 192, 206, 192, 192, 192, 192, 192, 198, - 192, 192, 198, 198, 198, 198, 198, 198, 198, 198, - 198, 198, 198, 192, 192, 192, 192, 204, 204, 192, - 192, 192, 205, 205, 192, 192, 206, 206, 206, 192, - 192, 192, 192, 192, 192, 198, 198, 198, 198, 198, - 198, 198, 198, 207, 192, 208, 192, 206, 198, 198, - 198, 198, 198, 198, 198, 207, 207, 207, 207, 192, - 208, 208, 208, 208, 192, 198, 198, 198, 207, 208, - 198, 0, 192, 192, 192, 192, 192, 192, 192, 192, - - 192, 192, 192, 192, 192, 192, 192, 192 + 196, 1, 197, 197, 198, 198, 199, 199, 200, 200, + 196, 196, 196, 196, 196, 196, 201, 202, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 202, 196, + 196, 196, 196, 196, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 196, 196, 196, + 196, 196, 196, 196, 203, 196, 204, 205, 196, 206, + 207, 196, 196, 196, 196, 201, 202, 196, 196, 196, + 196, 196, 196, 196, 196, 202, 202, 196, 196, 196, + 196, 202, 202, 202, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 196, 196, 196, 196, + + 196, 203, 196, 208, 196, 196, 196, 196, 205, 196, + 196, 209, 196, 196, 196, 207, 196, 210, 196, 196, + 196, 196, 196, 196, 202, 196, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 196, 196, 196, + 196, 208, 208, 196, 196, 196, 209, 209, 196, 196, + 210, 210, 210, 196, 196, 196, 196, 196, 196, 202, + 202, 202, 202, 202, 202, 202, 202, 211, 196, 212, + 196, 210, 202, 202, 202, 202, 202, 202, 202, 211, + 211, 211, 211, 196, 212, 212, 212, 212, 196, 202, + 202, 202, 211, 212, 202, 0, 196, 196, 196, 196, + + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196 } ; -static yyconst flex_int16_t yy_nxt[573] = +static yyconst flex_int16_t yy_nxt[593] = { 0, 12, 13, 14, 13, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 19, 19, 24, 25, 26, 27, 28, - 29, 30, 31, 32, 33, 34, 35, 18, 18, 36, - 37, 38, 39, 40, 41, 18, 18, 42, 18, 43, - 44, 18, 18, 45, 46, 12, 47, 34, 35, 18, - 18, 36, 37, 38, 39, 40, 41, 18, 18, 42, - 18, 43, 44, 18, 18, 45, 46, 48, 12, 12, - 12, 12, 12, 49, 50, 52, 52, 55, 55, 58, - 58, 62, 62, 62, 76, 64, 53, 53, 62, 62, - 62, 71, 74, 72, 74, 78, 80, 75, 66, 79, - - 81, 82, 73, 83, 84, 85, 86, 87, 89, 90, - 88, 91, 56, 56, 92, 123, 124, 78, 80, 59, - 59, 79, 81, 82, 73, 83, 84, 85, 86, 87, - 89, 90, 88, 91, 96, 69, 92, 123, 124, 97, - 99, 99, 99, 117, 116, 100, 106, 106, 106, 101, - 107, 108, 118, 63, 166, 102, 62, 62, 62, 125, - 63, 109, 114, 114, 114, 71, 116, 72, 119, 126, - 119, 127, 128, 120, 118, 129, 73, 130, 131, 132, - 133, 125, 134, 135, 136, 96, 99, 99, 99, 164, - 97, 126, 156, 127, 128, 137, 185, 129, 73, 130, - - 131, 132, 133, 106, 106, 106, 117, 142, 148, 114, - 114, 151, 103, 151, 156, 118, 152, 153, 110, 153, - 157, 158, 154, 159, 160, 161, 162, 163, 63, 164, - 164, 164, 192, 192, 115, 169, 180, 118, 166, 166, - 166, 170, 157, 158, 171, 159, 160, 161, 162, 163, - 148, 114, 114, 192, 192, 172, 173, 169, 103, 174, - 175, 192, 192, 170, 186, 187, 171, 177, 164, 164, - 182, 166, 166, 188, 183, 110, 178, 172, 173, 155, - 149, 174, 175, 192, 192, 154, 186, 187, 177, 164, - 164, 178, 192, 192, 191, 188, 154, 178, 192, 192, - - 178, 165, 192, 192, 152, 183, 178, 182, 166, 166, - 167, 183, 192, 192, 152, 183, 191, 192, 192, 168, - 183, 114, 149, 192, 192, 192, 192, 148, 183, 106, - 144, 178, 144, 99, 139, 139, 121, 120, 155, 179, - 150, 146, 184, 145, 144, 141, 140, 139, 62, 122, - 75, 121, 94, 113, 95, 94, 93, 77, 70, 69, - 179, 68, 67, 64, 189, 192, 61, 61, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 184, - 192, 192, 192, 190, 192, 192, 192, 192, 192, 177, - 192, 182, 51, 51, 51, 51, 51, 51, 51, 54, - - 54, 54, 54, 54, 54, 54, 57, 57, 57, 57, - 57, 57, 57, 60, 60, 60, 60, 60, 60, 60, - 65, 192, 65, 65, 65, 65, 65, 98, 98, 98, - 98, 192, 192, 98, 104, 104, 104, 104, 104, 104, - 104, 105, 105, 192, 105, 105, 192, 105, 111, 111, - 111, 111, 111, 111, 111, 112, 112, 112, 112, 112, - 112, 138, 192, 138, 138, 138, 138, 138, 143, 192, - 143, 143, 143, 143, 143, 147, 147, 147, 147, 147, - 147, 147, 176, 176, 176, 176, 176, 176, 176, 181, - 181, 181, 181, 181, 181, 181, 11, 192, 192, 192, - - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192 + 21, 22, 23, 19, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 18, 18, 37, + 38, 39, 40, 41, 42, 18, 18, 43, 18, 44, + 45, 18, 18, 46, 47, 12, 48, 35, 36, 18, + 18, 37, 38, 39, 40, 41, 42, 18, 18, 43, + 18, 44, 45, 18, 18, 46, 47, 49, 12, 12, + 12, 12, 12, 50, 51, 53, 53, 56, 56, 59, + 59, 63, 63, 63, 80, 65, 54, 54, 63, 63, + 63, 70, 71, 75, 72, 76, 78, 67, 78, 82, + + 84, 79, 85, 83, 77, 86, 87, 88, 89, 90, + 93, 91, 57, 57, 92, 94, 95, 96, 100, 60, + 60, 82, 84, 101, 85, 83, 77, 86, 87, 88, + 89, 90, 93, 91, 170, 73, 92, 94, 95, 96, + 103, 103, 103, 122, 121, 104, 110, 110, 110, 105, + 111, 112, 123, 64, 168, 106, 63, 63, 63, 127, + 64, 113, 118, 118, 118, 75, 121, 72, 75, 128, + 76, 129, 130, 124, 123, 124, 120, 131, 125, 77, + 132, 127, 133, 134, 135, 136, 137, 138, 139, 140, + 100, 128, 189, 129, 130, 101, 196, 196, 120, 131, + + 184, 77, 132, 159, 133, 134, 135, 136, 137, 152, + 118, 118, 107, 103, 103, 103, 159, 160, 114, 110, + 110, 110, 141, 146, 124, 156, 124, 156, 64, 155, + 157, 122, 161, 158, 119, 158, 162, 163, 159, 160, + 123, 164, 165, 166, 167, 168, 168, 168, 170, 170, + 170, 152, 118, 118, 161, 196, 196, 173, 162, 163, + 174, 175, 123, 164, 165, 166, 167, 176, 177, 157, + 178, 179, 196, 196, 157, 181, 168, 168, 190, 173, + 191, 153, 174, 175, 182, 107, 192, 155, 118, 176, + 177, 114, 178, 179, 186, 170, 170, 110, 187, 148, + + 190, 148, 191, 196, 196, 181, 168, 168, 192, 196, + 196, 182, 187, 195, 182, 103, 143, 169, 196, 196, + 171, 172, 143, 153, 196, 196, 182, 186, 170, 170, + 125, 187, 182, 196, 196, 195, 187, 155, 152, 196, + 196, 154, 187, 196, 196, 196, 196, 183, 187, 150, + 149, 182, 148, 145, 144, 143, 63, 126, 79, 79, + 73, 69, 98, 117, 99, 98, 188, 97, 81, 74, + 73, 69, 68, 65, 196, 62, 62, 183, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 193, 196, 196, 196, 196, 196, 196, 196, 196, 188, + + 196, 196, 196, 196, 196, 194, 196, 196, 196, 181, + 196, 186, 52, 52, 52, 52, 52, 52, 52, 55, + 55, 55, 55, 55, 55, 55, 58, 58, 58, 58, + 58, 58, 58, 61, 61, 61, 61, 61, 61, 61, + 66, 196, 66, 66, 66, 66, 66, 102, 102, 102, + 102, 196, 196, 102, 108, 108, 108, 108, 108, 108, + 108, 109, 109, 196, 109, 109, 196, 109, 115, 115, + 115, 115, 115, 115, 115, 116, 116, 116, 116, 116, + 116, 142, 196, 142, 142, 142, 142, 142, 147, 196, + 147, 147, 147, 147, 147, 151, 151, 151, 151, 151, + + 151, 151, 180, 180, 180, 180, 180, 180, 180, 185, + 185, 185, 185, 185, 185, 185, 11, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196 + } ; -static yyconst flex_int16_t yy_chk[573] = +static yyconst flex_int16_t yy_chk[593] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -558,61 +563,64 @@ static yyconst flex_int16_t yy_chk[573] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 4, 5, 6, 7, - 8, 13, 13, 13, 30, 30, 3, 4, 14, 14, - 14, 28, 29, 28, 29, 34, 35, 29, 198, 34, - - 36, 37, 28, 38, 39, 40, 41, 42, 43, 44, - 42, 45, 5, 6, 46, 78, 80, 34, 35, 7, - 8, 34, 36, 37, 28, 38, 39, 40, 41, 42, - 43, 44, 42, 45, 53, 69, 46, 78, 80, 53, - 55, 55, 55, 71, 69, 55, 58, 58, 58, 55, - 58, 58, 71, 13, 185, 55, 62, 62, 62, 81, - 14, 58, 68, 68, 68, 72, 69, 72, 73, 82, - 73, 83, 85, 73, 71, 86, 72, 89, 90, 91, - 92, 81, 95, 95, 95, 96, 99, 99, 99, 180, - 96, 82, 124, 83, 85, 99, 167, 86, 72, 89, - - 90, 91, 92, 106, 106, 106, 117, 106, 114, 114, - 114, 116, 55, 116, 124, 117, 116, 118, 58, 118, - 126, 127, 118, 128, 129, 130, 132, 133, 62, 140, - 140, 140, 147, 147, 68, 156, 165, 117, 145, 145, - 145, 157, 126, 127, 158, 128, 129, 130, 132, 133, - 148, 148, 148, 149, 149, 160, 161, 156, 99, 162, - 163, 168, 168, 157, 169, 171, 158, 164, 164, 164, - 166, 166, 166, 174, 166, 106, 164, 160, 161, 155, - 114, 162, 163, 176, 176, 154, 169, 171, 177, 177, - 177, 176, 178, 178, 186, 174, 153, 177, 179, 179, - - 178, 140, 181, 181, 152, 181, 179, 182, 182, 182, - 145, 182, 183, 183, 151, 183, 186, 184, 184, 149, - 184, 150, 148, 189, 189, 190, 190, 168, 190, 146, - 144, 189, 143, 141, 139, 138, 121, 120, 119, 164, - 115, 110, 166, 109, 108, 103, 102, 100, 94, 76, - 75, 74, 63, 61, 50, 49, 48, 32, 27, 26, - 177, 25, 20, 15, 179, 11, 10, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, - 0, 0, 0, 184, 0, 0, 0, 0, 0, 189, - 0, 190, 193, 193, 193, 193, 193, 193, 193, 194, - - 194, 194, 194, 194, 194, 194, 195, 195, 195, 195, - 195, 195, 195, 196, 196, 196, 196, 196, 196, 196, - 197, 0, 197, 197, 197, 197, 197, 199, 199, 199, - 199, 0, 0, 199, 200, 200, 200, 200, 200, 200, - 200, 201, 201, 0, 201, 201, 0, 201, 202, 202, - 202, 202, 202, 202, 202, 203, 203, 203, 203, 203, - 203, 204, 0, 204, 204, 204, 204, 204, 205, 0, - 205, 205, 205, 205, 205, 206, 206, 206, 206, 206, - 206, 206, 207, 207, 207, 207, 207, 207, 207, 208, - 208, 208, 208, 208, 208, 208, 192, 192, 192, 192, - - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192 + 8, 13, 13, 13, 31, 31, 3, 4, 14, 14, + 14, 26, 26, 29, 26, 29, 30, 202, 30, 35, + + 36, 30, 37, 35, 29, 38, 39, 40, 41, 42, + 44, 43, 5, 6, 43, 45, 46, 47, 54, 7, + 8, 35, 36, 54, 37, 35, 29, 38, 39, 40, + 41, 42, 44, 43, 189, 73, 43, 45, 46, 47, + 56, 56, 56, 75, 73, 56, 59, 59, 59, 56, + 59, 59, 75, 13, 184, 56, 63, 63, 63, 82, + 14, 59, 70, 70, 70, 72, 73, 72, 76, 84, + 76, 85, 86, 77, 75, 77, 72, 87, 77, 76, + 89, 82, 90, 93, 94, 95, 96, 99, 99, 99, + 100, 84, 171, 85, 86, 100, 151, 151, 72, 87, + + 169, 76, 89, 159, 90, 93, 94, 95, 96, 118, + 118, 118, 56, 103, 103, 103, 158, 128, 59, 110, + 110, 110, 103, 110, 120, 121, 120, 121, 63, 120, + 121, 122, 130, 123, 70, 123, 131, 132, 123, 128, + 122, 133, 134, 136, 137, 144, 144, 144, 149, 149, + 149, 152, 152, 152, 130, 153, 153, 160, 131, 132, + 161, 162, 122, 133, 134, 136, 137, 164, 165, 157, + 166, 167, 172, 172, 156, 168, 168, 168, 173, 160, + 175, 118, 161, 162, 168, 103, 178, 155, 154, 164, + 165, 110, 166, 167, 170, 170, 170, 150, 170, 148, + + 173, 147, 175, 180, 180, 181, 181, 181, 178, 185, + 185, 180, 185, 190, 181, 145, 143, 144, 182, 182, + 149, 153, 142, 152, 183, 183, 182, 186, 186, 186, + 125, 186, 183, 187, 187, 190, 187, 124, 172, 188, + 188, 119, 188, 193, 193, 194, 194, 168, 194, 114, + 113, 193, 112, 107, 106, 104, 98, 80, 79, 78, + 71, 69, 64, 62, 51, 50, 170, 49, 33, 28, + 27, 24, 20, 15, 11, 10, 9, 181, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 183, 0, 0, 0, 0, 0, 0, 0, 0, 186, + + 0, 0, 0, 0, 0, 188, 0, 0, 0, 193, + 0, 194, 197, 197, 197, 197, 197, 197, 197, 198, + 198, 198, 198, 198, 198, 198, 199, 199, 199, 199, + 199, 199, 199, 200, 200, 200, 200, 200, 200, 200, + 201, 0, 201, 201, 201, 201, 201, 203, 203, 203, + 203, 0, 0, 203, 204, 204, 204, 204, 204, 204, + 204, 205, 205, 0, 205, 205, 0, 205, 206, 206, + 206, 206, 206, 206, 206, 207, 207, 207, 207, 207, + 207, 208, 0, 208, 208, 208, 208, 208, 209, 0, + 209, 209, 209, 209, 209, 210, 210, 210, 210, 210, + + 210, 210, 211, 211, 211, 211, 211, 211, 211, 212, + 212, 212, 212, 212, 212, 212, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + 196, 196 + } ; /* The intent behind this definition is that it'll catch @@ -669,7 +677,7 @@ do {\ /*following character status will be rewrite by gen_parse.sh according to connection character*/ -#line 674 "ob_expr_parser_utf8_lex.c" +#line 682 "ob_expr_parser_utf8_lex.c" #define INITIAL 0 #define in_c_comment 1 @@ -919,7 +927,7 @@ YY_DECL #line 94 "ob_expr_parser_utf8.l" -#line 925 "ob_expr_parser_utf8_lex.c" +#line 933 "ob_expr_parser_utf8_lex.c" yylval = yylval_param; @@ -976,13 +984,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 193 ) + if ( yy_current_state >= 197 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 497 ); + while ( yy_base[yy_current_state] != 517 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -1024,12 +1032,12 @@ YY_RULE_SETUP case 4: YY_RULE_SETUP #line 98 "ob_expr_parser_utf8.l" -{ return SET; } +{ return VALUES; } YY_BREAK case 5: YY_RULE_SETUP #line 99 "ob_expr_parser_utf8.l" -{ return END_WHERE; } +{ return SET; } YY_BREAK case 6: YY_RULE_SETUP @@ -1049,27 +1057,27 @@ YY_RULE_SETUP case 9: YY_RULE_SETUP #line 103 "ob_expr_parser_utf8.l" -{ return ROWID; } +{ return END_WHERE; } YY_BREAK case 10: YY_RULE_SETUP -#line 105 "ob_expr_parser_utf8.l" -{ return JOIN; } +#line 104 "ob_expr_parser_utf8.l" +{ return ROWID; } YY_BREAK case 11: YY_RULE_SETUP #line 106 "ob_expr_parser_utf8.l" -{ return ON; } +{ return JOIN; } YY_BREAK case 12: YY_RULE_SETUP #line 107 "ob_expr_parser_utf8.l" -{ return BETWEEN; } +{ return ON; } YY_BREAK case 13: YY_RULE_SETUP #line 108 "ob_expr_parser_utf8.l" -{ return AND_OP; } +{ return BETWEEN; } YY_BREAK case 14: YY_RULE_SETUP @@ -1079,7 +1087,7 @@ YY_RULE_SETUP case 15: YY_RULE_SETUP #line 110 "ob_expr_parser_utf8.l" -{ return OR_OP; } +{ return AND_OP; } YY_BREAK case 16: YY_RULE_SETUP @@ -1089,52 +1097,52 @@ YY_RULE_SETUP case 17: YY_RULE_SETUP #line 112 "ob_expr_parser_utf8.l" -{ return IN; } +{ return OR_OP; } YY_BREAK case 18: YY_RULE_SETUP #line 113 "ob_expr_parser_utf8.l" -{ return COMP_EQ; } +{ return IN; } YY_BREAK case 19: YY_RULE_SETUP #line 114 "ob_expr_parser_utf8.l" -{ return COMP_NSEQ; } +{ return COMP_EQ; } YY_BREAK case 20: YY_RULE_SETUP #line 115 "ob_expr_parser_utf8.l" -{ return COMP_GE; } +{ return COMP_NSEQ; } YY_BREAK case 21: YY_RULE_SETUP #line 116 "ob_expr_parser_utf8.l" -{ return COMP_GT; } +{ return COMP_GE; } YY_BREAK case 22: YY_RULE_SETUP #line 117 "ob_expr_parser_utf8.l" -{ return COMP_LE; } +{ return COMP_GT; } YY_BREAK case 23: YY_RULE_SETUP #line 118 "ob_expr_parser_utf8.l" -{ return COMP_LT; } +{ return COMP_LE; } YY_BREAK case 24: YY_RULE_SETUP #line 119 "ob_expr_parser_utf8.l" -{ return COMP_NE; } +{ return COMP_LT; } YY_BREAK case 25: YY_RULE_SETUP #line 120 "ob_expr_parser_utf8.l" -{ return PLACE_HOLDER; } +{ return COMP_NE; } YY_BREAK case 26: YY_RULE_SETUP #line 121 "ob_expr_parser_utf8.l" -{ store_pos_place_holder(yytext + 1, yyscanner); return POS_PLACE_HOLDER; } +{ return PLACE_HOLDER; } YY_BREAK case 27: YY_RULE_SETUP @@ -1399,7 +1407,7 @@ YY_RULE_SETUP #line 256 "ob_expr_parser_utf8.l" ECHO; YY_BREAK -#line 1405 "ob_expr_parser_utf8_lex.c" +#line 1413 "ob_expr_parser_utf8_lex.c" case YY_END_OF_BUFFER: { @@ -1691,7 +1699,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 193 ) + if ( yy_current_state >= 197 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; @@ -1720,11 +1728,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 193 ) + if ( yy_current_state >= 197 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 192); + yy_is_jam = (yy_current_state == 196); return yy_is_jam ? 0 : yy_current_state; } diff --git a/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_tab.c b/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_tab.c index c6fe746c39b42919dbaca6ecbd8a379c4a21a442..c123daaeed58438a31f6d50092a7a72a87446262 100644 --- a/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_tab.c +++ b/src/obproxy/opsql/expr_parser/ob_expr_parser_utf8_tab.c @@ -365,18 +365,20 @@ static int64_t get_part_key_idx(ObProxyParseString *db_name, ObExprParseResult *result) { int64_t part_key_idx = IDX_NO_PART_KEY_COLUMN; - if (NULL != db_name && !is_equal(db_name, &result->table_info_.database_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != table_name - && !is_equal(table_name, &result->table_info_.table_name_) - && !is_equal(table_name, &result->table_info_.alias_name_)) { - part_key_idx = IDX_NO_PART_KEY_COLUMN; - } else if (NULL != column_name) { - int64_t i = 0; - for (i = 0; i < result->part_key_info_.key_num_ && part_key_idx < 0; ++i) { - if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { - part_key_idx = i; - break; + if (result->part_key_info_.key_num_ > 0) { + if (NULL != db_name && !is_equal(db_name, &result->table_info_.database_name_)) { + part_key_idx = IDX_NO_PART_KEY_COLUMN; + } else if (NULL != table_name + && !is_equal(table_name, &result->table_info_.table_name_) + && !is_equal(table_name, &result->table_info_.alias_name_)) { + part_key_idx = IDX_NO_PART_KEY_COLUMN; + } else if (NULL != column_name) { + int64_t i = 0; + for (i = 0; i < result->part_key_info_.key_num_ && part_key_idx < 0; ++i) { + if (is_equal(column_name, &result->part_key_info_.part_keys_[i].name_)) { + part_key_idx = i; + break; + } } } } @@ -746,16 +748,16 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 17 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 207 +#define YYLAST 221 /* YYNTOKENS -- Number of terminals. */ #define YYNTOKENS 44 /* YYNNTS -- Number of nonterminals. */ #define YYNNTS 24 /* YYNRULES -- Number of rules. */ -#define YYNRULES 79 +#define YYNRULES 80 /* YYNRULES -- Number of states. */ -#define YYNSTATES 143 +#define YYNSTATES 146 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 @@ -801,16 +803,17 @@ static const yytype_uint8 yytranslate[] = #if YYDEBUG /* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in YYRHS. */ -static const yytype_uint8 yyprhs[] = +static const yytype_uint16 yyprhs[] = { 0, 0, 3, 6, 9, 13, 16, 21, 23, 25, 27, 29, 31, 33, 36, 40, 43, 48, 52, 57, 63, 70, 72, 76, 82, 86, 92, 96, 102, 108, - 114, 118, 120, 122, 124, 126, 128, 130, 132, 134, - 138, 140, 142, 145, 146, 150, 154, 156, 160, 166, - 171, 176, 178, 180, 182, 184, 186, 188, 190, 192, - 194, 196, 198, 200, 205, 210, 214, 216, 220, 226, - 227, 231, 233, 237, 239, 241, 245, 247, 251, 252 + 114, 120, 124, 126, 128, 130, 132, 134, 136, 138, + 140, 144, 146, 148, 151, 152, 156, 160, 162, 166, + 172, 177, 182, 184, 186, 188, 190, 192, 194, 196, + 198, 200, 202, 204, 206, 211, 216, 220, 222, 226, + 232, 233, 237, 239, 243, 245, 247, 251, 253, 257, + 258 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ @@ -825,36 +828,38 @@ static const yytype_int8 yyrhs[] = 28, 28, -1, 10, 28, 34, 28, 6, 28, -1, 52, -1, 51, 11, 52, -1, 35, 51, 11, 52, 32, -1, 51, 12, 52, -1, 35, 51, 12, 52, - 32, -1, 55, 53, 55, -1, 35, 55, 53, 55, - 32, -1, 55, 13, 35, 54, 32, -1, 55, 15, - 55, 11, 55, -1, 16, 17, 29, -1, 17, -1, - 18, -1, 19, -1, 20, -1, 21, -1, 22, -1, - 23, -1, 55, -1, 54, 36, 55, -1, 56, -1, - 58, -1, 56, 58, -1, -1, 56, 36, 56, -1, - 57, 36, 56, -1, 28, -1, 28, 34, 28, -1, - 28, 34, 28, 34, 28, -1, 28, 35, 56, 32, - -1, 28, 35, 57, 32, -1, 30, -1, 29, -1, - 59, -1, 24, -1, 31, -1, 37, -1, 38, -1, - 39, -1, 40, -1, 41, -1, 42, -1, 43, -1, - 62, 7, 61, 47, -1, 8, 66, 67, 47, -1, - 14, 51, 47, -1, 46, -1, 35, 65, 32, -1, - 61, 36, 35, 65, 32, -1, -1, 35, 63, 32, - -1, 64, -1, 63, 36, 64, -1, 28, -1, 55, - -1, 65, 36, 55, -1, 52, -1, 66, 36, 52, - -1, -1, 5, 51, -1 + 32, -1, 55, 53, 55, -1, 55, 53, 35, 55, + 32, -1, 35, 55, 53, 55, 32, -1, 55, 13, + 35, 54, 32, -1, 55, 15, 55, 11, 55, -1, + 16, 17, 29, -1, 17, -1, 18, -1, 19, -1, + 20, -1, 21, -1, 22, -1, 23, -1, 55, -1, + 54, 36, 55, -1, 56, -1, 58, -1, 56, 58, + -1, -1, 56, 36, 56, -1, 57, 36, 56, -1, + 28, -1, 28, 34, 28, -1, 28, 34, 28, 34, + 28, -1, 28, 35, 56, 32, -1, 28, 35, 57, + 32, -1, 30, -1, 29, -1, 59, -1, 24, -1, + 31, -1, 37, -1, 38, -1, 39, -1, 40, -1, + 41, -1, 42, -1, 43, -1, 62, 7, 61, 47, + -1, 8, 66, 67, 47, -1, 14, 51, 47, -1, + 46, -1, 35, 65, 32, -1, 61, 36, 35, 65, + 32, -1, -1, 35, 63, 32, -1, 64, -1, 63, + 36, 64, -1, 28, -1, 55, -1, 65, 36, 55, + -1, 52, -1, 66, 36, 52, -1, -1, 5, 51, + -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 374, 374, 375, 377, 378, 379, 380, 382, 383, - 384, 385, 387, 388, 390, 392, 393, 394, 395, 396, - 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, - 418, 425, 426, 427, 428, 429, 430, 431, 433, 434, - 436, 438, 439, 441, 442, 443, 445, 451, 457, 463, - 469, 474, 475, 476, 477, 483, 489, 490, 491, 492, - 493, 494, 495, 497, 498, 499, 500, 502, 506, 511, - 512, 517, 521, 526, 532, 542, 552, 553, 555, 556 + 0, 376, 376, 377, 379, 380, 381, 382, 384, 385, + 386, 387, 389, 390, 392, 394, 395, 396, 397, 398, + 399, 401, 402, 403, 404, 405, 407, 408, 409, 410, + 411, 421, 428, 429, 430, 431, 432, 433, 434, 436, + 437, 439, 441, 442, 444, 445, 446, 448, 454, 460, + 466, 472, 478, 479, 480, 481, 487, 493, 494, 495, + 496, 497, 498, 499, 501, 502, 503, 504, 506, 510, + 515, 516, 521, 525, 530, 536, 546, 556, 557, 559, + 560 }; #endif @@ -897,11 +902,12 @@ static const yytype_uint8 yyr1[] = 0, 44, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47, 48, 48, 49, 50, 50, 50, 50, 50, 50, 51, 51, 51, 51, 51, 52, 52, 52, 52, - 52, 53, 53, 53, 53, 53, 53, 53, 54, 54, - 55, 56, 56, 57, 57, 57, 58, 58, 58, 58, - 58, 58, 58, 58, 58, 58, 59, 59, 59, 59, - 59, 59, 59, 60, 60, 60, 60, 61, 61, 62, - 62, 63, 63, 64, 65, 65, 66, 66, 67, 67 + 52, 52, 53, 53, 53, 53, 53, 53, 53, 54, + 54, 55, 56, 56, 57, 57, 57, 58, 58, 58, + 58, 58, 58, 58, 58, 58, 58, 59, 59, 59, + 59, 59, 59, 59, 60, 60, 60, 60, 61, 61, + 62, 62, 63, 63, 64, 65, 65, 66, 66, 67, + 67 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -910,11 +916,12 @@ static const yytype_uint8 yyr2[] = 0, 2, 2, 2, 3, 2, 4, 1, 1, 1, 1, 1, 1, 2, 3, 2, 4, 3, 4, 5, 6, 1, 3, 5, 3, 5, 3, 5, 5, 5, - 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, - 1, 1, 2, 0, 3, 3, 1, 3, 5, 4, - 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 4, 4, 3, 1, 3, 5, 0, - 3, 1, 3, 1, 1, 3, 1, 3, 0, 2 + 5, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 3, 1, 1, 2, 0, 3, 3, 1, 3, 5, + 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 4, 4, 3, 1, 3, 5, + 0, 3, 1, 3, 1, 1, 3, 1, 3, 0, + 2 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -923,113 +930,117 @@ static const yytype_uint8 yyr2[] = static const yytype_uint8 yydefact[] = { 0, 0, 0, 0, 7, 0, 0, 2, 0, 12, - 0, 0, 0, 0, 66, 3, 0, 1, 0, 54, - 46, 52, 51, 55, 0, 56, 57, 58, 59, 60, - 61, 62, 0, 21, 0, 40, 41, 53, 15, 0, - 8, 11, 9, 10, 5, 13, 0, 0, 76, 78, - 0, 73, 0, 71, 0, 0, 0, 43, 0, 0, - 0, 0, 4, 0, 0, 31, 32, 33, 34, 35, - 36, 37, 0, 42, 0, 17, 0, 0, 14, 0, - 0, 0, 0, 65, 70, 0, 0, 0, 30, 47, - 0, 0, 0, 0, 0, 22, 24, 0, 0, 26, - 18, 16, 6, 0, 79, 77, 64, 72, 74, 0, - 0, 63, 0, 49, 0, 50, 0, 22, 24, 26, - 0, 38, 0, 0, 19, 0, 67, 0, 0, 48, - 44, 45, 23, 25, 27, 28, 0, 29, 20, 75, - 0, 39, 68 + 0, 0, 0, 0, 67, 3, 0, 1, 0, 55, + 47, 53, 52, 56, 0, 57, 58, 59, 60, 61, + 62, 63, 0, 21, 0, 41, 42, 54, 15, 0, + 8, 11, 9, 10, 5, 13, 0, 0, 77, 79, + 0, 74, 0, 72, 0, 0, 0, 44, 0, 0, + 0, 0, 4, 0, 0, 32, 33, 34, 35, 36, + 37, 38, 0, 43, 0, 17, 0, 0, 14, 0, + 0, 0, 0, 66, 71, 0, 0, 0, 31, 48, + 0, 0, 0, 0, 0, 22, 24, 0, 0, 0, + 26, 18, 16, 6, 0, 80, 78, 65, 73, 75, + 0, 0, 64, 0, 50, 0, 51, 0, 22, 24, + 26, 0, 39, 0, 0, 0, 19, 0, 68, 0, + 0, 49, 45, 46, 23, 25, 28, 29, 0, 30, + 27, 20, 76, 0, 40, 69 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int8 yydefgoto[] = { -1, 3, 7, 44, 8, 9, 10, 32, 33, 72, - 120, 34, 35, 91, 36, 37, 15, 87, 16, 52, - 53, 109, 49, 82 + 121, 34, 35, 91, 36, 37, 15, 87, 16, 52, + 53, 110, 49, 82 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -54 +#define YYPACT_NINF -53 static const yytype_int16 yypact[] = { - 73, 19, 7, 6, -54, 86, -17, -54, 76, -54, - 2, 106, 86, -10, -54, -54, 30, -54, 27, -54, - 58, -54, -54, -54, 86, -54, -54, -54, -54, -54, - -54, -54, 57, -54, 177, 146, -54, -54, -3, 86, - -54, -54, -54, -54, -54, -54, 86, 146, -54, 0, - 57, -54, -6, -54, -7, 26, 22, 146, 88, 177, - 106, 106, -54, 23, 146, -54, -54, -54, -54, -54, - -54, -54, 146, -54, 32, -54, 37, 57, 94, 184, - 86, 106, 62, -54, -54, -10, 146, 13, -54, 33, - 126, 3, 106, 106, 146, -54, -54, 146, 69, -54, - -54, 4, -54, 146, 94, -54, -54, -54, -54, 11, - 53, -54, 63, -54, 146, -54, 146, 75, 79, 81, - 21, -54, 146, 90, -54, 81, -54, 146, 146, -54, - 146, 146, -54, -54, -54, -54, 146, -54, -54, -54, - 42, -54, -54 + 25, 83, 7, 26, -53, 80, -15, -53, 34, -53, + 10, 100, 80, 18, -53, -53, 57, -53, 69, -53, + 19, -53, -53, -53, 80, -53, -53, -53, -53, -53, + -53, -53, 65, -53, 191, 160, -53, -53, 21, 80, + -53, -53, -53, -53, -53, -53, 80, 160, -53, 1, + 65, -53, -25, -53, 54, 63, 66, 160, 23, 191, + 100, 100, -53, 60, 160, -53, -53, -53, -53, -53, + -53, -53, 140, -53, 74, -53, 75, 65, 67, 198, + 80, 100, 36, -53, -53, 18, 160, 0, -53, 73, + 120, -14, 100, 100, 140, -53, -53, 160, 101, 160, + -53, -53, 13, -53, 160, 67, -53, -53, -53, -53, + 24, 78, -53, 97, -53, 160, -53, 160, 94, 95, + 102, 49, -53, 160, 104, 105, -53, 102, -53, 160, + 160, -53, 160, 160, -53, -53, -53, -53, 160, -53, + -53, -53, -53, 55, -53, -53 }; /* YYPGOTO[NTERM-NUM]. */ -static const yytype_int8 yypgoto[] = +static const yytype_int16 yypgoto[] = { - -54, -54, 117, -23, -54, 112, -54, -5, -9, -46, - -54, -24, -53, -54, -34, -54, -54, -54, -54, -54, - 46, 5, -54, -54 + -53, -53, 130, -30, -53, 137, -53, -8, -10, -49, + -53, -24, -52, -53, -32, -53, -53, -53, -53, -53, + 61, 17, -53, -53 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -70 +#define YYTABLE_NINF -71 static const yytype_int16 yytable[] = { - 59, 73, 48, 74, 90, 80, 17, 50, 4, 62, - 123, 38, 5, 94, -69, 11, 46, 6, 51, 58, - 4, 12, 40, 79, 5, 75, 84, 83, 86, 6, - 85, 76, 124, 103, 77, 115, 81, 54, 41, 116, - 98, 78, 13, 126, 55, 42, 43, 127, 99, 110, - 89, 95, 96, 135, 102, 88, 73, 136, 97, 106, - 100, 130, 108, 131, 111, 101, 40, 112, 60, 61, - 119, 40, 105, 121, 142, 104, 1, 2, 127, 125, - 122, 39, 41, 117, 118, 40, 6, 41, 128, 42, - 43, 129, 56, 57, 42, 43, 73, 73, 137, 92, - 93, 41, 18, 139, 108, 60, 61, 132, 42, 43, - 19, 133, 141, 134, 20, 21, 22, 23, 138, 14, - 45, 24, 18, 25, 26, 27, 28, 29, 30, 31, - 19, 107, 0, 140, 20, 21, 22, 23, 0, 0, - 0, 47, 0, 25, 26, 27, 28, 29, 30, 31, - 19, 0, 0, 0, 20, 21, 22, 23, 113, 0, - 0, 0, 114, 25, 26, 27, 28, 29, 30, 31, - 19, 0, 0, 0, 20, 21, 22, 23, 0, 0, - 0, 0, 0, 25, 26, 27, 28, 29, 30, 31, - 63, 0, 64, 0, 65, 66, 67, 68, 69, 70, - 71, 65, 66, 67, 68, 69, 70, 71 + 59, 48, 62, 73, 50, 90, 80, 84, 4, 40, + 94, 85, 5, 38, -70, 11, 58, 6, 116, 125, + 83, 12, 117, 79, 46, 41, 17, 74, 1, 2, + 104, 77, 42, 43, 92, 93, 111, 81, 78, 39, + 98, 126, 13, 40, 6, 40, 51, 103, 100, 75, + 95, 96, 107, 56, 57, 76, 128, 112, 73, 41, + 129, 41, 109, 132, 54, 133, 42, 43, 42, 43, + 120, 106, 105, 122, 40, 124, 60, 61, 60, 61, + 127, 137, 118, 119, 4, 138, 55, 145, 5, 86, + 41, 129, 88, 6, 89, 97, 18, 42, 43, 139, + 73, 73, 101, 102, 19, 142, 109, 113, 20, 21, + 22, 23, 123, 130, 144, 24, 18, 25, 26, 27, + 28, 29, 30, 31, 19, 131, 134, 135, 20, 21, + 22, 23, 14, 141, 136, 47, 140, 25, 26, 27, + 28, 29, 30, 31, 19, 45, 108, 143, 20, 21, + 22, 23, 114, 0, 0, 0, 115, 25, 26, 27, + 28, 29, 30, 31, 19, 0, 0, 0, 20, 21, + 22, 23, 0, 0, 0, 99, 0, 25, 26, 27, + 28, 29, 30, 31, 19, 0, 0, 0, 20, 21, + 22, 23, 0, 0, 0, 0, 0, 25, 26, 27, + 28, 29, 30, 31, 63, 0, 64, 0, 65, 66, + 67, 68, 69, 70, 71, 65, 66, 67, 68, 69, + 70, 71 }; static const yytype_int16 yycheck[] = { - 24, 35, 11, 6, 57, 5, 0, 12, 1, 32, - 6, 28, 5, 59, 7, 8, 14, 10, 28, 24, - 1, 14, 9, 47, 5, 28, 32, 50, 35, 10, - 36, 34, 28, 79, 39, 32, 36, 7, 25, 36, - 64, 46, 35, 32, 17, 32, 33, 36, 72, 36, - 28, 60, 61, 32, 77, 29, 90, 36, 35, 82, - 28, 114, 86, 116, 87, 28, 9, 34, 11, 12, - 94, 9, 81, 97, 32, 80, 3, 4, 36, 103, - 11, 5, 25, 92, 93, 9, 10, 25, 35, 32, - 33, 28, 34, 35, 32, 33, 130, 131, 122, 11, - 12, 25, 16, 127, 128, 11, 12, 32, 32, 33, - 24, 32, 136, 32, 28, 29, 30, 31, 28, 2, - 8, 35, 16, 37, 38, 39, 40, 41, 42, 43, - 24, 85, -1, 128, 28, 29, 30, 31, -1, -1, - -1, 35, -1, 37, 38, 39, 40, 41, 42, 43, - 24, -1, -1, -1, 28, 29, 30, 31, 32, -1, - -1, -1, 36, 37, 38, 39, 40, 41, 42, 43, - 24, -1, -1, -1, 28, 29, 30, 31, -1, -1, - -1, -1, -1, 37, 38, 39, 40, 41, 42, 43, - 13, -1, 15, -1, 17, 18, 19, 20, 21, 22, - 23, 17, 18, 19, 20, 21, 22, 23 + 24, 11, 32, 35, 12, 57, 5, 32, 1, 9, + 59, 36, 5, 28, 7, 8, 24, 10, 32, 6, + 50, 14, 36, 47, 14, 25, 0, 6, 3, 4, + 79, 39, 32, 33, 11, 12, 36, 36, 46, 5, + 64, 28, 35, 9, 10, 9, 28, 77, 72, 28, + 60, 61, 82, 34, 35, 34, 32, 87, 90, 25, + 36, 25, 86, 115, 7, 117, 32, 33, 32, 33, + 94, 81, 80, 97, 9, 99, 11, 12, 11, 12, + 104, 32, 92, 93, 1, 36, 17, 32, 5, 35, + 25, 36, 29, 10, 28, 35, 16, 32, 33, 123, + 132, 133, 28, 28, 24, 129, 130, 34, 28, 29, + 30, 31, 11, 35, 138, 35, 16, 37, 38, 39, + 40, 41, 42, 43, 24, 28, 32, 32, 28, 29, + 30, 31, 2, 28, 32, 35, 32, 37, 38, 39, + 40, 41, 42, 43, 24, 8, 85, 130, 28, 29, + 30, 31, 32, -1, -1, -1, 36, 37, 38, 39, + 40, 41, 42, 43, 24, -1, -1, -1, 28, 29, + 30, 31, -1, -1, -1, 35, -1, 37, 38, 39, + 40, 41, 42, 43, 24, -1, -1, -1, 28, 29, + 30, 31, -1, -1, -1, -1, -1, 37, 38, 39, + 40, 41, 42, 43, 13, -1, 15, -1, 17, 18, + 19, 20, 21, 22, 23, 17, 18, 19, 20, 21, + 22, 23 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing @@ -1045,12 +1056,12 @@ static const yytype_uint8 yystos[] = 11, 12, 47, 13, 15, 17, 18, 19, 20, 21, 22, 23, 53, 58, 6, 28, 34, 51, 51, 55, 5, 36, 67, 47, 32, 36, 35, 61, 29, 28, - 56, 57, 11, 12, 53, 52, 52, 35, 55, 55, - 28, 28, 47, 53, 51, 52, 47, 64, 55, 65, - 36, 47, 34, 32, 36, 32, 36, 52, 52, 55, - 54, 55, 11, 6, 28, 55, 32, 36, 35, 28, - 56, 56, 32, 32, 32, 32, 36, 55, 28, 55, - 65, 55, 32 + 56, 57, 11, 12, 53, 52, 52, 35, 55, 35, + 55, 28, 28, 47, 53, 51, 52, 47, 64, 55, + 65, 36, 47, 34, 32, 36, 32, 36, 52, 52, + 55, 54, 55, 11, 55, 6, 28, 55, 32, 36, + 35, 28, 56, 56, 32, 32, 32, 32, 36, 55, + 32, 28, 55, 65, 55, 32 }; #define yyerrok (yyerrstatus = 0) @@ -1959,16 +1970,21 @@ yyreduce: case 27: - { (yyval.relation) = get_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func), (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func),(yyvsp[(4) - (5)].list)); ;} + { add_relation(result, (yyvsp[(1) - (5)].list), (yyvsp[(2) - (5)].func),(yyvsp[(4) - (5)].list)); (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), (yyvsp[(2) - (5)].func), (yyvsp[(4) - (5)].list)); ;} break; case 28: - { (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ, (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ,(yyvsp[(4) - (5)].list)); ;} + { (yyval.relation) = get_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func), (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(2) - (5)].list), (yyvsp[(3) - (5)].func),(yyvsp[(4) - (5)].list)); ;} break; case 29: + { (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ, (yyvsp[(4) - (5)].list)); add_relation(result, (yyvsp[(1) - (5)].list), F_COMP_EQ,(yyvsp[(4) - (5)].list)); ;} + break; + + case 30: + { (yyval.relation) = get_relation(result, (yyvsp[(1) - (5)].list), F_COMP_GE, (yyvsp[(3) - (5)].list)); check_and_add_relation(result, (yyval.relation)); @@ -1980,7 +1996,7 @@ yyreduce: ;} break; - case 30: + case 31: { result->has_rowid_ =true; @@ -1989,78 +2005,83 @@ yyreduce: ;} break; - case 31: + case 32: { (yyval.func) = F_COMP_EQ; ;} break; - case 32: + case 33: { (yyval.func) = F_COMP_NSEQ; ;} break; - case 33: + case 34: { (yyval.func) = F_COMP_GE; ;} break; - case 34: + case 35: { (yyval.func) = F_COMP_GT; ;} break; - case 35: + case 36: { (yyval.func) = F_COMP_LE; ;} break; - case 36: + case 37: { (yyval.func) = F_COMP_LT; ;} break; - case 37: + case 38: { (yyval.func) = F_COMP_NE; ;} break; - case 38: + case 39: { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 39: + case 40: { (yyval.list) = (yyvsp[(1) - (3)].list); add_token_list((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); ;} break; - case 40: + case 41: { (yyval.list) = (yyvsp[(1) - (1)].list); ;} break; - case 41: + case 42: { malloc_list((yyval.list), result, (yyvsp[(1) - (1)].node)); ;} break; - case 42: + case 43: { add_token((yyvsp[(1) - (2)].list), result, (yyvsp[(2) - (2)].node)); (yyval.list) = (yyvsp[(1) - (2)].list); ;} break; case 44: - {;} + { (yyval.list) = NULL; ;} break; case 45: - {;} + { add_token_list((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); (yyval.list) = (yyvsp[(1) - (3)].list); ;} break; case 46: + { add_token_list((yyvsp[(1) - (3)].list), (yyvsp[(3) - (3)].list)); (yyval.list) = (yyvsp[(1) - (3)].list); ;} + break; + + case 47: + { malloc_node((yyval.node), result, TOKEN_COLUMN); (yyval.node)->part_key_idx_ = get_part_key_idx(NULL, NULL, &(yyvsp[(1) - (1)].str), result); @@ -2068,7 +2089,7 @@ yyreduce: ;} break; - case 47: + case 48: { malloc_node((yyval.node), result, TOKEN_COLUMN); @@ -2077,7 +2098,7 @@ yyreduce: ;} break; - case 48: + case 49: { malloc_node((yyval.node), result, TOKEN_COLUMN); @@ -2086,7 +2107,7 @@ yyreduce: ;} break; - case 49: + case 50: { malloc_node((yyval.node), result, TOKEN_FUNC); @@ -2095,30 +2116,31 @@ yyreduce: ;} break; - case 50: + case 51: { malloc_node((yyval.node), result, TOKEN_FUNC); (yyval.node)->str_value_ = (yyvsp[(1) - (4)].str); + (yyval.node)->child_ = (yyvsp[(3) - (4)].list); ;} break; - case 51: + case 52: { malloc_node((yyval.node), result, TOKEN_INT_VAL); (yyval.node)->int_value_ = (yyvsp[(1) - (1)].num); ;} break; - case 52: + case 53: { malloc_node((yyval.node), result, TOKEN_STR_VAL); (yyval.node)->str_value_ = (yyvsp[(1) - (1)].str); ;} break; - case 53: + case 54: { malloc_node((yyval.node), result, TOKEN_OPERATOR); (yyval.node)->operator_ = (yyvsp[(1) - (1)].operator); ;} break; - case 54: + case 55: { result->placeholder_list_idx_++; @@ -2127,7 +2149,7 @@ yyreduce: ;} break; - case 55: + case 56: { malloc_node((yyval.node), result, TOKEN_PLACE_HOLDER); @@ -2135,71 +2157,71 @@ yyreduce: ;} break; - case 56: + case 57: { (yyval.operator) = OPT_ADD; ;} break; - case 57: + case 58: { (yyval.operator) = OPT_MINUS; ;} break; - case 58: + case 59: { (yyval.operator) = OPT_MUL; ;} break; - case 59: + case 60: { (yyval.operator) = OPT_DIV; ;} break; - case 60: + case 61: { (yyval.operator) = OPT_MOD; ;} break; - case 61: + case 62: { (yyval.operator) = OPT_AND; ;} break; - case 62: + case 63: { (yyval.operator) = OPT_NOT; ;} break; - case 63: + case 64: { YYACCEPT; ;} break; - case 64: + case 65: { YYACCEPT; ;} break; - case 65: + case 66: { YYACCEPT; ;} break; - case 67: + case 68: { result->multi_param_values_++; ;} break; - case 68: + case 69: { result->multi_param_values_++; ;} break; - case 71: + case 72: { malloc_list((yyval.list), result, (yyvsp[(1) - (1)].node)); @@ -2207,7 +2229,7 @@ yyreduce: ;} break; - case 72: + case 73: { malloc_list((yyval.list), result, (yyvsp[(3) - (3)].node)); @@ -2215,7 +2237,7 @@ yyreduce: ;} break; - case 73: + case 74: { set_part_key_column_idx(result, &(yyvsp[(1) - (1)].str)); @@ -2225,7 +2247,7 @@ yyreduce: ;} break; - case 74: + case 75: { if (result->multi_param_values_ < 1) { @@ -2238,7 +2260,7 @@ yyreduce: ;} break; - case 75: + case 76: { if (result->multi_param_values_ < 1) { @@ -2250,17 +2272,17 @@ yyreduce: ;} break; - case 76: + case 77: { check_and_add_relation(result, (yyvsp[(1) - (1)].relation)); ;} break; - case 77: + case 78: { check_and_add_relation(result, (yyvsp[(3) - (3)].relation)); ;} break; - case 79: + case 80: {;} break; diff --git a/src/obproxy/opsql/expr_resolver/ob_expr_resolver.cpp b/src/obproxy/opsql/expr_resolver/ob_expr_resolver.cpp index ae1669d06ee4aec5d2d1b5f08f1e342021f2be93..ecb6802a614e7784a2269a99522d9a39a74c4dca 100644 --- a/src/obproxy/opsql/expr_resolver/ob_expr_resolver.cpp +++ b/src/obproxy/opsql/expr_resolver/ob_expr_resolver.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX PROXY #include "opsql/expr_resolver/ob_expr_resolver.h" #include "proxy/route/obproxy_part_info.h" +#include "proxy/route/obproxy_expr_calculator.h" #include "proxy/mysqllib/ob_proxy_mysql_request.h" #include "proxy/mysqllib/ob_mysql_request_analyzer.h" #include "proxy/mysqllib/ob_proxy_session_info.h" @@ -80,7 +81,7 @@ int ObExprResolver::resolve(ObExprResolverContext &ctx, ObExprResolverResult &re ctx.part_info_, ctx.client_request_, ctx.client_info_, - ctx.ps_entry_, + ctx.ps_id_entry_, ctx.text_ps_entry_, result.ranges_[part_idx]))) { LOG_INFO("fail to resolve token list, ignore it", K(ret)); @@ -105,7 +106,7 @@ int ObExprResolver::resolve_token_list(ObProxyRelationExpr *relation, ObProxyPartInfo *part_info, ObProxyMysqlRequest *client_request, ObClientSessionInfo *client_info, - ObPsEntry *ps_entry, + ObPsIdEntry *ps_id_entry, ObTextPsEntry *text_ps_entry, ObNewRange &range) { @@ -136,9 +137,13 @@ int ObExprResolver::resolve_token_list(ObProxyRelationExpr *relation, target_obj->set_int(token->int_value_); } else if (TOKEN_PLACE_HOLDER == token->type_) { int64_t param_index = token->placeholder_idx_; - if (OB_FAIL(get_obj_with_param(*target_obj, client_request, - client_info, ps_entry, param_index))) { - LOG_WARN("fail to get target obj with param", K(ret)); + if (OB_FAIL(get_obj_with_param(*target_obj, client_request, client_info, + part_info, ps_id_entry, param_index))) { + LOG_DEBUG("fail to get target obj with param", K(ret)); + } + } else if (TOKEN_FUNC == token->type_) { + if (OB_FAIL(calc_token_func_obj(token, client_info, *target_obj))) { + LOG_WARN("fail to calc token func obj", K(ret)); } } else { ret = OB_INVALID_ARGUMENT; @@ -207,6 +212,76 @@ int ObExprResolver::resolve_token_list(ObProxyRelationExpr *relation, return ret; } +/* + * currently only support to_date/to_timestamp function + * currently support at least one param, at most two params now + */ +int ObExprResolver::calc_token_func_obj(ObProxyTokenNode *token, + ObClientSessionInfo *client_session_info, + ObObj &target_obj) +{ + int ret = OB_SUCCESS; + ObObjType target_type = ObMaxType; + ObString func_name(token->str_value_.str_len_ ,token->str_value_.str_); + + if (func_name.case_compare("to_date") == 0) { + target_type = ObDateTimeType; + } else if (func_name.case_compare("to_timestamp") == 0) { + target_type = ObTimestampNanoType; + } else { + /* more function could be supported */ + } + + if (target_type == ObMaxType) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to parse token func, unsupported function name", K(ret), K(func_name)); + } else if (OB_ISNULL(token->child_) + || OB_ISNULL(token->child_->head_)){ + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to parse token func, param list null", K(ret), K(token->str_value_.str_)); + } else { + ObCollationType collation = ObCharset::get_default_collation(ObCharset::get_default_charset()); + target_obj.set_collation_type(collation); + + ObProxyTokenNode *param_node = token->child_->head_; + if (param_node->type_ == TOKEN_STR_VAL) { + target_obj.set_varchar(param_node->str_value_.str_, param_node->str_value_.str_len_); + } else if (param_node->type_ == TOKEN_INT_VAL) { + target_obj.set_int(param_node->int_value_); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected token node type, please check", K(ret), K(param_node->type_)); + } + + if (OB_SUCC(ret)) { + ObString nls_format; + if (param_node->next_ != NULL) { + nls_format.assign_ptr(param_node->next_->str_value_.str_, param_node->next_->str_value_.str_len_); + } + + ObTimeZoneInfo tz_info; + ObDataTypeCastParams dtc_params; + + if (!nls_format.empty()) { + dtc_params.tz_info_ = &tz_info; + dtc_params.set_nls_format_by_type(target_type, nls_format); + } else { + dtc_params.tz_info_ = &tz_info; + if (OB_FAIL(ObExprCalcTool::build_dtc_params(client_session_info, target_type, dtc_params))) { + LOG_WARN("fail to build dtc params", K(ret), K(target_type)); + } + } + + ObCastCtx cast_ctx(&allocator_, &dtc_params, CM_NULL_ON_WARN, collation); + if (OB_SUCC(ret) && OB_FAIL(ObObjCasterV2::to_type(target_type, collation, cast_ctx, target_obj, target_obj))) { + LOG_WARN("fail to cast obj", K(ret), K(target_obj), K(target_type), K(collation)); + } + } + } + + return ret; +} + int ObExprResolver::calc_generated_key_value(ObObj &obj, const ObProxyPartKey &part_key, const bool is_oracle_mode) { int ret = OB_SUCCESS; @@ -247,9 +322,12 @@ int ObExprResolver::calc_generated_key_value(ObObj &obj, const ObProxyPartKey &p return ret; } -int ObExprResolver::get_obj_with_param(ObObj &target_obj, ObProxyMysqlRequest *client_request, +int ObExprResolver::get_obj_with_param(ObObj &target_obj, + ObProxyMysqlRequest *client_request, ObClientSessionInfo *client_info, - ObPsEntry *ps_entry, const int64_t param_index) + ObProxyPartInfo *part_info, + ObPsIdEntry *ps_id_entry, + const int64_t param_index) { int ret = OB_SUCCESS; if (OB_ISNULL(client_request) || OB_ISNULL(client_info) || OB_UNLIKELY(param_index < 0)) { @@ -287,19 +365,20 @@ int ObExprResolver::get_obj_with_param(ObObj &target_obj, ObProxyMysqlRequest *c } else { need_use_execute_param = true; } - if (OB_SUCC(ret) && need_use_execute_param + if (OB_SUCC(ret) + && need_use_execute_param && OB_MYSQL_COM_STMT_EXECUTE == client_request->get_packet_meta().cmd_) { // for com_stmt_prepare, we have no execute_params, so no need continue, just return LOG_DEBUG("will cal obj with value from execute param", K(execute_param_index)); - if (OB_ISNULL(ps_entry)) { + if (OB_ISNULL(ps_id_entry)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ps entry is null", K(ret)); - } else if (OB_UNLIKELY(execute_param_index >= ps_entry->get_param_count()) + LOG_WARN("client ps id entry is null", K(ret), KPC(ps_id_entry)); + } else if (OB_UNLIKELY(execute_param_index >= ps_id_entry->get_param_count()) || execute_param_index < 0) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid placeholder idx", K(execute_param_index), KPC(ps_entry), K(ret)); - } else if (OB_FAIL(ObMysqlRequestAnalyzer::analyze_execute_param(ps_entry->get_param_count(), - ps_entry->get_ps_sql_meta().get_param_types(), *client_request, execute_param_index, target_obj))) { + LOG_WARN("invalid placeholder idx", K(execute_param_index), KPC(ps_id_entry), K(ret)); + } else if (OB_FAIL(ObMysqlRequestAnalyzer::analyze_execute_param(ps_id_entry->get_param_count(), + ps_id_entry->get_ps_sql_meta().get_param_types(), *client_request, execute_param_index, target_obj))) { LOG_WARN("fail to analyze execute param", K(ret)); } } @@ -308,7 +387,8 @@ int ObExprResolver::get_obj_with_param(ObObj &target_obj, ObProxyMysqlRequest *c LOG_DEBUG("prepare sql with only placeholder, will return fail", K(ret)); } - if (OB_SUCC(ret) && need_use_execute_param + if (OB_SUCC(ret) + && need_use_execute_param && client_request->get_parse_result().is_text_ps_execute_stmt()) { ObSqlParseResult &parse_result = client_request->get_parse_result(); ObProxyTextPsExecuteInfo execute_info = parse_result.text_ps_execute_info_; @@ -325,7 +405,8 @@ int ObExprResolver::get_obj_with_param(ObObj &target_obj, ObProxyMysqlRequest *c } } - if (OB_SUCC(ret) && need_use_execute_param + if (OB_SUCC(ret) + && need_use_execute_param && OB_MYSQL_COM_STMT_PREPARE_EXECUTE == client_request->get_packet_meta().cmd_) { LOG_DEBUG("will cal obj with value from execute param", K(execute_param_index)); if (OB_UNLIKELY(execute_param_index < 0)) { @@ -335,6 +416,16 @@ int ObExprResolver::get_obj_with_param(ObObj &target_obj, ObProxyMysqlRequest *c LOG_WARN("fail to analyze execute param", K(ret)); } } + + if (OB_SUCC(ret) + && need_use_execute_param + && OB_MYSQL_COM_STMT_SEND_LONG_DATA == client_request->get_packet_meta().cmd_) { + LOG_DEBUG("will calc obj with execute param for send long data"); + if (OB_FAIL(ObMysqlRequestAnalyzer::analyze_send_long_data_param(*client_request, execute_param_index, + part_info, ps_id_entry, target_obj))) { + LOG_DEBUG("fail to analyze send long data param", K(ret)); + } + } } return ret; } diff --git a/src/obproxy/opsql/expr_resolver/ob_expr_resolver.h b/src/obproxy/opsql/expr_resolver/ob_expr_resolver.h index b4ed8f7c4a5e89ea0b38932b4f489038b7de5c6e..fc515185de73a46311b50b87f2c58abc975754b7 100644 --- a/src/obproxy/opsql/expr_resolver/ob_expr_resolver.h +++ b/src/obproxy/opsql/expr_resolver/ob_expr_resolver.h @@ -29,7 +29,7 @@ namespace proxy { class ObProxyPartInfo; class ObProxyMysqlRequest; -class ObPsEntry; +class ObPsIdEntry; class ObTextPsEntry; class ObClientSessionInfo; } @@ -38,12 +38,13 @@ namespace opsql struct ObExprResolverContext { ObExprResolverContext() : relation_info_(NULL), part_info_(NULL), client_request_(NULL), - ps_entry_(NULL), text_ps_entry_(NULL), client_info_(NULL) {} + ps_id_entry_(NULL), text_ps_entry_(NULL), client_info_(NULL) {} // parse result ObProxyRelationInfo *relation_info_; proxy::ObProxyPartInfo *part_info_; proxy::ObProxyMysqlRequest *client_request_; - proxy::ObPsEntry *ps_entry_; + // proxy::ObPsEntry *ps_entry_; + proxy::ObPsIdEntry *ps_id_entry_; proxy::ObTextPsEntry *text_ps_entry_; proxy::ObClientSessionInfo *client_info_; }; @@ -71,14 +72,18 @@ private: proxy::ObProxyPartInfo *part_info, proxy::ObProxyMysqlRequest *client_request, proxy::ObClientSessionInfo *client_info, - proxy::ObPsEntry *ps_entry, + proxy::ObPsIdEntry *ps_entry, proxy::ObTextPsEntry *text_ps_entry, common::ObNewRange &range); + int calc_token_func_obj(ObProxyTokenNode *token, + proxy::ObClientSessionInfo *client_session_info, + common::ObObj &target_obj); int calc_generated_key_value(common::ObObj &obj, const ObProxyPartKey &part_key, const bool is_oracle_mode); int get_obj_with_param(common::ObObj &target_obj, proxy::ObProxyMysqlRequest *client_request, proxy::ObClientSessionInfo *client_info, - proxy::ObPsEntry *ps_entry, + proxy::ObProxyPartInfo *part_info, + proxy::ObPsIdEntry *ps_entry, const int64_t param_index); common::ObIAllocator &allocator_; diff --git a/src/obproxy/opsql/func_expr_parser/ob_func_expr_parse_result.cpp b/src/obproxy/opsql/func_expr_parser/ob_func_expr_parse_result.cpp index 4922ed9dca720114de086505a99044af5a74a87f..0ca103bae27fd2f0323b084116ba5ac3e5e98b2e 100644 --- a/src/obproxy/opsql/func_expr_parser/ob_func_expr_parse_result.cpp +++ b/src/obproxy/opsql/func_expr_parser/ob_func_expr_parse_result.cpp @@ -38,12 +38,15 @@ const char* get_generate_function_type(const ObProxyExprType type) case OB_PROXY_EXPR_TYPE_SHARDING_CONST: str_ret = "OB_PROXY_EXPR_TYPE_SHARDING_CONST"; break; - case OB_PROXY_EXPR_TYPE_SHARDING_ALIAS: - str_ret = "OB_PROXY_EXPR_TYPE_SHARDING_ALIAS"; + case OB_PROXY_EXPR_TYPE_TABLE: + str_ret = "OB_PROXY_EXPR_TYPE_TABLE"; break; case OB_PROXY_EXPR_TYPE_COLUMN: str_ret = "OB_PROXY_EXPR_TYPE_COLUMN"; break; + case OB_PROXY_EXPR_TYPE_STAR: + str_ret = "OB_PROXY_EXPR_TYPE_STAR"; + break; case OB_PROXY_EXPR_TYPE_FUNC_HASH: str_ret = "OB_PROXY_EXPR_TYPE_FUNC_HASH"; break; @@ -86,6 +89,9 @@ const char* get_generate_function_type(const ObProxyExprType type) case OB_PROXY_EXPR_TYPE_FUNC_ORDER: str_ret = "OB_PROXY_EXPR_TYPE_FUNC_ORDER"; break; + case OB_PROXY_EXPR_TYPE_FUNC_GROUP: + str_ret = "OB_PROXY_EXPR_TYPE_FUNC_GROUP"; + break; case OB_PROXY_EXPR_TYPE_FUNC_TESTLOAD: str_ret = "OB_PROXY_EXPR_TYPE_FUNC_TESTLOAD"; break; diff --git a/src/obproxy/opsql/func_expr_parser/ob_func_expr_parser_lex.c b/src/obproxy/opsql/func_expr_parser/ob_func_expr_parser_lex.c index 91e39fd11dd429380a2338d4b801370ab0bbed6f..0a5152330a4976f74bcd76973275f0807e0125b2 100644 --- a/src/obproxy/opsql/func_expr_parser/ob_func_expr_parser_lex.c +++ b/src/obproxy/opsql/func_expr_parser/ob_func_expr_parser_lex.c @@ -742,7 +742,7 @@ static int input (yyscan_t yyscanner ); /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0) +#define ECHO fwrite( yytext, yyleng, 1, yyout ) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, @@ -753,7 +753,7 @@ static int input (yyscan_t yyscanner ); if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ { \ int c = '*'; \ - unsigned n; \ + int n; \ for ( n = 0; n < max_size && \ (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ buf[n] = (char) c; \ diff --git a/src/obproxy/opsql/func_expr_resolver/ob_func_expr_resolver.cpp b/src/obproxy/opsql/func_expr_resolver/ob_func_expr_resolver.cpp index 6f2585e1e10e8e10a1ef477fa8c714d0c0a1eb47..f586ef93f7f7065c7b8d5c44d62f4209e876eb4c 100644 --- a/src/obproxy/opsql/func_expr_resolver/ob_func_expr_resolver.cpp +++ b/src/obproxy/opsql/func_expr_resolver/ob_func_expr_resolver.cpp @@ -253,4 +253,4 @@ int ObFuncExprResolver::create_func_expr_by_type(const ObFuncExprNode *func_expr } // end opsql } // end obproxy -} // end oceanbase \ No newline at end of file +} // end oceanbase diff --git a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.cpp b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.cpp index 0cdf9157db44685692b341fd3a45d50491bae804..9c3bb18f048eae98afc23debe3bd842d095d928c 100644 --- a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.cpp +++ b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.cpp @@ -27,18 +27,19 @@ using namespace dbconfig; namespace opsql { -int get_number_obj_for_calc(ObIAllocator *allocator, ObObj &left, ObObj &right) +template +int get_obj_for_calc(ObIAllocator *allocator, ObObj &left, ObObj &right) { int ret = OB_SUCCESS; ObCastCtx cast_ctx(allocator, NULL, CM_NULL_ON_WARN, CS_TYPE_UTF8MB4_GENERAL_CI); - if (ObNumberTC != left.get_type_class()) { - if (OB_FAIL(ObObjCasterV2::to_type(ObNumberType, cast_ctx, left, left))) { + if (obj_type_class != left.get_type_class()) { + if (OB_FAIL(ObObjCasterV2::to_type(obj_type, cast_ctx, left, left))) { LOG_WARN("failed to cast obj", K(ret)); } } - if (OB_SUCC(ret) && ObNumberTC != right.get_type_class()) { - if (OB_FAIL(ObObjCasterV2::to_type(ObNumberType, cast_ctx, right, right))) { + if (OB_SUCC(ret) && obj_type_class != right.get_type_class()) { + if (OB_FAIL(ObObjCasterV2::to_type(obj_type, cast_ctx, right, right))) { LOG_WARN("failed to cast obj", K(ret)); } } @@ -54,32 +55,34 @@ static void get_proxy_expr_result_tree_str(ObProxyExpr *root, const int level, c for (int i = 0 ; i < level * 2; i++) { pos += snprintf(buf + pos, length - pos, "-"); } - pos += snprintf (buf + pos, length - pos, " type:%s, index:%ld, has_agg:%d, has_alias:%d, is_func_expr:%d, is_star:%d, addr:%p\n", + pos += snprintf (buf + pos, length - pos, " type:%s, index:%ld, has_agg:%d, is_func_expr:%d, addr:%p\n", get_expr_type_name(root->type_), root->index_, root->has_agg_, - root->has_alias_, root->is_func_expr_, - root->is_star_expr(), root); if (root->is_func_expr()) { - ObProxyFuncExpr* func_expr = static_cast(root); + ObProxyFuncExpr* func_expr = dynamic_cast(root); for (int i = 0; i < func_expr->get_param_array().count(); i++) { get_proxy_expr_result_tree_str(func_expr->get_param_array().at(i), level + 1, buf, pos, length - pos); } - } else if (OB_PROXY_EXPR_TYPE_SHARDING_CONST == root->type_) { - get_proxy_expr_result_tree_str(static_cast(root)->expr_, level + 1, buf, pos, length - pos); + } else if (OB_PROXY_EXPR_TYPE_FUNC_GROUP == root->get_expr_type() + || OB_PROXY_EXPR_TYPE_FUNC_ORDER == root->get_expr_type()) { + ObProxyGroupItem* group_expr = dynamic_cast(root); + get_proxy_expr_result_tree_str(group_expr->get_expr(), level + 1, buf, pos, length - pos); } } void ObProxyExpr::print_proxy_expr(ObProxyExpr *root) { - char buf[256 * 1024]; - int pos = 0; - get_proxy_expr_result_tree_str(root, 0, buf, pos, 256 * 1024); - ObString tree_str(16 * 1024, buf); - LOG_DEBUG("proxy_expr is \n", K(tree_str)); + if (OB_UNLIKELY(IS_DEBUG_ENABLED())) { + char buf[256 * 1024]; + int pos = 0; + get_proxy_expr_result_tree_str(root, 0, buf, pos, 256 * 1024); + ObString tree_str(16 * 1024, buf); + LOG_DEBUG("proxy_expr is \n", K(tree_str)); + } } int64_t ObProxyExpr::to_string(char *buf, int64_t buf_len) const @@ -91,41 +94,19 @@ int64_t ObProxyExpr::to_string(char *buf, int64_t buf_len) const J_KV(K_(index)); J_COMMA(); J_KV(K_(has_agg)); - J_COMMA(); - J_KV(K_(has_alias)); J_OBJ_END(); return pos; } -bool ObProxyExpr::is_star_expr() -{ - int ret = OB_SUCCESS; - bool bret = false; - if (OB_PROXY_EXPR_TYPE_SHARDING_CONST == type_) { - ObProxyExprShardingConst *expr = static_cast(this); - ObString str; - if (OB_FAIL(expr->get_object().get_string(str))) { - // do nothing - } else if (str == "*") { - bret = true; - } - } - - return bret; -} - int ObProxyExpr::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array) { int ret = OB_SUCCESS; UNUSED(ctx); - if (is_star_expr()) { - ret = OB_EXPR_CALC_ERROR; - LOG_WARN("* can't calc", K(ret)); - } else if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_) { + if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_) { int64_t len = calc_item.obj_array_->count(); if (index_ >= len - ||OB_FAIL(result_obj_array.push_back(*calc_item.obj_array_->at(len - 1 - index_)))) { + || OB_FAIL(result_obj_array.push_back(*calc_item.obj_array_->at(index_)))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("calc expr failed", K(ret), K(index_), K(calc_item.obj_array_->count())); } @@ -167,17 +148,36 @@ int ObProxyExpr::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc return ret; } -bool ObProxyExpr::is_alias() +int ObProxyExpr::to_sql_string(ObSqlString& sql_string) { - bool bret = false; - if (OB_PROXY_EXPR_TYPE_SHARDING_CONST != type_) { - // do nothing - } else { - ObProxyExprShardingConst *const_expr = static_cast(this); - bret = const_expr->is_alias_; + int ret = OB_SUCCESS; + if (OB_FAIL(to_column_string(sql_string))) { + LOG_WARN("fail to column string", K(ret)); + } else if (!alias_name_.empty()) { + if (OB_FAIL(sql_string.append(" AS "))){ + LOG_WARN("append failed", K(ret)); + } else { + ret = sql_string.append_fmt("%.*s", alias_name_.length(), alias_name_.ptr()); + } + } + + if (OB_FAIL(ret)) { + LOG_WARN("fail to sql_string", K_(alias_name), K(ret)); } - return bret; + return ret; +} + +int ObProxyExprConst::to_column_string(ObSqlString& sql_string) +{ + int ret = OB_SUCCESS; + if (obj_.get_type() == ObIntType) { + ret = sql_string.append_fmt("%ld", obj_.get_int()); + } else { + ObString str = obj_.get_string(); + ret = sql_string.append_fmt("'%.*s'", str.length(), str.ptr()); + } + return ret; } int ObProxyExprConst::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, @@ -197,45 +197,22 @@ int ObProxyExprConst::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem return ret; } -int ObProxyExprShardingConst::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array) +int ObProxyExprColumn::to_column_string(ObSqlString& sql_string) { int ret = OB_SUCCESS; - int64_t len = result_obj_array.count(); - if (index_ != -1 && is_alias_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("alias index is not -1", K(ret), K(index_)); - } else if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (result_obj_array.count() == len) { - if (NULL != expr_ && is_alias_) { - ret = expr_->calc(ctx, calc_item, result_obj_array); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("cant calc expr", K(ret)); + + if (!table_name_.empty()) { + if (OB_FAIL(sql_string.append_fmt("%.*s.", table_name_.length(), table_name_.ptr()))) { + LOG_WARN("fail to append table name", K(table_name_), K(ret)); } } - ObProxyExpr::print_proxy_expr(this); - - return ret; -} -int ObProxyExprShardingConst::to_sql_string(ObSqlString& sql_string) -{ - int ret = OB_SUCCESS; - if (obj_.get_type() == ObIntType) { - ret = sql_string.append_fmt("%ld", obj_.get_int()); - } else { - ObString str = obj_.get_string(); - if (is_column_) { - ret = sql_string.append_fmt("%.*s", str.length(), str.ptr()); - } else { - ret = sql_string.append_fmt("'%.*s'", str.length(), str.ptr()); + if (OB_SUCC(ret)) { + if (OB_FAIL(sql_string.append_fmt("%.*s", column_name_.length(), column_name_.ptr()))) { + LOG_WARN("fail to append table name", K(column_name_), K(ret)); } } - if (OB_FAIL(ret)) { - LOG_WARN("fail to sql_string", K(obj_)); - } + return ret; } @@ -270,7 +247,7 @@ int ObProxyExprColumn::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem LOG_WARN("push back obj failed", K(ret), K(result_obj)); } } else { - ret = OB_ERR_COULUMN_VALUE_NOT_MATCH; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("sql_column_value value type invalid", K(sql_column_value.value_type_)); } } @@ -289,13 +266,52 @@ int ObProxyExprColumn::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem return ret; } +int ObProxyGroupItem::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + ObIArray &result_obj_array) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KP_(expr), K(ret)); + } else if (OB_FAIL(expr_->calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("fail to calc expr", K(ret)); + } + return ret; +} + +int ObProxyGroupItem::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + ObIArray &result_obj_array) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KP_(expr), K(ret)); + } else if (OB_FAIL(expr_->calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("fail to calc expr", K(ret)); + } + return ret; +} + +int ObProxyGroupItem::to_column_string(ObSqlString& sql_string) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(expr_->to_column_string(sql_string))) { + LOG_WARN("to column string failed", K(ret)); + } + + return ret; +} + int ObProxyOrderItem::to_sql_string(ObSqlString& sql_string) { int ret = OB_SUCCESS; if (OB_ISNULL(expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); - } else if (OB_FAIL(expr_->to_sql_string(sql_string))) { + } else if (OB_FAIL(expr_->to_column_string(sql_string))) { LOG_WARN("to sql_string failed", K(ret)); } else if (order_direction_ == NULLS_FIRST_ASC){ if (OB_FAIL(sql_string.append(" ASC"))) { @@ -307,6 +323,14 @@ int ObProxyOrderItem::to_sql_string(ObSqlString& sql_string) return ret; } +ObProxyFuncExpr::~ObProxyFuncExpr() +{ + for (int64_t i = 0; i < param_array_.count(); i++) { + ObProxyExpr *expr = param_array_.at(i); + expr->~ObProxyExpr(); + } +} + int ObProxyFuncExpr::calc_param_expr(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObSEArray, 4> ¶m_result, @@ -403,40 +427,6 @@ int ObProxyFuncExpr::check_varchar_empty(const common::ObObj& result) return ret; } -int ObProxyShardingAliasExpr::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array) -{ - int ret = OB_SUCCESS; - int64_t len = result_obj_array.count(); - if (param_array_.count() != 2) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("wrong param array count", K(ret), K(param_array_.count())); - } else if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (result_obj_array.count() == len && OB_FAIL(param_array_.at(0)->calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc failed", K(ret), K(*param_array_.at(0))); - } - ObProxyExpr::print_proxy_expr(this); - - return ret; -} - -int ObProxyShardingAliasExpr::to_sql_string(ObSqlString& sql_string) -{ - int ret = OB_SUCCESS; - if (param_array_.count() != 2) { - LOG_WARN("unexpected count", K(param_array_.count())); - ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(param_array_.at(0)->to_sql_string(sql_string))) { - LOG_WARN("to sql_string failed", K(ret)); - } else if (OB_FAIL(sql_string.append(" AS "))){ - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(param_array_.at(1)->to_sql_string(sql_string))) { - LOG_WARN("to sql_string failed", K(ret)); - } - return ret; -} - int ObProxyExprHash::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array) @@ -784,10 +774,14 @@ int ObProxyExprDiv::calc(const ObProxyExprCtx &ctx, common::ObSEArray, 4> param_result_array; int cnt = 0; int64_t len = result_obj_array.count(); + + if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_ && !has_agg_) { + if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("calc expr failed", K(ret)); + } + } - if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (len == result_obj_array.count()) { + if (OB_SUCC(ret) && len == result_obj_array.count()) { if (2 != param_array_.count()) { ret = OB_EXPR_CALC_ERROR; LOG_WARN("div should have two param", K(ret), K(param_array_.count())); @@ -813,18 +807,38 @@ int ObProxyExprDiv::calc(const ObProxyExprCtx &ctx, ObObj obj1 = param_result.at(0); ObObj obj2 = param_result.at(1); ObObj result_obj; - if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { + if (ObDoubleTC == obj1.get_type_class() || ObDoubleTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get double obj failed", K(obj1), K(obj2), K(ret)); + } else if (fabs(obj2.get_double()) == 0.0) { + result_obj.set_null(); + } else { + double obj1_d = obj1.get_double(); + double obj2_d = obj2.get_double(); + result_obj.set_double(obj1_d / obj2_d); + } + } else if (ObFloatTC == obj1.get_type_class() || ObFloatTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get float obj failed", K(obj1), K(obj2), K(ret)); + } else { + float obj1_f = obj1.get_float(); + float obj2_f = obj2.get_float(); + result_obj.set_float(obj1_f / obj2_f); + } + } else if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { number::ObNumber res_nmb; - if (OB_FAIL(get_number_obj_for_calc(ctx.allocator_, obj1, obj2))) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { LOG_WARN("get number obj failed", K(ret), K(obj1), K(obj2)); } else if (OB_UNLIKELY(obj2.get_number().is_zero())) { result_obj.set_null(); } else if (OB_FAIL(obj1.get_number().div(obj2.get_number(), res_nmb, *ctx.allocator_))) { LOG_WARN("failed to div numbers", K(ret), K(obj1), K(obj2)); } else { - if (ctx.scale_ >= 0) { - if (OB_FAIL(res_nmb.trunc(ctx.scale_))) { - LOG_WARN("failed to trunc result number", K(ret), K(res_nmb), K(ctx.scale_)); + if (ctx.scale_ >= 0 || accuracy_.get_scale() >= 0) { + int64_t scale = ctx.scale_ >= 0 ? ctx.scale_ : accuracy_.get_scale(); + if (OB_FAIL(res_nmb.round(scale))) { + LOG_WARN("failed to round result number", K(res_nmb), + K(scale), K(accuracy_), K(ctx.scale_), K(ret)); } } if (OB_SUCC(ret)) { @@ -869,9 +883,13 @@ int ObProxyExprAdd::calc(const ObProxyExprCtx &ctx, int cnt = 0; int64_t len = result_obj_array.count(); - if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (len == result_obj_array.count()) { + if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_ && !has_agg_) { + if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("calc expr failed", K(ret)); + } + } + + if (OB_SUCC(ret) && len == result_obj_array.count()) { if (2 != param_array_.count()) { ret = OB_EXPR_CALC_ERROR; LOG_WARN("div should have two param", K(ret), K(param_array_.count())); @@ -897,16 +915,34 @@ int ObProxyExprAdd::calc(const ObProxyExprCtx &ctx, ObObj obj1 = param_result.at(0); ObObj obj2 = param_result.at(1); ObObj result_obj; - if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { + if (ObDoubleTC == obj1.get_type_class() || ObDoubleTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get double obj failed", K(obj1), K(obj2), K(ret)); + } else { + double obj1_d = obj1.get_double(); + double obj2_d = obj2.get_double(); + result_obj.set_double(obj1_d + obj2_d); + } + } else if (ObFloatTC == obj1.get_type_class() || ObFloatTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get float obj failed", K(obj1), K(obj2), K(ret)); + } else { + float obj1_f = obj1.get_float(); + float obj2_f = obj2.get_float(); + result_obj.set_float(obj1_f + obj2_f); + } + } else if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { number::ObNumber res_nmb; - if (OB_FAIL(get_number_obj_for_calc(ctx.allocator_, obj1, obj2))) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { LOG_WARN("get number obj failed", K(ret), K(obj1), K(obj2)); } else if (OB_FAIL(obj1.get_number().add(obj2.get_number(), res_nmb, *ctx.allocator_))) { LOG_WARN("failed to div numbers", K(ret), K(obj1), K(obj2)); } else { - if (ctx.scale_ >= 0) { - if (OB_FAIL(res_nmb.trunc(ctx.scale_))) { - LOG_WARN("failed to trunc result number", K(ret), K(res_nmb), K(ctx.scale_)); + if (ctx.scale_ >= 0 || accuracy_.get_scale() >= 0) { + int64_t scale = ctx.scale_ >= 0 ? ctx.scale_ : accuracy_.get_scale(); + if (OB_FAIL(res_nmb.round(scale))) { + LOG_WARN("failed to round result number", K(res_nmb), + K(scale), K(accuracy_), K(ctx.scale_), K(ret)); } } if (OB_SUCC(ret)) { @@ -951,9 +987,13 @@ int ObProxyExprSub::calc(const ObProxyExprCtx &ctx, int cnt = 0; int64_t len = result_obj_array.count(); - if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (len == result_obj_array.count()) { + if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_ && !has_agg_) { + if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("calc expr failed", K(ret)); + } + } + + if (OB_SUCC(ret) && len == result_obj_array.count()) { if (2 != param_array_.count()) { ret = OB_EXPR_CALC_ERROR; LOG_WARN("div should have two param", K(ret), K(param_array_.count())); @@ -979,16 +1019,34 @@ int ObProxyExprSub::calc(const ObProxyExprCtx &ctx, ObObj obj1 = param_result.at(0); ObObj obj2 = param_result.at(1); ObObj result_obj; - if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { + if (ObDoubleTC == obj1.get_type_class() || ObDoubleTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get double obj failed", K(obj1), K(obj2), K(ret)); + } else { + double obj1_d = obj1.get_double(); + double obj2_d = obj2.get_double(); + result_obj.set_double(obj1_d - obj2_d); + } + } else if (ObFloatTC == obj1.get_type_class() || ObFloatTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get float obj failed", K(obj1), K(obj2), K(ret)); + } else { + float obj1_f = obj1.get_float(); + float obj2_f = obj2.get_float(); + result_obj.set_float(obj1_f - obj2_f); + } + } else if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { number::ObNumber res_nmb; - if (OB_FAIL(get_number_obj_for_calc(ctx.allocator_, obj1, obj2))) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { LOG_WARN("get number obj failed", K(ret), K(obj1), K(obj2)); } else if (OB_FAIL(obj1.get_number().sub(obj2.get_number(), res_nmb, *ctx.allocator_))) { LOG_WARN("failed to div numbers", K(ret), K(obj1), K(obj2)); } else { - if (ctx.scale_ >= 0) { - if (OB_FAIL(res_nmb.trunc(ctx.scale_))) { - LOG_WARN("failed to trunc result number", K(ret), K(res_nmb), K(ctx.scale_)); + if (ctx.scale_ >= 0 || accuracy_.get_scale() >= 0) { + int64_t scale = ctx.scale_ >= 0 ? ctx.scale_ : accuracy_.get_scale(); + if (OB_FAIL(res_nmb.round(scale))) { + LOG_WARN("failed to round result number", K(res_nmb), + K(scale), K(accuracy_), K(ctx.scale_), K(ret)); } } if (OB_SUCC(ret)) { @@ -1033,9 +1091,13 @@ int ObProxyExprMul::calc(const ObProxyExprCtx &ctx, int cnt = 0; int64_t len = result_obj_array.count(); - if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { - LOG_WARN("calc expr failed", K(ret)); - } else if (len == result_obj_array.count()) { + if (-1 != index_ && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_ && !has_agg_) { + if (OB_FAIL(ObProxyExpr::calc(ctx, calc_item, result_obj_array))) { + LOG_WARN("calc expr failed", K(ret)); + } + } + + if (OB_SUCC(ret) && len == result_obj_array.count()) { if (2 != param_array_.count()) { ret = OB_EXPR_CALC_ERROR; LOG_WARN("div should have two param", K(ret), K(param_array_.count())); @@ -1061,16 +1123,34 @@ int ObProxyExprMul::calc(const ObProxyExprCtx &ctx, ObObj obj1 = param_result.at(0); ObObj obj2 = param_result.at(1); ObObj result_obj; - if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { + if (ObDoubleTC == obj1.get_type_class() || ObDoubleTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get double obj failed", K(obj1), K(obj2), K(ret)); + } else { + double obj1_d = obj1.get_double(); + double obj2_d = obj2.get_double(); + result_obj.set_double(obj1_d * obj2_d); + } + } else if (ObFloatTC == obj1.get_type_class() || ObFloatTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get float obj failed", K(obj1), K(obj2), K(ret)); + } else { + float obj1_f = obj1.get_float(); + float obj2_f = obj2.get_float(); + result_obj.set_float(obj1_f * obj2_f); + } + } else if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { number::ObNumber res_nmb; - if (OB_FAIL(get_number_obj_for_calc(ctx.allocator_, obj1, obj2))) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { LOG_WARN("get number obj failed", K(ret), K(obj1), K(obj2)); } else if (OB_FAIL(obj1.get_number().mul(obj2.get_number(), res_nmb, *ctx.allocator_))) { LOG_WARN("failed to div numbers", K(ret), K(obj1), K(obj2)); } else { - if (ctx.scale_ >= 0) { - if (OB_FAIL(res_nmb.trunc(ctx.scale_))) { - LOG_WARN("failed to trunc result number", K(ret), K(res_nmb), K(ctx.scale_)); + if (ctx.scale_ >= 0 || accuracy_.get_scale() >= 0) { + int64_t scale = ctx.scale_ >= 0 ? ctx.scale_ : accuracy_.get_scale(); + if (OB_FAIL(res_nmb.round(scale))) { + LOG_WARN("failed to round result number", K(res_nmb), + K(scale), K(accuracy_), K(ctx.scale_), K(ret)); } } if (OB_SUCC(ret)) { @@ -1307,138 +1387,90 @@ int ObProxyExprAvg::calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &c common::ObIArray &result_obj_array) { int ret = OB_SUCCESS; - UNUSED(ctx); - if (sum_index_ >= 0 && count_index_ >= 0 && ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_) { - if (sum_index_ >= calc_item.obj_array_->count() || count_index_ >= calc_item.obj_array_->count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can't calc avg expr", K(ret), K(sum_index_), K(count_index_)); - } else { - int64_t len = calc_item.obj_array_->count(); - ObObj obj; - ObObj left = *(calc_item.obj_array_->at(len - 1 - sum_index_)); - ObObj right = *(calc_item.obj_array_->at(len - 1 - count_index_)); - number::ObNumber res_nmb; - if (OB_FAIL(get_number_obj_for_calc(ctx.allocator_, left, right))) { - LOG_WARN("get number obj failed", K(ret), K(left), K(right)); - } else if (OB_UNLIKELY(right.get_number().is_zero())) { - obj.set_null(); - } else if (OB_FAIL(left.get_number().div(right.get_number(), res_nmb, *ctx.allocator_))) { - LOG_WARN("failed to div numbers", K(ret), K(left), K(right)); + if (ObProxyExprCalcItem::FROM_OBJ_ARRAY == calc_item.source_) { + if (OB_NOT_NULL(sum_expr_) && OB_NOT_NULL(count_expr_)) { + common::ObSEArray param_result; + if (sum_expr_->calc(ctx, calc_item, param_result)) { + LOG_WARN("fail to calc sum", K(ret)); + } else if (count_expr_->calc(ctx, calc_item, param_result)) { + LOG_WARN("fail to calc count", K(ret)); + } else if (OB_UNLIKELY(2 != param_result.count())) { + ret = OB_EXPR_CALC_ERROR; + LOG_WARN("avg not enough param", "count", param_result.count(), K(ret)); } else { - if (ctx.scale_ >= 0) { - int64_t scale = ctx.scale_; - if (scale > number::ObNumber::MAX_SCALE) { - scale = number::ObNumber::MAX_SCALE; + ObObj obj1 = param_result.at(0); + ObObj obj2 = param_result.at(1); + ObObj result_obj; + if (ObDoubleTC == obj1.get_type_class() || ObDoubleTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get double obj failed", K(obj1), K(obj2), K(ret)); + } else if (fabs(obj2.get_double()) == 0.0) { + result_obj.set_null(); + } else { + double obj1_d = obj1.get_double(); + double obj2_d = obj2.get_double(); + result_obj.set_double(obj1_d / obj2_d); } - if (OB_FAIL(res_nmb.trunc(scale))) { - LOG_WARN("failed to trunc result number", K(ret), K(res_nmb), K(ctx.scale_), K(scale)); + } else if (ObFloatTC == obj1.get_type_class() || ObFloatTC == obj2.get_type_class()) { + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get float obj failed", K(obj1), K(obj2), K(ret)); + } else { + float obj1_f = obj1.get_float(); + float obj2_f = obj2.get_float(); + result_obj.set_float(obj1_f / obj2_f); + } + } else if (ObIntTC != obj1.get_type_class() || ObIntTC != obj2.get_type_class()) { + number::ObNumber res_nmb; + if (OB_FAIL((get_obj_for_calc(ctx.allocator_, obj1, obj2)))) { + LOG_WARN("get number obj failed", K(ret), K(obj1), K(obj2)); + } else if (OB_UNLIKELY(obj2.get_number().is_zero())) { + result_obj.set_null(); + } else if (OB_FAIL(obj1.get_number().div(obj2.get_number(), res_nmb, *ctx.allocator_))) { + LOG_WARN("failed to div numbers", K(ret), K(obj1), K(obj2)); + } else { + if (ctx.scale_ >= 0 || accuracy_.get_scale() >= 0) { + int64_t scale = ctx.scale_ >= 0 ? ctx.scale_ : accuracy_.get_scale(); + if (OB_FAIL(res_nmb.round(scale))) { + LOG_WARN("failed to round result number", K(res_nmb), + K(scale), K(accuracy_), K(ctx.scale_), K(ret)); + } + } + if (OB_SUCC(ret)) { + result_obj.set_number(res_nmb); + } + } + } else { + if (OB_FAIL(get_int_obj(param_result.at(0), obj1))) { + LOG_WARN("get int obj failed", K(ret)); + } else if (OB_FAIL(get_int_obj(param_result.at(1), obj2))) { + LOG_WARN("get int obj failed", K(ret)); + } else { + int64_t num1 = obj1.get_int(); + int64_t num2 = obj2.get_int(); + if (0 == num2) { + ret = OB_EXPR_CALC_ERROR; + LOG_WARN("div failed, num2 is 0", K(ret)); + } else { + result_obj.set_int(num1 / num2); + } } } - if (OB_SUCC(ret)) { - obj.set_number(res_nmb); - } - } - if (OB_SUCC(ret) && OB_FAIL(result_obj_array.push_back(obj))) { - LOG_WARN("result obj push back failed", K(ret)); + if (OB_SUCC(ret) && OB_FAIL(result_obj_array.push_back(result_obj))) { + LOG_WARN("result obj array push back failed", K(ret)); + } } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can't calc avg expr", KP(sum_expr_), KP(count_expr_), K(ret)); } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can't calc avg expr", K(ret), K(sum_index_), K(count_index_)); } - ObProxyExpr::print_proxy_expr(this); - return ret; -} - -static int func_expr_for_two_args_to_sql_string(ObSqlString& sql_string, - common::ObSEArray& param_aray, - const char* op) -{ - int ret = OB_SUCCESS; - if (2 != param_aray.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should have two param", K(ret), K(param_aray.count())); - } else if (OB_FAIL(sql_string.append("("))) { - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(param_aray.at(0)->to_sql_string(sql_string))){ - LOG_WARN("to sql_string failed", K(ret)); - } else if (OB_FAIL(sql_string.append(op))) { - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(param_aray.at(1)->to_sql_string(sql_string))){ - LOG_WARN("to sql_string failed", K(ret)); - } else if (OB_FAIL(sql_string.append(")"))) { - LOG_WARN("append failed", K(ret)); - } - return ret; -} + ObProxyExpr::print_proxy_expr(this); -static int func_expr_for_one_args_to_sql_string(ObSqlString& sql_string, - common::ObSEArray& param_aray, - const char* op) -{ - int ret = OB_SUCCESS; - if (1 != param_aray.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("should have one param", "op", op, K(ret), K(param_aray.count())); - } else if (OB_FAIL(sql_string.append(op))) { - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(sql_string.append("("))) { - LOG_WARN("append failed", K(ret)); - } else if (OB_FAIL(param_aray.at(0)->to_sql_string(sql_string))) { - LOG_WARN("to sql_string failed", K(ret)); - } else if (OB_FAIL(sql_string.append(")"))) { - LOG_WARN("append failed", K(ret)); - } return ret; } -int ObProxyExprDiv::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_two_args_to_sql_string(sql_string, param_array_, "/"); -} - -int ObProxyExprAdd::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_two_args_to_sql_string(sql_string, param_array_, "+"); -} - -int ObProxyExprMul::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_two_args_to_sql_string(sql_string, param_array_, "*"); -} - -int ObProxyExprSub::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_two_args_to_sql_string(sql_string, param_array_, "-"); -} - -int ObProxyExprSum::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_one_args_to_sql_string(sql_string, param_array_, "SUM"); -} - -int ObProxyExprCount::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_one_args_to_sql_string(sql_string, param_array_, "COUNT"); -} - -int ObProxyExprAvg::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_one_args_to_sql_string(sql_string, param_array_, "AVG"); -} - -int ObProxyExprMax::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_one_args_to_sql_string(sql_string, param_array_, "MAX"); -} - -int ObProxyExprMin::to_sql_string(ObSqlString& sql_string) -{ - return func_expr_for_one_args_to_sql_string(sql_string, param_array_, "MIN"); -} - } // end opsql } // end obproxy } // end oceanbase diff --git a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.h b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.h index bf9a120f32abe1f4f706dca13e8d9c3ddff09579..cda8fca6ae2dfe5f89f937399243884285528842 100644 --- a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.h +++ b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr.h @@ -55,6 +55,8 @@ struct ObProxyExprCalcItem { FROM_INVALID }; + ObProxyExprCalcItem() : sql_result_(NULL), obj_array_(NULL), source_(FROM_SQL_FIELD) {} + ObProxyExprCalcItem(obutils::SqlFieldResult *sql_result) : sql_result_(sql_result), obj_array_(NULL), source_(FROM_SQL_FIELD) {} @@ -70,8 +72,8 @@ struct ObProxyExprCalcItem { class ObProxyExpr { public: - explicit ObProxyExpr() : type_(OB_PROXY_EXPR_TYPE_NONE), index_(-1), has_agg_(0), - has_alias_(0), is_func_expr_(0), reserved_(0) {} + explicit ObProxyExpr() : type_(OB_PROXY_EXPR_TYPE_NONE), index_(-1), accuracy_(-1), has_agg_(0), + is_func_expr_(0), reserved_(0), alias_name_() {} ~ObProxyExpr() {} void set_expr_type(const ObProxyExprType type) { type_ = type; } @@ -82,21 +84,29 @@ public: common::ObIArray &result_obj_array); bool has_agg() const { return has_agg_ == 1; } - bool has_alias() const { return has_alias_ == 1; } void set_index(int64_t index) { index_ = index; } int64_t get_index() const { return index_; } + void set_accuracy(common::ObAccuracy accuracy) { accuracy_ = accuracy; } + common::ObAccuracy get_accuracy() const { return accuracy_; } bool is_func_expr() const { return is_func_expr_ == 1; } - bool is_star_expr(); + bool is_star_expr() { return OB_PROXY_EXPR_TYPE_STAR == type_; } int64_t to_string(char *buf, int64_t buf_len) const; - virtual int to_sql_string(common::ObSqlString& sql_string) - { - int ret = common::OB_SUCCESS; - UNUSED(sql_string); - return ret; + virtual int to_sql_string(common::ObSqlString& sql_string); + virtual int to_column_string(common::ObSqlString& sql_string) { + return sql_string.append(expr_name_); } static void print_proxy_expr(ObProxyExpr *root); + void set_alias_name(const char *buf, const int64_t length) { + alias_name_.assign_ptr(buf, static_cast(length)); + } + void set_alias_name(const ObString &alias_name) { alias_name_ = alias_name; } + ObString& get_alias_name() { return alias_name_; } + void set_expr_name(const char *buf, const int64_t length) { + expr_name_.assign_ptr(buf, static_cast(length)); + } + ObString& get_expr_name() { return expr_name_; } bool is_agg() { @@ -107,16 +117,17 @@ public: || OB_PROXY_EXPR_TYPE_FUNC_AVG == type_; } - bool is_alias(); public: ObProxyExprType type_; int64_t index_; + common::ObAccuracy accuracy_; struct { uint16_t has_agg_ : 1;// whethere hava agg func, 1 means true - uint16_t has_alias_ : 1; // whether hava alias, 1 means true uint16_t is_func_expr_ : 1; - uint16_t reserved_ : 13; + uint16_t reserved_ : 14; }; + common::ObString expr_name_; + common::ObString alias_name_; }; class ObProxyExprConst : public ObProxyExpr @@ -128,47 +139,75 @@ public: void set_object(const common::ObObj& obj) { obj_ = obj; } virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); + virtual int to_column_string(common::ObSqlString& sql_string); protected: common::ObObj obj_; }; // add for save sharding const column -class ObProxyExprShardingConst : public ObProxyExprConst +class ObProxyExprShardingConst : public ObProxyExpr { public: - explicit ObProxyExprShardingConst() : expr_(NULL), is_column_(false) , is_alias_(false) {} + explicit ObProxyExprShardingConst() {} virtual ~ObProxyExprShardingConst() {} - virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array); - virtual int to_sql_string(common::ObSqlString& sql_string); - void set_is_column(bool is_column) - { - is_column_ = is_column; - } - void set_is_alias(bool is_alias) - { - is_alias_ = is_alias; - } - ObProxyExpr* expr_; +}; + +class ObProxyExprTable : public ObProxyExpr +{ public: - bool is_column_; - bool is_alias_; + explicit ObProxyExprTable() : database_name_(), table_name_() {} + ~ObProxyExprTable() {} + + void set_database_name(const char *buf, const int32_t length) { database_name_.assign_ptr(buf, length); } + ObString& get_database_name() { return database_name_; } + void set_table_name(const char *buf, const int32_t length) { table_name_.assign_ptr(buf, length); } + ObString& get_table_name() { return table_name_; } +private: + common::ObString database_name_; + common::ObString table_name_; }; class ObProxyExprColumn : public ObProxyExpr { public: - explicit ObProxyExprColumn() : column_name_() {} + explicit ObProxyExprColumn() : real_table_name_(), table_name_(), column_name_() {} ~ObProxyExprColumn() {} virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - void set_column_name(char *buf, const int32_t length) { column_name_.assign(buf, length); } + virtual int to_column_string(common::ObSqlString& sql_string); + void set_column_name(const char *buf, const int32_t length) { column_name_.assign_ptr(buf, length); } + ObString &get_column_name() { return column_name_; } + void set_table_name(const char *buf, const int32_t length) { table_name_.assign_ptr(buf, length); } + ObString &get_table_name() { return table_name_; } + void set_real_table_name(const ObString &real_table_name) { real_table_name_ = real_table_name; } + ObString &get_real_table_name() { return real_table_name_; } private: + common::ObString real_table_name_; + common::ObString table_name_; common::ObString column_name_; }; +class ObProxyExprStar : public ObProxyExpr +{ +public: + explicit ObProxyExprStar() : table_name_() {} + ~ObProxyExprStar() {} + + virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + common::ObIArray &result_obj_array) { + UNUSED(ctx); + UNUSED(calc_item); + UNUSED(result_obj_array); + return OB_SUCCESS; + }; + void set_table_name(const char *buf, const int32_t length) { table_name_.assign_ptr(buf, length); } + ObString &get_table_name() { return table_name_; } +private: + common::ObString table_name_; +}; + enum ObProxyOrderDirection { NULLS_FIRST_ASC = 0, // Forward, NULLs first @@ -179,25 +218,33 @@ enum ObProxyOrderDirection MAX_DIR, // invalid }; -class ObProxyOrderItem : public ObProxyExpr +class ObProxyGroupItem : public ObProxyExpr { public: - explicit ObProxyOrderItem() : order_direction_(NULLS_FIRST_ASC) {} - virtual ~ObProxyOrderItem() {} - virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array) - { - int ret = common::OB_SUCCESS; - if (OB_ISNULL(expr_)) { - ret = common::OB_ERR_UNEXPECTED; - } else { - ret = expr_->calc(ctx, calc_item, result_obj_array); + explicit ObProxyGroupItem() : expr_(NULL) {} + virtual ~ObProxyGroupItem() { + if (NULL != expr_) { + expr_->~ObProxyExpr(); } - return ret; } - virtual int to_sql_string(common::ObSqlString& sql_string); + virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + common::ObIArray &result_obj_array); + virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + common::ObIArray &result_obj_array); + virtual int to_column_string(common::ObSqlString& sql_string); + void set_expr(ObProxyExpr* expr) { expr_ = expr; } + ObProxyExpr* get_expr() { return expr_; } public: ObProxyExpr* expr_; +}; + +class ObProxyOrderItem : public ObProxyGroupItem +{ +public: + explicit ObProxyOrderItem() : order_direction_(NULLS_FIRST_ASC) {} + virtual ~ObProxyOrderItem() {} + virtual int to_sql_string(common::ObSqlString& sql_string); +public: ObProxyOrderDirection order_direction_; }; @@ -205,7 +252,7 @@ class ObProxyFuncExpr : public ObProxyExpr { public: explicit ObProxyFuncExpr() : param_array_(common::ObModIds::OB_PROXY_SHARDING_EXPR, common::OB_MALLOC_NORMAL_BLOCK_SIZE) { is_func_expr_ = true; } - virtual ~ObProxyFuncExpr() {} + virtual ~ObProxyFuncExpr(); int add_param_expr(ObProxyExpr* expr) { return param_array_.push_back(expr); } int calc_param_expr(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, @@ -224,18 +271,6 @@ protected: common::ObSEArray param_array_; }; -class ObProxyShardingAliasExpr : public ObProxyFuncExpr -{ -public: - explicit ObProxyShardingAliasExpr() {} - virtual ~ObProxyShardingAliasExpr() {} - - virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array); - virtual int to_sql_string(common::ObSqlString& sql_string); - -}; - class ObProxyExprHash : public ObProxyFuncExpr { public: @@ -279,7 +314,6 @@ public: ~ObProxyExprDiv() {} int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprAdd : public ObProxyFuncExpr @@ -289,8 +323,6 @@ public: ~ObProxyExprAdd() {} int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprSub : public ObProxyFuncExpr @@ -300,8 +332,6 @@ public: ~ObProxyExprSub() {} int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprMul : public ObProxyFuncExpr @@ -311,7 +341,6 @@ public: ~ObProxyExprMul() {} int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprTestLoad : public ObProxyFuncExpr @@ -329,7 +358,6 @@ class ObProxyExprSum : public ObProxyFuncExpr public: explicit ObProxyExprSum() {} ~ObProxyExprSum() {} - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprCount : public ObProxyFuncExpr @@ -337,22 +365,30 @@ class ObProxyExprCount : public ObProxyFuncExpr public: explicit ObProxyExprCount() {} ~ObProxyExprCount() {} - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprAvg : public ObProxyFuncExpr { public: - explicit ObProxyExprAvg() : sum_index_(-1), count_index_(-1) {} - ~ObProxyExprAvg() {} + explicit ObProxyExprAvg() : sum_expr_(NULL), count_expr_(NULL) {} + ~ObProxyExprAvg() { + if (NULL != sum_expr_) { + sum_expr_->~ObProxyExprSum(); + } + + if (NULL != count_expr_) { + count_expr_->~ObProxyExprCount(); + } + } int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, common::ObIArray &result_obj_array); - virtual int to_sql_string(common::ObSqlString& sql_string); - void set_sum_index(int64_t index) { sum_index_ = index; } - void set_count_index(int64_t index) { count_index_ = index; } + void set_sum_expr(ObProxyExprSum* sum_expr) { sum_expr_ = sum_expr; } + ObProxyExprSum *get_sum_expr() { return sum_expr_; } + void set_count_expr(ObProxyExprCount *count_expr) { count_expr_ = count_expr; } + ObProxyExprCount *get_count_expr() { return count_expr_; } private: - int64_t sum_index_; - int64_t count_index_; + ObProxyExprSum *sum_expr_; + ObProxyExprCount *count_expr_; }; class ObProxyExprMax : public ObProxyFuncExpr @@ -360,7 +396,6 @@ class ObProxyExprMax : public ObProxyFuncExpr public: explicit ObProxyExprMax() {} ~ObProxyExprMax() {} - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprMin : public ObProxyFuncExpr @@ -368,7 +403,6 @@ class ObProxyExprMin : public ObProxyFuncExpr public: explicit ObProxyExprMin() {} ~ObProxyExprMin() {} - virtual int to_sql_string(common::ObSqlString& sql_string); }; class ObProxyExprSplit : public ObProxyFuncExpr @@ -377,8 +411,8 @@ public: explicit ObProxyExprSplit() {} ~ObProxyExprSplit() {} - virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, - common::ObIArray &result_obj_array); + virtual int calc(const ObProxyExprCtx &ctx, const ObProxyExprCalcItem &calc_item, + common::ObIArray &result_obj_array); }; } // end opsql diff --git a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr_type.h b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr_type.h index 09da3ad790fa6d6393f77256b2f60da75b9512ed..6554cb50407929af81e558b81b1409aa3e88d38c 100644 --- a/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr_type.h +++ b/src/obproxy/opsql/func_expr_resolver/proxy_expr/ob_proxy_expr_type.h @@ -18,8 +18,9 @@ typedef enum ObProxyExprType OB_PROXY_EXPR_TYPE_NONE = 0, OB_PROXY_EXPR_TYPE_CONST, OB_PROXY_EXPR_TYPE_SHARDING_CONST, - OB_PROXY_EXPR_TYPE_SHARDING_ALIAS, + OB_PROXY_EXPR_TYPE_TABLE, OB_PROXY_EXPR_TYPE_COLUMN, + OB_PROXY_EXPR_TYPE_STAR, OB_PROXY_EXPR_TYPE_FUNC_HASH, OB_PROXY_EXPR_TYPE_FUNC_SUBSTR, OB_PROXY_EXPR_TYPE_FUNC_CONCAT, @@ -33,6 +34,7 @@ typedef enum ObProxyExprType OB_PROXY_EXPR_TYPE_FUNC_MAX, OB_PROXY_EXPR_TYPE_FUNC_MIN, OB_PROXY_EXPR_TYPE_FUNC_AVG, + OB_PROXY_EXPR_TYPE_FUNC_GROUP, OB_PROXY_EXPR_TYPE_FUNC_ORDER, OB_PROXY_EXPR_TYPE_FUNC_TESTLOAD, OB_PROXY_EXPR_TYPE_FUNC_SPLIT, @@ -53,12 +55,15 @@ const char* get_expr_type_name(int expr_type) case OB_PROXY_EXPR_TYPE_SHARDING_CONST: type_name = "OB_PROXY_EXPR_TYPE_SHARDING_CONST"; break; - case OB_PROXY_EXPR_TYPE_SHARDING_ALIAS: - type_name = "OB_PROXY_EXPR_TYPE_SHARDING_ALIAS"; + case OB_PROXY_EXPR_TYPE_TABLE: + type_name = "OB_PROXY_EXPR_TYPE_TABLE"; break; case OB_PROXY_EXPR_TYPE_COLUMN: type_name = "OB_PROXY_EXPR_TYPE_COLUMN"; break; + case OB_PROXY_EXPR_TYPE_STAR: + type_name = "OB_PROXY_EXPR_TYPE_STAR"; + break; case OB_PROXY_EXPR_TYPE_FUNC_HASH: type_name = "OB_PROXY_EXPR_TYPE_FUNC_HASH"; break; @@ -101,6 +106,9 @@ const char* get_expr_type_name(int expr_type) case OB_PROXY_EXPR_TYPE_FUNC_ORDER: type_name = "OB_PROXY_EXPR_TYPE_FUNC_ORDER"; break; + case OB_PROXY_EXPR_TYPE_FUNC_GROUP: + type_name = "OB_PROXY_EXPR_TYPE_FUNC_GROUP"; + break; case OB_PROXY_EXPR_TYPE_FUNC_TESTLOAD: type_name = "OB_PROXY_EXPR_TYPE_FUNC_TESTLOAD"; break; @@ -112,4 +120,4 @@ const char* get_expr_type_name(int expr_type) } return type_name; } -#endif \ No newline at end of file +#endif diff --git a/src/obproxy/opsql/ob_proxy_parse_define.h b/src/obproxy/opsql/ob_proxy_parse_define.h index 4e1e57cc030d469b94f97c982fb8fb9411b6ad93..cabff3cb61f4b27d133394e313e592b1d1a2587c 100644 --- a/src/obproxy/opsql/ob_proxy_parse_define.h +++ b/src/obproxy/opsql/ob_proxy_parse_define.h @@ -54,6 +54,10 @@ static const int OB_PARSER_ERR_PARSE_SQL = -5001; #endif /* OB_PERF_MODE */ #endif /* OB_ISNULL */ +#ifdef NDEBUG +#define YYDEBUG 0 +#else #define YYDEBUG 1 +#endif #endif /* end of OBPROXY_PARSER_PARSE_DEFINE_H */ diff --git a/src/obproxy/opsql/parser/ob_proxy_parse_result.cpp b/src/obproxy/opsql/parser/ob_proxy_parse_result.cpp index 0716d55dab1feaa41f72cc0e4da7f95578c07620..2af63a5a084c40242a67ed514942f6f2eff0351a 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parse_result.cpp +++ b/src/obproxy/opsql/parser/ob_proxy_parse_result.cpp @@ -114,7 +114,7 @@ const char* get_print_stmt_name(const ObProxyBasicStmtType type) str_ret = "BEGIN"; break; case OBPROXY_T_SELECT_TX_RO: - str_ret = "SELECT"; + str_ret = "SELECT_TX_RO"; break; case OBPROXY_T_SET_AC_0: str_ret = "SET"; @@ -128,6 +128,9 @@ const char* get_print_stmt_name(const ObProxyBasicStmtType type) case OBPROXY_T_SET_ROUTE_ADDR: str_ret = "SET_ROUTE_ADDR"; break; + case OBPROXY_T_SELECT_PROXY_VERSION: + str_ret = "SELECT_PROXY_VERSION"; + break; // use last session case OBPROXY_T_SHOW_WARNINGS: @@ -147,8 +150,8 @@ const char* get_print_stmt_name(const ObProxyBasicStmtType type) case OBPROXY_T_SET_CHARSET: case OBPROXY_T_SET_PASSWORD: case OBPROXY_T_SET_DEFAULT: - case OBPROXY_T_SET_OB_READ_CONSISTENCY: - case OBPROXY_T_SET_TX_READ_ONLY: + case OBPROXY_T_SET_OB_READ_CONSISTENCY: + case OBPROXY_T_SET_TX_READ_ONLY: str_ret = "SET"; break; case OBPROXY_T_USE_DB: @@ -339,6 +342,9 @@ const char* get_obproxy_stmt_name(const ObProxyBasicStmtType type) case OBPROXY_T_SELECT_TX_RO: str_ret = "OBPROXY_T_SELECT_TX_RO"; break; + case OBPROXY_T_SELECT_PROXY_VERSION: + str_ret = "OBPROXY_T_SELECT_PROXY_VERSION"; + break; case OBPROXY_T_SET_AC_0: str_ret = "OBPROXY_T_SET_AC_0"; break; @@ -500,6 +506,18 @@ const char* get_obproxy_sub_stmt_name(const ObProxyBasicStmtSubType type) case OBPROXY_T_SUB_SHOW_DB_VERSION: str_ret = "OBPROXY_T_SUB_SHOW_DB_VERSION"; break; + case OBPROXY_T_SUB_SELECT_DATABASE: + str_ret = "OBPROXY_T_SUB_SELECT_DATABASE"; + break; + case OBPROXY_T_SUB_DESC_TABLE: + str_ret = "OBPROXY_T_SUB_DESC_TABLE"; + break; + case OBPROXY_T_SUB_CREATE_TABLE: + str_ret = "OBPROXY_T_SUB_CREATE_TABLE"; + break; + case OBPROXY_T_SUB_CREATE_INDEX: + str_ret = "OBPROXY_T_SUB_CREATE_INDEX"; + break; case OBPROXY_T_SUB_MAX: str_ret = "OBPROXY_T_SUB_MAX"; break; diff --git a/src/obproxy/opsql/parser/ob_proxy_parse_result.h b/src/obproxy/opsql/parser/ob_proxy_parse_result.h index b0e6cc54ed061045abfc820669f9ff119f61a9f5..5cf911bd14a0f5aba347be094bf1653143d54365 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parse_result.h +++ b/src/obproxy/opsql/parser/ob_proxy_parse_result.h @@ -91,6 +91,7 @@ typedef enum ObProxyBasicStmtType OBPROXY_T_PING_PROXY, OBPROXY_T_SELECT_ROUTE_ADDR, OBPROXY_T_SET_ROUTE_ADDR, + OBPROXY_T_SELECT_PROXY_VERSION, // use last session OBPROXY_T_SHOW_WARNINGS, @@ -201,6 +202,10 @@ typedef enum ObProxyBasicStmtSubType //desc OBPROXY_T_SUB_DESC_TABLE, + //ddl + OBPROXY_T_SUB_CREATE_TABLE, + OBPROXY_T_SUB_CREATE_INDEX, + OBPROXY_T_SUB_MAX } ObProxyBasicStmtSubType; diff --git a/src/obproxy/opsql/parser/ob_proxy_parser.h b/src/obproxy/opsql/parser/ob_proxy_parser.h index 97f505438e1ec4ab59d2f533c207e18f99083f16..3941f3c97cd498ac8f44085fb5f9c029777b20ab 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser.h +++ b/src/obproxy/opsql/parser/ob_proxy_parser.h @@ -61,10 +61,6 @@ private: inline ObProxyParser::ObProxyParser(common::ObIAllocator &allocator, ObProxyParseMode parse_mode) : allocator_(allocator), parse_mode_(parse_mode) { - common::ObArenaAllocator *arena_allocator = dynamic_cast(&allocator); - if (NULL != arena_allocator) { - arena_allocator->set_mod_id(common::ObModIds::OB_PROXY_SHARDING_PARSE); - } } inline int ObProxyParser::init_result(ObProxyParseResult &parse_result, const char *start_pos) @@ -83,13 +79,9 @@ inline int ObProxyParser::init_result(ObProxyParseResult &parse_result, const ch parse_result.cmd_info_.err_type_ = OBPROXY_T_ERR_INVALID; for (int64_t i = 0; i < OBPROXY_ICMD_MAX_VALUE_COUNT; ++i) { parse_result.cmd_info_.integer_[i] = -1; - MEMSET(parse_result.cmd_info_.string_ + i, 0, sizeof(ObProxyParseString)); } - MEMSET(&parse_result.call_parse_info_, 0, sizeof(ObProxyCallParseInfo)); - MEMSET(&parse_result.simple_route_info_, 0, sizeof(ObProxySimpleRouteParseInfo)); parse_result.has_simple_route_info_ = false; parse_result.placeholder_list_idx_ = 0; - MEMSET(&parse_result.text_ps_execute_parse_info_, 0, sizeof(ObProxyTextPsExecuteParseInfo)); parse_result.text_ps_name_.str_len_ = 0; parse_result.has_shard_comment_ = false; @@ -103,14 +95,11 @@ inline int ObProxyParser::init_result(ObProxyParseResult &parse_result, const ch parse_result.dbmesh_route_info_.head_ = NULL; parse_result.dbmesh_route_info_.tail_ = NULL; parse_result.dbmesh_route_info_.index_count_ = 0; - memset(&parse_result.dbmesh_route_info_.index_tb_name_, 0, - OBPROXY_MAX_HINT_INDEX_COUNT * sizeof(ObProxyParseString)); parse_result.set_parse_info_.node_count_ = 0; parse_result.set_parse_info_.head_ = NULL; parse_result.set_parse_info_.tail_ = NULL; - memset(&parse_result.dbp_route_info_, 0, sizeof(ObDbpRouteInfo)); parse_result.comment_begin_ = NULL; parse_result.comment_end_ = NULL; diff --git a/src/obproxy/opsql/parser/ob_proxy_parser.l b/src/obproxy/opsql/parser/ob_proxy_parser.l index 6e30c5e51426f2600877b05106f0dcf54b2a9c4a..008511631635520cfb4b78f9015f38c5d19208bc 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser.l +++ b/src/obproxy/opsql/parser/ob_proxy_parser.l @@ -359,6 +359,7 @@ alter_proxyresource (alter{space}+proxyresource) ping_proxy (ping{space}+proxy) kill_proxysession (kill{space}+proxysession) kill_globalsession (kill{space}+globalsession) +select_proxy_version (select{space}+proxy_version{space}*({leftbracket}|{multi_byte_left_parenthesis})({rightbracket}|{multi_byte_right_parenthesis})) select_obproxy_route_addr (select{space}+@obproxy_route_addr) set_obproxy_route_addr (set{space}+@obproxy_route_addr) @@ -395,6 +396,8 @@ ALTER { return ALTER; } TRUNCATE { return TRUNCATE; } RENAME { return RENAME; } INDEX { return INDEX; } +TABLE { return TABLE; } +UNIQUE { return UNIQUE; } /* ps stmt */ USING { return USING; } @@ -419,6 +422,7 @@ UNION { return UNION;} AS { return AS; } WHERE { return WHERE; } VALUES { return VALUES; } +VALUE { return VALUES; } EXPLAIN { SET_HAS_EXPLAIN(); return EXPLAIN; } DESC { SET_HAS_EXPLAIN(); return DESC; } DESCRIBE { SET_HAS_EXPLAIN(); return DESCRIBE; } @@ -547,6 +551,7 @@ IDC { RETURN_NON_RESERVED_KEYWORD(IDC); } {show_tables} { SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_TABLES; } {show_create_table} { SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_CREATE_TABLE; } {select_database} { SET_DML_STMT(OBPROXY_T_SELECT); return SELECT_DATABASE; } +{select_proxy_version} { SET_DML_STMT(OBPROXY_T_SELECT_PROXY_VERSION); return SELECT_PROXY_VERSION; } /*alter config*/ {alter_proxyconfig} { SET_ICMD_STMT(OBPROXY_T_ICMD_ALTER_CONFIG); return ALTER_PROXYCONFIG; } diff --git a/src/obproxy/opsql/parser/ob_proxy_parser.y b/src/obproxy/opsql/parser/ob_proxy_parser.y index 98e0100a5edc81cc31d7c7cbfef61e7d4355531e..d3c63e32e9fd23af7df91634c198d92a6c1606d0 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser.y +++ b/src/obproxy/opsql/parser/ob_proxy_parser.y @@ -86,7 +86,6 @@ do {\ if ((OBPROXY_T_INVALID < result->cur_stmt_type_ && result->cur_stmt_type_ < OBPROXY_T_ICMD_MAX) || (OBPROXY_T_PING_PROXY == result->cur_stmt_type_)) {\ result->cmd_info_.err_type_ = OBPROXY_T_ERR_PARSE;\ }\ - result->sub_stmt_type_ = OBPROXY_T_SUB_INVALID;\ handle_stmt_end(result);\ HANDLE_ACCEPT();\ } while (0); @@ -285,7 +284,7 @@ extern void *obproxy_parse_malloc(const size_t nbyte, void *malloc_pool); /* dummy token */ %token DUMMY_WHERE_CLAUSE DUMMY_INSERT_CLAUSE /* reserved keyword */ -%token SELECT DELETE INSERT UPDATE REPLACE MERGE SHOW SET CALL CREATE DROP ALTER TRUNCATE RENAME +%token SELECT DELETE INSERT UPDATE REPLACE MERGE SHOW SET CALL CREATE DROP ALTER TRUNCATE RENAME TABLE UNIQUE %token GRANT REVOKE ANALYZE PURGE COMMENT %token FROM DUAL %token PREPARE EXECUTE USING @@ -325,7 +324,7 @@ extern void *obproxy_parse_malloc(const size_t nbyte, void *malloc_pool); %token SHOW_PROXYTRACE %token SHOW_PROXYINFO BINARY UPGRADE IDC %token SHOW_TOPOLOGY GROUP_NAME SHOW_DB_VERSION -%token SHOW_DATABASES SHOW_TABLES SELECT_DATABASE SHOW_CREATE_TABLE +%token SHOW_DATABASES SHOW_TABLES SELECT_DATABASE SHOW_CREATE_TABLE SELECT_PROXY_VERSION %token ALTER_PROXYCONFIG %token ALTER_PROXYRESOURCE %token PING_PROXY @@ -392,12 +391,17 @@ comment_stmt: comment_expr_list select_stmt ddl_stmt: mysql_ddl_stmt | oracle_ddl_stmt -mysql_ddl_stmt: CREATE { result->cur_stmt_type_ = OBPROXY_T_CREATE; } +mysql_ddl_stmt: CREATE create_dll_expr { result->cur_stmt_type_ = OBPROXY_T_CREATE; } | DROP { result->cur_stmt_type_ = OBPROXY_T_DROP; } | ALTER { result->cur_stmt_type_ = OBPROXY_T_ALTER; } | TRUNCATE { result->cur_stmt_type_ = OBPROXY_T_TRUNCATE; } | RENAME { result->cur_stmt_type_ = OBPROXY_T_RENAME; } +create_dll_expr : /* empty */ + | TABLE { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_TABLE; } + | INDEX { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_INDEX; } + | UNIQUE INDEX { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_INDEX; } + text_ps_from_stmt: select_stmt {} | insert_stmt {} | replace_stmt {} @@ -468,10 +472,14 @@ select_tx_read_only_stmt: SELECT TX_READ_ONLY { result->cur_stmt_type_ = OBPROXY | SELECT TX_READ_ONLY select_expr_list FROM fromlist | SELECT TX_READ_ONLY expr_list +select_proxy_version_stmt: SELECT_PROXY_VERSION + | SELECT_PROXY_VERSION AS var_name { result->col_name_ = $3; } + set_autocommit_0_stmt: SET AUTOCOMMIT_0 { result->cur_stmt_type_ = OBPROXY_T_SET_AC_0; } | SET AUTOCOMMIT_0 expr_list hooked_stmt: select_tx_read_only_stmt {} + | select_proxy_version_stmt {} | set_autocommit_0_stmt {} | select_obproxy_route_addr_stmt {} | set_obproxy_route_addr_stmt {} diff --git a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.c b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.c index 1c813f2b21212a4a8f6816fa395e0131e9bb2b9a..d07893b86f32c4d6f1ce1c401f1f412479994276 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.c +++ b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.c @@ -348,8 +348,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ); yyg->yy_hold_char = *yy_cp; \ yyg->yy_c_buf_p = yy_cp; -#define YY_NUM_RULES 319 -#define YY_END_OF_BUFFER 320 +#define YY_NUM_RULES 323 +#define YY_END_OF_BUFFER 324 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -357,236 +357,240 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static yyconst flex_int16_t yy_accept[2073] = +static yyconst flex_int16_t yy_accept[2102] = { 0, - 130, 130, 173, 173, 0, 0, 0, 0, 186, 186, - 214, 214, 0, 0, 0, 0, 0, 0, 242, 242, + 134, 134, 177, 177, 0, 0, 0, 0, 190, 190, + 218, 218, 0, 0, 0, 0, 0, 0, 246, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 291, 291, 0, 0, - 0, 0, 309, 309, 313, 313, 0, 0, 152, 152, - 0, 0, 320, 318, 140, 140, 136, 266, 140, 130, - 255, 137, 136, 128, 318, 136, 136, 129, 135, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 272, 318, 318, 175, 319, 173, 172, 175, - - 175, 166, 173, 173, 173, 173, 173, 173, 175, 179, - 178, 179, 181, 181, 319, 181, 181, 181, 181, 181, - 181, 181, 181, 181, 181, 181, 181, 181, 191, 191, - 186, 191, 191, 186, 186, 191, 218, 217, 217, 214, - 212, 218, 214, 213, 214, 216, 215, 214, 214, 214, - 214, 214, 218, 218, 224, 223, 223, 220, 224, 221, - 224, 224, 226, 225, 225, 225, 226, 226, 226, 244, - 241, 241, 242, 236, 237, 244, 242, 240, 242, 242, - 242, 242, 242, 244, 244, 246, 245, 319, 319, 252, - 251, 251, 249, 247, 248, 252, 252, 252, 254, 253, - - 253, 253, 254, 254, 254, 261, 262, 319, 256, 257, - 319, 267, 268, 319, 274, 275, 277, 278, 294, 292, - 292, 282, 292, 291, 283, 281, 294, 293, 294, 291, - 291, 291, 291, 291, 284, 294, 294, 317, 317, 301, - 300, 300, 298, 300, 299, 296, 297, 301, 301, 301, - 301, 311, 310, 310, 304, 310, 309, 305, 302, 303, - 311, 309, 309, 309, 311, 311, 314, 313, 315, 313, - 162, 162, 160, 154, 154, 158, 154, 152, 157, 160, - 151, 160, 160, 150, 156, 155, 155, 152, 152, 152, - 159, 160, 160, 110, 108, 108, 108, 110, 109, 110, - - 110, 110, 140, 0, 132, 0, 140, 130, 0, 131, - 129, 0, 134, 176, 134, 129, 130, 130, 130, 39, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 8, 0, 133, 0, 0, 173, 0, 174, 166, - 166, 173, 173, 173, 173, 173, 173, 0, 0, 177, - 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, - - 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 188, 0, 186, 0, 187, - 0, 190, 186, 186, 0, 189, 214, 0, 219, 214, - 214, 214, 214, 214, 214, 214, 214, 0, 0, 0, - 222, 0, 0, 0, 225, 0, 0, 242, 242, 242, - 0, 243, 242, 242, 242, 242, 242, 242, 0, 0, - 0, 0, 0, 250, 0, 0, 0, 253, 0, 0, - 261, 0, 0, 263, 0, 0, 264, 256, 0, 0, - 258, 0, 0, 259, 0, 269, 0, 0, 0, 270, - 274, 273, 277, 276, 292, 0, 292, 291, 0, 0, - - 291, 291, 291, 291, 291, 291, 0, 0, 0, 300, - 0, 300, 0, 0, 0, 0, 310, 0, 310, 309, - 0, 309, 309, 309, 0, 0, 313, 313, 0, 154, - 0, 154, 152, 150, 0, 153, 153, 150, 152, 0, - 152, 152, 152, 0, 0, 108, 0, 108, 0, 0, - 0, 0, 140, 0, 0, 134, 0, 0, 130, 49, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 37, 130, 130, 130, 130, 130, 130, 104, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - - 130, 130, 130, 130, 130, 130, 130, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 13, 130, - 130, 130, 130, 130, 130, 130, 130, 130, 47, 130, - 130, 130, 130, 130, 130, 138, 139, 128, 173, 173, - 173, 173, 173, 173, 163, 164, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 186, 186, 214, - 214, 214, 214, 214, 214, 214, 214, 217, 213, 223, - 221, 225, 242, 242, 242, 242, 242, 242, 242, 242, - 241, 238, 239, 240, 246, 245, 251, 248, 253, 265, - 0, 265, 0, 0, 260, 0, 260, 0, 0, 271, - - 0, 271, 0, 0, 292, 0, 0, 0, 0, 291, - 291, 291, 291, 291, 291, 281, 0, 300, 0, 0, - 296, 297, 310, 0, 309, 309, 309, 302, 303, 313, - 0, 154, 0, 0, 153, 0, 0, 152, 0, 0, - 152, 152, 152, 151, 108, 0, 0, 0, 140, 140, - 140, 0, 0, 134, 0, 134, 134, 130, 130, 130, - 130, 130, 130, 14, 130, 130, 130, 130, 130, 130, - 130, 130, 43, 86, 18, 130, 130, 130, 130, 130, - 130, 130, 130, 68, 130, 130, 130, 130, 55, 121, - 130, 50, 130, 130, 130, 130, 130, 130, 65, 130, - - 130, 130, 130, 130, 130, 130, 45, 130, 130, 130, - 130, 130, 130, 130, 130, 0, 0, 7, 130, 130, - 84, 130, 130, 130, 130, 130, 130, 130, 87, 130, - 130, 130, 130, 16, 130, 46, 173, 173, 173, 173, - 173, 168, 0, 0, 0, 0, 192, 0, 0, 0, - 0, 0, 0, 186, 186, 214, 214, 214, 214, 214, - 214, 214, 214, 242, 229, 242, 242, 242, 242, 242, - 242, 0, 0, 0, 0, 0, 0, 292, 292, 292, - 0, 0, 0, 0, 291, 291, 280, 291, 291, 291, - 0, 300, 300, 300, 0, 0, 310, 310, 310, 0, - - 309, 309, 309, 312, 161, 154, 154, 154, 0, 0, - 153, 0, 153, 153, 0, 0, 152, 152, 152, 108, - 108, 108, 0, 106, 0, 140, 19, 130, 130, 32, - 9, 130, 130, 130, 130, 130, 60, 130, 130, 130, - 130, 130, 130, 130, 130, 130, 130, 26, 34, 130, - 130, 130, 22, 130, 0, 0, 130, 48, 130, 6, - 130, 130, 130, 36, 130, 0, 0, 130, 29, 122, - 59, 130, 130, 130, 130, 130, 130, 130, 130, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, - 10, 130, 63, 130, 130, 38, 130, 130, 23, 130, - - 130, 130, 40, 173, 171, 173, 173, 173, 227, 0, - 0, 0, 0, 0, 182, 0, 0, 0, 186, 184, - 214, 214, 214, 214, 214, 214, 214, 214, 242, 242, - 242, 242, 242, 242, 235, 0, 0, 265, 0, 0, - 0, 0, 260, 0, 0, 0, 0, 271, 0, 0, - 292, 0, 0, 0, 291, 291, 0, 0, 0, 0, - 291, 291, 291, 316, 300, 0, 310, 309, 309, 309, - 154, 0, 0, 152, 152, 152, 108, 107, 0, 0, - 130, 130, 52, 130, 11, 130, 130, 17, 130, 130, - 2, 130, 62, 130, 130, 130, 130, 130, 35, 130, - - 54, 3, 0, 0, 0, 130, 130, 0, 0, 0, - 130, 130, 78, 130, 0, 0, 130, 130, 21, 130, - 27, 130, 130, 130, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 295, 295, 0, 0, + 0, 0, 313, 313, 317, 317, 0, 0, 156, 156, + 0, 0, 324, 322, 144, 144, 140, 270, 144, 134, + 259, 141, 140, 132, 322, 140, 140, 133, 139, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 276, 322, 322, 179, 323, 177, 176, 179, + + 179, 170, 177, 177, 177, 177, 177, 177, 179, 183, + 182, 183, 185, 185, 323, 185, 185, 185, 185, 185, + 185, 185, 185, 185, 185, 185, 185, 185, 195, 195, + 190, 195, 195, 190, 190, 195, 222, 221, 221, 218, + 216, 222, 218, 217, 218, 220, 219, 218, 218, 218, + 218, 218, 222, 222, 228, 227, 227, 224, 228, 225, + 228, 228, 230, 229, 229, 229, 230, 230, 230, 248, + 245, 245, 246, 240, 241, 248, 246, 244, 246, 246, + 246, 246, 246, 248, 248, 250, 249, 323, 323, 256, + 255, 255, 253, 251, 252, 256, 256, 256, 258, 257, + + 257, 257, 258, 258, 258, 265, 266, 323, 260, 261, + 323, 271, 272, 323, 278, 279, 281, 282, 298, 296, + 296, 286, 296, 295, 287, 285, 298, 297, 298, 295, + 295, 295, 295, 295, 288, 298, 298, 321, 321, 305, + 304, 304, 302, 304, 303, 300, 301, 305, 305, 305, + 305, 315, 314, 314, 308, 314, 313, 309, 306, 307, + 315, 313, 313, 313, 315, 315, 318, 317, 319, 317, + 166, 166, 164, 158, 158, 162, 158, 156, 161, 164, + 155, 164, 164, 154, 160, 159, 159, 156, 156, 156, + 163, 164, 164, 113, 111, 111, 111, 113, 112, 113, + + 113, 113, 144, 0, 136, 0, 144, 134, 0, 135, + 133, 0, 138, 180, 138, 133, 134, 134, 134, 41, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 8, 0, 137, 0, 0, 177, 0, 178, + 170, 170, 177, 177, 177, 177, 177, 177, 0, 0, + 181, 0, 0, 0, 205, 0, 0, 0, 0, 0, + + 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 192, 0, 190, 0, + 191, 0, 194, 190, 190, 0, 193, 218, 0, 223, + 218, 218, 218, 218, 218, 218, 218, 218, 0, 0, + 0, 226, 0, 0, 0, 229, 0, 0, 246, 246, + 246, 0, 247, 246, 246, 246, 246, 246, 246, 0, + 0, 0, 0, 0, 254, 0, 0, 0, 257, 0, + 0, 265, 0, 0, 267, 0, 0, 268, 260, 0, + 0, 262, 0, 0, 263, 0, 273, 0, 0, 0, + 274, 278, 277, 281, 280, 296, 0, 296, 295, 0, + + 0, 295, 295, 295, 295, 295, 295, 0, 0, 0, + 304, 0, 304, 0, 0, 0, 0, 314, 0, 314, + 313, 0, 313, 313, 313, 0, 0, 317, 317, 0, + 158, 0, 158, 156, 154, 0, 157, 157, 154, 156, + 0, 156, 156, 156, 0, 0, 111, 0, 111, 0, + 0, 0, 0, 144, 0, 0, 138, 0, 0, 134, + 52, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 39, 134, 134, 134, 134, 134, 134, 107, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 13, + 134, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 50, 134, 134, 134, 134, 134, 134, 142, 143, 132, + 177, 177, 177, 177, 177, 177, 167, 168, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, + 190, 218, 218, 218, 218, 218, 218, 218, 218, 221, + 217, 227, 225, 229, 246, 246, 246, 246, 246, 246, + 246, 246, 245, 242, 243, 244, 250, 249, 255, 252, + 257, 269, 0, 269, 0, 0, 264, 0, 264, 0, + + 0, 275, 0, 275, 0, 0, 296, 0, 0, 0, + 0, 295, 295, 295, 295, 295, 295, 285, 0, 304, + 0, 0, 300, 301, 314, 0, 313, 313, 313, 306, + 307, 317, 0, 158, 0, 0, 157, 0, 0, 156, + 0, 0, 156, 156, 156, 155, 111, 0, 0, 0, + 144, 144, 144, 0, 0, 138, 0, 138, 138, 134, + 134, 134, 134, 134, 134, 14, 134, 134, 134, 134, + 134, 134, 134, 134, 46, 89, 18, 134, 134, 134, + 134, 134, 134, 134, 134, 71, 134, 134, 134, 134, + 58, 125, 134, 53, 134, 134, 134, 134, 134, 134, + + 68, 134, 134, 134, 134, 134, 134, 134, 48, 134, + 134, 134, 134, 134, 134, 134, 134, 0, 0, 7, + 134, 134, 87, 134, 134, 134, 134, 134, 134, 134, + 134, 134, 90, 134, 134, 134, 134, 16, 134, 49, + 177, 177, 177, 177, 177, 172, 0, 0, 0, 0, + 196, 0, 0, 0, 0, 0, 0, 190, 190, 218, + 218, 218, 218, 218, 218, 218, 218, 246, 233, 246, + 246, 246, 246, 246, 246, 0, 0, 0, 0, 0, + 0, 296, 296, 296, 0, 0, 0, 0, 295, 295, + 284, 295, 295, 295, 0, 304, 304, 304, 0, 0, + + 314, 314, 314, 0, 313, 313, 313, 316, 165, 158, + 158, 158, 0, 0, 157, 0, 157, 157, 0, 0, + 156, 156, 156, 111, 111, 111, 0, 109, 0, 144, + 19, 134, 134, 34, 9, 134, 134, 134, 134, 134, + 63, 134, 134, 134, 134, 134, 134, 134, 134, 134, + 134, 28, 36, 134, 134, 134, 22, 134, 0, 0, + 134, 51, 134, 6, 134, 134, 134, 38, 134, 0, + 0, 134, 31, 126, 62, 134, 134, 134, 134, 134, + 134, 134, 134, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 134, 10, 23, 134, 66, 134, 134, + + 40, 134, 134, 134, 25, 44, 134, 134, 42, 177, + 175, 177, 177, 177, 231, 0, 0, 0, 0, 0, + 186, 0, 0, 0, 190, 188, 218, 218, 218, 218, + 218, 218, 218, 218, 246, 246, 246, 246, 246, 246, + 239, 0, 0, 269, 0, 0, 0, 0, 264, 0, + 0, 0, 0, 275, 0, 0, 296, 0, 0, 0, + 295, 295, 0, 0, 0, 0, 295, 295, 295, 320, + 304, 0, 314, 313, 313, 313, 158, 0, 0, 156, + 156, 156, 111, 110, 0, 0, 134, 134, 55, 134, + 11, 134, 134, 17, 134, 134, 2, 134, 65, 134, + + 134, 134, 134, 134, 37, 134, 57, 3, 0, 0, + 0, 134, 134, 0, 0, 0, 134, 134, 81, 134, + 0, 0, 134, 134, 21, 134, 29, 134, 134, 134, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 134, 79, + 134, 134, 24, 4, 134, 43, 134, 134, 174, 177, + 177, 173, 0, 0, 0, 0, 206, 0, 0, 203, + 0, 190, 218, 218, 218, 215, 218, 218, 213, 218, + 232, 246, 246, 246, 246, 246, 0, 0, 0, 0, + 0, 0, 295, 295, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 292, 295, 295, 299, 313, 313, 313, + 0, 0, 156, 152, 156, 0, 0, 30, 134, 32, + 134, 134, 15, 60, 0, 0, 0, 134, 27, 45, + 134, 134, 134, 134, 0, 0, 0, 0, 0, 134, + 134, 0, 0, 35, 99, 134, 0, 26, 103, 5, + 134, 96, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 130, 76, 130, 130, 4, 130, 41, 130, - 130, 170, 173, 173, 169, 0, 0, 0, 0, 202, - 0, 0, 199, 0, 186, 214, 214, 214, 211, 214, - 214, 209, 214, 228, 242, 242, 242, 242, 242, 0, - 0, 0, 0, 0, 0, 291, 291, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 288, 291, 291, 295, - - 309, 309, 309, 0, 0, 152, 148, 152, 0, 0, - 28, 130, 30, 130, 130, 15, 57, 0, 0, 0, - 130, 25, 42, 130, 130, 130, 130, 0, 0, 0, - 0, 0, 130, 130, 0, 0, 33, 96, 130, 0, - 24, 100, 5, 130, 93, 130, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 134, 134, 0, 0, 0, + 106, 134, 134, 177, 177, 0, 0, 0, 0, 0, + 0, 0, 0, 190, 218, 218, 218, 218, 218, 218, + + 218, 246, 246, 246, 246, 246, 0, 0, 0, 295, + 295, 0, 0, 0, 0, 0, 0, 295, 295, 313, + 313, 313, 0, 0, 156, 153, 0, 134, 134, 134, + 0, 0, 47, 134, 134, 134, 134, 0, 0, 0, + 0, 134, 134, 0, 0, 150, 134, 0, 0, 0, + 0, 12, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 130, 130, 130, - 0, 0, 0, 103, 130, 130, 173, 173, 0, 0, - 0, 0, 0, 0, 0, 0, 186, 214, 214, 214, - 214, 214, 214, 214, 242, 242, 242, 242, 242, 0, - - 0, 0, 291, 291, 0, 0, 0, 0, 0, 0, - 291, 291, 309, 309, 309, 0, 0, 152, 149, 0, - 130, 130, 130, 0, 0, 44, 130, 130, 130, 130, - 0, 0, 0, 0, 130, 130, 0, 0, 146, 130, - 0, 0, 0, 0, 12, 130, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 70, 134, 20, 0, + 0, 134, 64, 177, 177, 0, 0, 197, 187, 198, + 0, 201, 204, 189, 218, 218, 207, 208, 218, 211, + + 214, 238, 246, 234, 246, 246, 0, 0, 0, 295, + 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 295, 295, 313, 313, 313, 0, 0, + 156, 0, 85, 134, 134, 0, 0, 147, 33, 134, + 134, 134, 0, 0, 148, 0, 0, 134, 134, 0, + 0, 0, 0, 54, 0, 0, 0, 130, 0, 0, + 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 67, 130, - 20, 0, 0, 130, 61, 173, 173, 0, 0, 193, - 183, 194, 0, 197, 200, 185, 214, 214, 203, 204, - 214, 207, 210, 234, 242, 230, 242, 242, 0, 0, - - 0, 291, 291, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 291, 291, 309, 309, 309, - 0, 0, 152, 0, 82, 130, 130, 0, 0, 143, - 31, 130, 130, 130, 0, 0, 144, 0, 0, 130, - 130, 0, 0, 0, 0, 51, 0, 0, 0, 126, - 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 134, 0, 0, 146, 86, 177, + 177, 0, 0, 0, 218, 218, 218, 246, 246, 246, + + 0, 0, 0, 295, 295, 283, 0, 0, 0, 0, + 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, + 0, 295, 290, 313, 313, 311, 0, 0, 156, 0, + 80, 69, 0, 0, 0, 0, 129, 56, 134, 0, + 0, 0, 0, 0, 0, 134, 134, 0, 0, 0, + 0, 122, 0, 0, 149, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 130, 0, 0, 142, 83, - 173, 173, 0, 0, 0, 214, 214, 214, 242, 242, - 242, 0, 0, 0, 291, 291, 279, 0, 0, 0, - - 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, - 0, 0, 291, 286, 309, 309, 307, 0, 0, 152, - 0, 77, 66, 0, 0, 0, 0, 125, 53, 130, - 0, 0, 0, 0, 0, 0, 130, 130, 0, 0, - 0, 0, 118, 0, 0, 145, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 130, 0, 0, 0, 0, 173, 173, - 0, 196, 195, 214, 206, 205, 242, 242, 232, 0, - 0, 0, 0, 0, 291, 285, 0, 0, 0, 0, + 0, 0, 0, 134, 0, 0, 0, 0, 177, 177, + 0, 200, 199, 218, 210, 209, 246, 246, 236, 0, - 0, 0, 291, 306, 309, 0, 0, 0, 0, 152, - 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, - 0, 0, 130, 130, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 295, 289, 0, 0, 0, 0, + 0, 0, 295, 310, 313, 0, 0, 0, 0, 156, + 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, + 0, 0, 134, 134, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 116, 0, 67, 0, 0, 0, 0, 177, + 177, 0, 218, 237, 246, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 283, 295, 313, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 134, 0, 0, 0, 0, 134, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 113, 0, 64, 0, 0, 0, 0, 173, 173, - 0, 214, 233, 242, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 279, 291, 309, 0, 0, + 0, 0, 0, 73, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 91, 0, 0, 0, 0, 101, + 0, 0, 177, 177, 0, 218, 235, 0, 0, 0, + 0, 0, 0, 294, 0, 0, 283, 295, 313, 0, + 0, 0, 0, 151, 0, 0, 0, 0, 0, 61, + 0, 0, 0, 134, 150, 0, 0, 0, 0, 0, - 130, 0, 0, 0, 0, 130, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 70, 0, 72, 0, 0, 0, 0, 0, 0, + 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 78, 0, 0, 0, 0, 0, 97, + 100, 0, 169, 177, 0, 218, 0, 0, 0, 0, + 295, 313, 0, 0, 0, 0, 147, 0, 148, 0, + 0, 0, 131, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 88, 0, 0, 0, 0, 98, 0, 0, - 173, 173, 0, 214, 231, 0, 0, 0, 0, 0, - 0, 290, 0, 0, 279, 291, 309, 0, 0, 0, - 0, 147, 0, 0, 0, 0, 0, 58, 0, 0, - 0, 130, 146, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, + 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 0, 102, 0, 146, 0, 177, + 0, 218, 0, 0, 293, 291, 312, 0, 0, 0, + 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 75, 0, 0, 0, 0, 0, 94, 97, 0, 165, - 173, 0, 214, 0, 0, 0, 0, 291, 309, 0, - 0, 0, 0, 143, 0, 144, 0, 0, 0, 127, - 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, - 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, - 0, 99, 0, 142, 0, 173, 0, 214, 0, 0, - 289, 287, 308, 0, 0, 0, 0, 0, 0, 0, - 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, - 0, 0, 0, 0, 0, 0, 0, 92, 0, 101, - 0, 173, 198, 208, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, + 0, 0, 95, 0, 104, 0, 177, 202, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 74, 0, 0, 0, 81, 0, 85, - 0, 95, 0, 0, 167, 0, 0, 0, 116, 0, - 0, 119, 0, 115, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 74, 0, 114, 0, 0, 89, 0, - 0, 80, 0, 0, 0, 0, 120, 0, 0, 0, - 0, 0, 74, 0, 70, 0, 0, 0, 0, 79, - - 0, 90, 0, 0, 0, 117, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, + 0, 0, 0, 84, 0, 88, 0, 98, 0, 0, + 171, 0, 0, 0, 120, 0, 0, 123, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, - 0, 0, 0, 289, 0, 0, 0, 0, 124, 73, - 0, 0, 0, 0, 73, 0, 0, 73, 0, 0, - 0, 123, 0, 74, 0, 0, 0, 0, 0, 0, - 73, 0 + 77, 0, 117, 0, 0, 92, 0, 0, 83, 0, + + 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, + 77, 0, 73, 0, 0, 0, 0, 82, 0, 93, + 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, + 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, + 0, 293, 0, 0, 0, 0, 0, 128, 76, 0, + 0, 0, 0, 76, 0, 0, 76, 0, 0, 0, + 127, 0, 77, 0, 0, 0, 0, 0, 0, 76, + + 0 } ; static yyconst flex_int32_t yy_ec[256] = @@ -634,491 +638,499 @@ static yyconst flex_int32_t yy_meta[88] = 1, 1, 1, 1, 1, 2, 1 } ; -static yyconst flex_int16_t yy_base[2171] = +static yyconst flex_int16_t yy_base[2200] = { 0, 0, 0, 87, 0, 172, 173, 185, 0, 272, 0, 359, 0, 445, 449, 462, 466, 0, 0, 553, 0, 639, 642, 651, 662, 673, 684, 441, 446, 447, 451, - 456, 461, 5403, 5402, 5401, 5400, 771, 0, 469, 470, + 456, 461, 5462, 5461, 5460, 5459, 771, 0, 469, 470, 857, 867, 954, 0, 1041, 1128, 476, 478, 1215, 0, - 1301, 1308, 5452, 6130, 1070, 1073, 6130, 5445, 0, 0, - 5441, 6130, 159, 6130, 164, 162, 5437, 470, 6130, 472, + 1301, 1308, 5511, 6183, 1070, 1073, 6183, 5504, 0, 0, + 5500, 6183, 159, 6183, 164, 162, 5496, 470, 6183, 472, 461, 479, 472, 630, 653, 435, 673, 684, 451, 678, - 462, 454, 849, 691, 455, 684, 851, 852, 850, 625, - 873, 636, 5395, 5366, 5360, 6130, 6130, 0, 6130, 470, - - 508, 523, 626, 632, 636, 676, 670, 710, 5359, 6130, - 6130, 639, 6130, 1393, 1478, 1080, 692, 1317, 1086, 6130, - 1103, 1337, 1377, 1355, 1413, 1443, 1563, 5362, 6130, 5436, - 0, 5432, 738, 860, 861, 5387, 6130, 6130, 6130, 0, - 6130, 744, 0, 6130, 0, 6130, 6130, 859, 860, 855, - 859, 1132, 5358, 5353, 6130, 6130, 6130, 6130, 746, 6130, - 5356, 5351, 6130, 6130, 6130, 6130, 752, 5354, 5349, 6130, - 6130, 6130, 859, 6130, 6130, 920, 0, 6130, 0, 865, - 886, 1053, 1044, 5352, 5347, 6130, 6130, 5350, 5345, 6130, - 6130, 6130, 6130, 6130, 6130, 921, 5347, 5342, 6130, 6130, - - 6130, 6130, 929, 5345, 5340, 0, 1469, 0, 0, 1483, - 0, 6130, 1498, 0, 0, 5371, 0, 5370, 6130, 1108, - 1324, 6130, 0, 0, 6130, 6130, 5407, 6130, 5397, 1004, - 1125, 1032, 1054, 1052, 6130, 5339, 5334, 6130, 1088, 6130, - 1332, 1428, 6130, 0, 6130, 6130, 6130, 5402, 1118, 5335, - 5330, 6130, 1440, 1466, 6130, 0, 0, 6130, 6130, 6130, - 5399, 1114, 1129, 1116, 5332, 5327, 6130, 0, 6130, 1120, - 6130, 1125, 6130, 1513, 1521, 6130, 0, 0, 6130, 629, - 6130, 930, 701, 1346, 6130, 6130, 5387, 1123, 1133, 1142, - 6130, 5328, 5322, 6130, 1528, 1537, 0, 5391, 6130, 1131, - - 1143, 5323, 1540, 5322, 6130, 5396, 0, 0, 5391, 6130, - 748, 1554, 1288, 6130, 1382, 1492, 1442, 1509, 1152, 0, - 1134, 1285, 1283, 1280, 1289, 1523, 1308, 1342, 1346, 1352, - 1383, 1533, 1411, 1535, 1556, 1428, 1483, 1503, 1519, 1516, - 1543, 1557, 1553, 1592, 1556, 1562, 1583, 1578, 1583, 1582, - 1591, 1586, 1593, 1606, 1594, 1610, 1621, 1608, 1618, 1606, - 1625, 1627, 1621, 1641, 1636, 1666, 1656, 1634, 1629, 1648, - 1634, 0, 5346, 6130, 5315, 868, 0, 930, 6130, 1041, - 1048, 1648, 1665, 1672, 1677, 1661, 1679, 1021, 1168, 6130, - 1732, 1725, 1740, 6130, 1755, 1762, 1778, 1806, 1821, 1906, - - 5312, 1332, 6130, 1840, 1855, 1847, 1884, 1900, 1895, 1952, - 1972, 1978, 1997, 1977, 5311, 6130, 5385, 0, 5381, 6130, - 1339, 6130, 1666, 1684, 5336, 6130, 0, 1422, 6130, 1674, - 1695, 1682, 1698, 1741, 1730, 1739, 1759, 5307, 5303, 1516, - 6130, 5304, 5215, 1707, 6130, 5181, 5151, 0, 1746, 1737, - 1783, 6130, 1748, 1757, 1779, 1781, 1812, 1815, 5124, 1413, - 5122, 5086, 1833, 6130, 5072, 5069, 1834, 6130, 5016, 1306, - 0, 1871, 5076, 6130, 5058, 4986, 6130, 0, 1875, 5046, - 6130, 5031, 4949, 6130, 2001, 6130, 5022, 5012, 4942, 6130, - 0, 6130, 0, 6130, 1943, 4907, 0, 0, 1993, 1989, - - 1808, 1809, 1823, 1826, 1839, 1849, 4904, 4902, 1898, 2014, - 4901, 0, 2020, 1894, 4883, 1406, 2043, 4798, 0, 0, - 2049, 1889, 1899, 1896, 4768, 1629, 0, 1972, 1976, 2052, - 4764, 0, 0, 1733, 2063, 2053, 2055, 2059, 2076, 2014, - 1986, 1995, 2026, 4755, 4751, 2099, 4750, 0, 2105, 2037, - 2052, 4749, 2112, 4741, 2105, 2103, 2112, 1811, 1920, 0, - 2061, 2061, 2057, 2071, 2079, 2109, 2100, 2100, 2101, 2102, - 2117, 2107, 2121, 2118, 2117, 2108, 2112, 2125, 2117, 2111, - 0, 2117, 2119, 2113, 2126, 2120, 2132, 0, 2128, 2141, - 2149, 2151, 2156, 2150, 2166, 2164, 4763, 2169, 2156, 2162, - - 2160, 2155, 2176, 2162, 2177, 2169, 2179, 2169, 2185, 2185, - 2172, 2190, 2180, 2178, 2182, 2176, 4746, 2193, 2250, 2180, - 2188, 2206, 2207, 2228, 2220, 2220, 2235, 2219, 2220, 2225, - 2219, 2234, 2230, 2231, 2238, 6130, 6130, 6130, 2221, 2243, - 2232, 2247, 2237, 2243, 6130, 6130, 2266, 2269, 2265, 2264, - 1185, 2267, 1347, 2294, 2270, 1397, 2297, 2268, 2279, 2291, - 2274, 2273, 4743, 2283, 2276, 4740, 2294, 6130, 6130, 6130, - 6130, 6130, 2277, 2282, 2281, 2302, 2290, 2287, 2295, 2305, - 6130, 6130, 6130, 6130, 6130, 6130, 6130, 6130, 6130, 6130, - 4700, 4679, 2360, 4594, 6130, 4663, 4652, 2363, 4570, 6130, - - 4639, 4638, 2366, 4561, 2369, 4531, 2307, 2345, 2327, 2337, - 2339, 2341, 2353, 2336, 4554, 6130, 2349, 2381, 4516, 2357, - 6130, 6130, 2385, 4515, 2352, 2347, 4543, 6130, 6130, 2355, - 2354, 2392, 4493, 2399, 2381, 2410, 1932, 1965, 2352, 2369, - 2362, 2376, 2370, 6130, 2430, 4466, 2379, 2382, 1986, 2433, - 2437, 4442, 2007, 2018, 2074, 2315, 2425, 2405, 2399, 2417, - 2407, 2416, 2414, 0, 2428, 2429, 2426, 2416, 2418, 2438, - 2415, 2421, 2424, 0, 0, 2426, 2424, 2445, 2440, 2445, - 2430, 2435, 2443, 0, 4470, 2440, 2436, 2454, 0, 2503, - 4453, 0, 2456, 2468, 2480, 2484, 2475, 2486, 0, 2475, - - 2487, 2519, 2499, 2497, 2478, 2493, 0, 2501, 2494, 2507, - 2498, 2508, 2502, 2509, 2512, 2604, 4409, 2548, 2497, 2498, - 0, 2520, 2518, 2505, 2527, 2520, 2522, 2542, 0, 2539, - 2543, 2548, 2542, 0, 2551, 0, 2552, 2534, 2535, 4418, - 2547, 0, 1366, 2574, 2572, 2578, 6130, 2044, 2587, 2563, - 2562, 2596, 2606, 4416, 2557, 2554, 2579, 2584, 2592, 2601, - 2597, 2603, 2608, 2594, 0, 2599, 2604, 4413, 2613, 2621, - 2622, 2672, 4377, 2680, 4373, 2689, 4367, 2652, 2683, 2693, - 4352, 2634, 2645, 4379, 2667, 2670, 2697, 2677, 4370, 2676, - 2667, 2705, 2711, 2707, 4340, 2679, 2715, 2718, 2721, 4335, - - 2688, 4362, 2699, 0, 6130, 2725, 2734, 2736, 4331, 2723, - 2725, 2728, 2730, 2732, 2686, 2690, 2725, 2728, 2723, 2757, - 2761, 2764, 4329, 6130, 2728, 2767, 2774, 2722, 2753, 0, - 0, 2731, 2746, 2743, 2762, 2747, 0, 2763, 2751, 2765, - 2766, 2763, 2755, 2755, 2767, 2777, 4354, 0, 4346, 2774, - 2766, 2778, 0, 2766, 2838, 4316, 2781, 0, 2773, 2847, - 2793, 2791, 2789, 0, 2793, 2850, 4295, 2796, 0, 0, - 0, 2803, 2827, 2830, 2829, 2838, 2827, 2829, 2826, 2837, - 2839, 2843, 2848, 2848, 2852, 2831, 4293, 2935, 4290, 2848, - 0, 2853, 0, 2857, 2858, 0, 2855, 2860, 0, 2846, - - 2865, 2855, 0, 2858, 0, 4312, 2887, 2884, 6130, 2879, - 2888, 2913, 2910, 2900, 2915, 2916, 2914, 2918, 2892, 0, - 2896, 2914, 4310, 2920, 4308, 2912, 2926, 4302, 2929, 4282, - 2923, 2935, 2928, 4205, 0, 2993, 2996, 3006, 3009, 4169, - 3013, 3021, 3023, 3025, 4160, 3032, 3037, 3041, 3045, 4152, - 2969, 2946, 2988, 2991, 2998, 4180, 3052, 3061, 4215, 4146, - 3006, 3013, 3013, 6130, 3016, 3013, 3069, 4170, 3025, 3030, - 3074, 3031, 3031, 3040, 3047, 3045, 3088, 6130, 3091, 4137, - 3056, 3043, 0, 3052, 0, 3056, 3057, 0, 3075, 3077, - 3118, 3084, 0, 3082, 3077, 3091, 3077, 3086, 0, 3085, - - 0, 3155, 3096, 3094, 4134, 3099, 3108, 3171, 4173, 4100, - 3098, 3113, 0, 3118, 3110, 4097, 3125, 3130, 0, 3134, - 0, 3137, 3138, 3123, 3182, 3151, 3150, 3154, 3152, 3150, - 4110, 3146, 4107, 3156, 3175, 3165, 3166, 3161, 3165, 3182, - 3184, 4072, 3171, 0, 3186, 3170, 3242, 3191, 0, 3185, - 3191, 0, 3181, 3187, 0, 3218, 3221, 3236, 3224, 4128, - 3240, 3242, 4126, 3246, 3231, 3232, 3236, 3231, 0, 3232, - 3241, 0, 3238, 0, 3239, 3245, 3239, 3242, 3243, 3282, - 3302, 3309, 3260, 3255, 3268, 3277, 3276, 3274, 3318, 3327, - 3334, 3349, 3364, 3395, 3355, 4059, 0, 3294, 3295, 6130, - - 3302, 3307, 3315, 3338, 3339, 3339, 0, 3340, 3340, 4046, - 0, 3341, 0, 3353, 3358, 0, 0, 3415, 4113, 4023, - 3364, 0, 0, 3368, 3357, 3375, 3375, 3440, 4087, 4014, - 3387, 3388, 3387, 3392, 2799, 4011, 0, 0, 3394, 3395, - 0, 0, 3444, 3401, 0, 3401, 3467, 4079, 4009, 3419, - 3420, 3403, 3413, 3412, 3433, 3435, 3425, 3430, 3433, 3448, - 3435, 3460, 3447, 3449, 3453, 3464, 3450, 3450, 3454, 3470, - 3508, 4074, 4002, 0, 3471, 3460, 3472, 3469, 3493, 3481, - 3515, 3497, 3521, 3522, 3523, 3524, 3467, 3499, 4025, 3514, - 3516, 3523, 3522, 3526, 3527, 3520, 3521, 3527, 3534, 3521, - - 3522, 3537, 3526, 3527, 3542, 3595, 3601, 3610, 3617, 3993, - 3539, 3555, 3562, 3563, 3569, 3576, 3579, 3590, 0, 3589, - 3602, 3594, 3596, 3271, 3991, 0, 3602, 3592, 3603, 3602, - 3311, 3988, 3616, 3595, 3617, 3606, 3368, 1900, 3647, 3615, - 3606, 3682, 4056, 3975, 0, 3615, 3622, 3639, 3556, 3969, - 3989, 3622, 3985, 3631, 3655, 3631, 3635, 3660, 3645, 3666, - 3672, 3675, 3658, 3680, 3683, 3686, 3676, 3675, 0, 3681, - 0, 3703, 3950, 3674, 0, 3682, 3677, 3706, 3712, 4013, - 4010, 3973, 3688, 3946, 3931, 0, 3859, 3690, 0, 0, - 3695, 0, 0, 0, 3701, 0, 3849, 3699, 3700, 3700, - - 3716, 3714, 3703, 3715, 3768, 3782, 3788, 3798, 3804, 3820, - 3834, 3868, 3885, 3895, 3813, 3724, 3715, 3717, 3740, 3755, - 3767, 3776, 3782, 3769, 0, 3780, 3777, 3743, 2068, 3854, - 0, 3787, 3802, 3809, 3765, 2929, 3863, 3827, 3804, 3822, - 3843, 3905, 3914, 3852, 3792, 0, 3854, 3866, 3761, 0, - 3878, 3868, 3914, 2969, 3932, 3877, 3878, 3886, 3890, 3904, - 3908, 3902, 3941, 3914, 3912, 3928, 3911, 3930, 3917, 3938, - 3936, 3920, 3925, 3943, 3937, 3935, 3965, 2989, 3984, 0, - 3950, 3950, 3979, 3981, 3988, 3953, 3977, 3977, 3964, 3975, - 3982, 3976, 4017, 3788, 3970, 3972, 6130, 4045, 4051, 4060, - - 4078, 4094, 4108, 4117, 4126, 4145, 4154, 4189, 4206, 4221, - 4240, 4249, 3986, 0, 3987, 3990, 0, 4028, 4168, 4027, - 4027, 0, 0, 4173, 4184, 4045, 3748, 0, 0, 4051, - 4197, 4235, 4054, 3739, 4053, 4071, 4082, 4090, 4203, 3731, - 4100, 3728, 6130, 4011, 3200, 4258, 4110, 4141, 4269, 4277, - 4164, 3713, 4190, 4196, 4237, 4221, 4230, 4232, 4238, 4297, - 3706, 4256, 4243, 4260, 4262, 4268, 4269, 4268, 4265, 4357, - 4282, 4269, 4277, 4280, 4325, 4321, 4299, 3700, 4297, 4297, - 4299, 3746, 3741, 4302, 0, 0, 4311, 4320, 0, 4317, - 4354, 4364, 3528, 4312, 4383, 0, 4389, 4432, 4449, 4459, - - 4469, 4476, 4308, 0, 4315, 4340, 4439, 4490, 3519, 4423, - 4440, 4496, 3493, 4371, 3484, 4371, 4517, 3441, 4382, 3439, - 4416, 4452, 3420, 4436, 3385, 4451, 4525, 4499, 4474, 3359, - 4466, 4491, 4549, 3347, 4494, 3331, 4513, 4500, 4518, 4499, - 4520, 4554, 4561, 3307, 3318, 4526, 3302, 4567, 4516, 4546, - 4532, 4536, 4545, 4542, 4557, 4558, 4560, 4561, 4549, 4559, - 4560, 6130, 4563, 0, 4622, 3269, 4562, 3258, 4556, 4563, - 4616, 4589, 0, 4568, 4587, 4646, 4588, 4589, 3254, 3152, - 4600, 4652, 4655, 3147, 4661, 4676, 3163, 3094, 4608, 4682, - 4624, 3062, 3047, 4690, 4699, 3021, 4638, 4653, 2980, 4643, - - 4646, 2972, 4652, 4670, 4662, 4677, 0, 4711, 4718, 2922, - 4685, 2913, 4688, 4704, 2887, 4686, 4703, 4699, 4712, 2901, - 4700, 4741, 2868, 4746, 2854, 4719, 4751, 4712, 4761, 2750, - 4728, 2739, 4732, 4731, 4725, 4734, 4743, 4737, 4733, 4735, - 4734, 4738, 6130, 4757, 4758, 4744, 4743, 6130, 2692, 4744, - 4745, 4762, 4777, 4750, 0, 4822, 4756, 2702, 2582, 4773, - 4837, 6130, 2576, 2538, 4849, 4781, 4786, 4856, 4784, 2519, - 4863, 6130, 2512, 2510, 4795, 4799, 4866, 0, 4870, 4803, - 4811, 4833, 4882, 2357, 2261, 4819, 4828, 4837, 4885, 2287, - 4891, 2173, 4840, 4894, 2081, 2075, 4852, 4897, 1961, 4864, - - 4878, 1947, 4861, 4877, 4869, 4881, 4872, 4886, 4879, 4880, - 6130, 4882, 4878, 4881, 4881, 4899, 6130, 6130, 4925, 0, - 4892, 4918, 4891, 4898, 4896, 4891, 1928, 4929, 4930, 4921, - 1897, 4931, 4923, 4970, 1855, 4991, 1851, 4927, 4942, 0, - 1847, 4994, 4927, 4949, 4998, 1825, 4952, 5010, 5013, 1812, - 4942, 4964, 5016, 1782, 4968, 1766, 4973, 4995, 6130, 4976, - 4996, 4982, 4986, 4987, 5000, 5005, 6130, 4994, 4992, 5010, - 5007, 6130, 5012, 5040, 1751, 5018, 5032, 5004, 5022, 5032, - 6130, 0, 0, 5037, 5038, 5027, 1726, 1716, 5046, 5043, - 5103, 1698, 1717, 1752, 1632, 5044, 5111, 5061, 5041, 1628, - - 1557, 5053, 5047, 5065, 1539, 5052, 5049, 5067, 5080, 6130, - 5077, 5073, 5090, 5091, 5082, 5079, 5087, 6130, 5091, 6130, - 1530, 5082, 1582, 0, 5094, 5109, 5096, 5105, 5095, 5101, - 5103, 1513, 5100, 3316, 1500, 5105, 1520, 1518, 1478, 5108, - 5101, 5103, 5116, 5178, 5127, 5119, 5128, 6130, 5133, 6130, - 5133, 6130, 5156, 5148, 0, 5150, 5160, 5154, 6130, 5165, - 5155, 6130, 5155, 6130, 1452, 1445, 5152, 5156, 5157, 5151, - 5172, 5205, 5159, 5230, 1396, 6130, 5167, 5173, 6130, 5180, - 5185, 6130, 5181, 1423, 5182, 5196, 6130, 5181, 1369, 5203, - 5208, 5209, 5242, 1375, 5247, 1291, 5197, 1280, 5219, 6130, - - 5214, 6130, 5223, 5218, 5225, 6130, 5215, 5217, 5236, 5238, - 5243, 1243, 5240, 5232, 5234, 5229, 5236, 5231, 5247, 5233, - 5250, 5251, 5252, 5245, 5246, 6130, 5292, 5257, 5315, 1257, - 5281, 1143, 1136, 5283, 5288, 6130, 5271, 5296, 5285, 5297, - 5286, 5284, 5278, 6130, 5301, 5303, 5292, 5294, 6130, 5332, - 5310, 5292, 5304, 5307, 5343, 1079, 5306, 5360, 5308, 5308, - 1032, 6130, 5332, 5371, 5323, 5324, 5340, 5338, 5352, 5332, - 5380, 6130, 5466, 5476, 5486, 5496, 5506, 5516, 5526, 5536, - 5546, 5556, 5566, 5576, 5586, 5596, 5606, 5616, 5623, 5628, - 1082, 5634, 5636, 1076, 5645, 5650, 1065, 5651, 5653, 5659, - - 898, 5666, 5676, 5686, 5696, 5706, 5716, 5725, 5734, 876, - 5744, 5754, 860, 857, 5764, 752, 5774, 5781, 5786, 749, - 5792, 5794, 742, 5803, 5808, 725, 5809, 5811, 5817, 703, - 5824, 5834, 5844, 5854, 5864, 5874, 5883, 5892, 695, 5902, - 5912, 662, 654, 5922, 652, 5932, 5942, 546, 498, 5948, - 480, 5952, 478, 5959, 5969, 5979, 5989, 5999, 6009, 6019, - 6029, 6039, 6049, 6059, 6069, 6079, 6089, 6099, 6109, 6119 + 462, 454, 849, 691, 455, 684, 851, 860, 850, 625, + 873, 636, 5454, 5425, 5420, 6183, 6183, 0, 6183, 470, + + 508, 523, 626, 632, 636, 676, 670, 710, 5419, 6183, + 6183, 639, 6183, 1393, 1478, 1080, 692, 1317, 1086, 6183, + 1103, 1337, 1377, 1355, 1413, 1443, 1563, 5422, 6183, 5496, + 0, 5492, 738, 840, 859, 5447, 6183, 6183, 6183, 0, + 6183, 744, 0, 6183, 0, 6183, 6183, 853, 860, 855, + 859, 1132, 5418, 5413, 6183, 6183, 6183, 6183, 746, 6183, + 5416, 5411, 6183, 6183, 6183, 6183, 752, 5414, 5409, 6183, + 6183, 6183, 1026, 6183, 6183, 920, 0, 6183, 0, 859, + 882, 1053, 1121, 5412, 5406, 6183, 6183, 5409, 5404, 6183, + 6183, 6183, 6183, 6183, 6183, 921, 5406, 5401, 6183, 6183, + + 6183, 6183, 928, 5404, 5399, 0, 1469, 0, 0, 1483, + 0, 6183, 1498, 0, 0, 5430, 0, 5429, 6183, 1108, + 1324, 6183, 0, 0, 6183, 6183, 5466, 6183, 5456, 878, + 1125, 1013, 1054, 1047, 6183, 5398, 5393, 6183, 1062, 6183, + 1332, 1428, 6183, 0, 6183, 6183, 6183, 5462, 1118, 5395, + 5390, 6183, 1440, 1466, 6183, 0, 0, 6183, 6183, 6183, + 5459, 1114, 1129, 1116, 5392, 5387, 6183, 0, 6183, 1120, + 6183, 1130, 6183, 1513, 1521, 6183, 0, 0, 6183, 629, + 6183, 931, 701, 1346, 6183, 6183, 5447, 1124, 1135, 1144, + 6183, 5389, 5384, 6183, 1528, 1537, 0, 5453, 6183, 1135, + + 1146, 5386, 1540, 5385, 6183, 5459, 0, 0, 5455, 6183, + 748, 1554, 1288, 6183, 1382, 1492, 1442, 1509, 1288, 0, + 1270, 1290, 1294, 1299, 1313, 1523, 1327, 1544, 1339, 1337, + 1349, 1396, 1449, 1541, 1494, 1488, 1510, 1542, 1550, 1543, + 1575, 1555, 1550, 1561, 1548, 1562, 1583, 1579, 1598, 1593, + 1603, 1593, 1598, 1609, 1597, 1612, 1631, 1607, 1615, 1605, + 1621, 1625, 1627, 1613, 1641, 1629, 1661, 1664, 1660, 1634, + 1649, 1643, 0, 5410, 6183, 5381, 1413, 0, 930, 6183, + 1041, 1048, 1655, 1671, 1672, 1678, 1662, 1683, 1021, 1168, + 6183, 1736, 1732, 1740, 6183, 1755, 1759, 1787, 1795, 1831, + + 1916, 5380, 1422, 6183, 1812, 1849, 1830, 1874, 1912, 1841, + 1934, 1953, 1960, 1991, 1942, 5377, 6183, 5451, 0, 5445, + 6183, 1542, 6183, 1670, 1691, 5398, 6183, 0, 1697, 6183, + 1707, 1729, 1738, 1757, 1766, 1758, 1775, 1806, 5369, 5364, + 1707, 6183, 5366, 5361, 1769, 6183, 5363, 5359, 0, 1793, + 1791, 1770, 6183, 1817, 1815, 1849, 1850, 1854, 1857, 5361, + 1823, 5358, 5356, 1825, 6183, 5336, 5318, 1879, 6183, 5273, + 1122, 0, 2008, 5333, 6183, 5317, 5213, 6183, 0, 2012, + 5251, 6183, 5221, 5084, 6183, 2020, 6183, 5135, 5058, 4987, + 6183, 0, 6183, 0, 6183, 2025, 4926, 0, 0, 2034, + + 2000, 1849, 1849, 1871, 1885, 1887, 1896, 4924, 4903, 1917, + 2038, 4893, 0, 2045, 1953, 4886, 1306, 2048, 4879, 0, + 0, 2053, 1956, 1962, 1984, 4876, 1331, 0, 1994, 2001, + 2063, 4865, 0, 0, 1470, 2066, 2057, 2060, 2064, 1973, + 2038, 2014, 2020, 2018, 4801, 4789, 2093, 4781, 0, 2097, + 2045, 2056, 4780, 2100, 4771, 2108, 2090, 2117, 1509, 1692, + 0, 2083, 2080, 2075, 2092, 2095, 2104, 2097, 2100, 2100, + 2102, 2117, 2108, 2122, 2126, 2125, 2116, 2120, 2133, 2125, + 2120, 0, 2126, 2127, 2123, 2137, 2134, 2143, 0, 2138, + 2149, 2152, 2148, 2155, 2151, 2167, 2165, 4797, 2169, 2156, + + 2162, 2162, 2159, 2180, 2167, 2181, 2173, 2183, 2173, 2190, + 2190, 2177, 2196, 2186, 2184, 2190, 2184, 4785, 2203, 2256, + 2186, 2195, 2212, 2200, 2210, 2236, 2211, 2228, 2228, 2224, + 2228, 2233, 2227, 2240, 2237, 2238, 2245, 6183, 6183, 6183, + 2229, 2252, 2242, 2258, 2248, 2253, 6183, 6183, 2274, 2295, + 2275, 2277, 1097, 2281, 1347, 2291, 2296, 1930, 2303, 2250, + 2285, 2299, 2283, 2283, 4784, 2293, 2287, 4778, 2305, 6183, + 6183, 6183, 6183, 6183, 2288, 2291, 2290, 2311, 2299, 2296, + 2303, 2314, 6183, 6183, 6183, 6183, 6183, 6183, 6183, 6183, + 6183, 6183, 4820, 4817, 2342, 4698, 6183, 4768, 4758, 2369, + + 4655, 6183, 4724, 4685, 2372, 4606, 2375, 4541, 2307, 2351, + 2334, 2344, 2346, 2348, 2360, 2343, 4554, 6183, 2355, 2387, + 4517, 2363, 6183, 6183, 2391, 4497, 2358, 2353, 4480, 6183, + 6183, 2361, 2361, 2399, 4427, 2405, 2412, 2419, 1915, 2386, + 2361, 2378, 2371, 2393, 2379, 6183, 2440, 4416, 2396, 2398, + 2442, 2450, 2446, 4415, 2438, 2441, 2445, 2447, 2449, 2405, + 2422, 2439, 2430, 2437, 2436, 0, 2450, 2451, 2448, 2438, + 2440, 2461, 2438, 2444, 2447, 0, 0, 2448, 2446, 2467, + 2461, 2466, 2451, 2456, 2461, 0, 4440, 2460, 2457, 2464, + 0, 2535, 4438, 0, 2470, 2489, 2501, 2504, 2494, 2511, + + 0, 2499, 2509, 2542, 2522, 2519, 2500, 2515, 0, 2522, + 2515, 2528, 2519, 2529, 2523, 2530, 2533, 2627, 4396, 2561, + 2523, 2523, 0, 2539, 2547, 2544, 2531, 2548, 2549, 2543, + 2546, 2566, 0, 2562, 2565, 2573, 2572, 0, 2577, 0, + 2578, 2560, 2560, 4423, 2572, 0, 1366, 2597, 2599, 2584, + 6183, 2055, 2616, 2583, 2609, 2617, 2628, 4421, 2617, 2605, + 2608, 2616, 2625, 2630, 2624, 2628, 2633, 2619, 0, 2624, + 2632, 4420, 2642, 2644, 2647, 2705, 4385, 2708, 4381, 2716, + 4379, 2494, 2721, 2700, 4373, 2654, 2658, 4377, 2679, 2687, + 2730, 2696, 4376, 2699, 2690, 2735, 2738, 2744, 4341, 2702, + + 2748, 2757, 2750, 4316, 2708, 4344, 2716, 0, 6183, 2759, + 2764, 2766, 4309, 2755, 2757, 2759, 2761, 2764, 2712, 2721, + 2744, 2759, 2753, 2784, 2793, 2786, 4308, 6183, 2763, 2795, + 2801, 2751, 2780, 0, 0, 2759, 2771, 2766, 2784, 2769, + 0, 2788, 2777, 2791, 2793, 2790, 2783, 2783, 2795, 2803, + 4327, 0, 4321, 2801, 2793, 2805, 0, 2792, 2866, 4283, + 2809, 0, 2802, 2878, 2812, 2815, 2815, 0, 2816, 2882, + 4277, 2822, 0, 0, 0, 2829, 2846, 2851, 2861, 2866, + 2854, 2855, 2852, 2874, 2867, 2871, 2877, 2877, 2880, 2860, + 4275, 2967, 4274, 2877, 0, 0, 2882, 0, 2886, 2887, + + 0, 2885, 2886, 2888, 0, 2874, 2892, 2886, 0, 2889, + 0, 4302, 2903, 2902, 6183, 2932, 2939, 2664, 2938, 2933, + 2934, 2941, 2947, 2951, 2901, 0, 2914, 2927, 4288, 2949, + 4269, 2940, 2955, 4237, 2955, 4235, 2952, 2966, 2959, 4226, + 0, 3025, 3028, 3038, 3041, 4193, 3045, 3053, 3055, 3057, + 4181, 3012, 3065, 3069, 3073, 4176, 3048, 2954, 2954, 3021, + 3026, 4203, 3078, 3084, 4239, 4172, 3038, 3045, 3045, 6183, + 3086, 3051, 3093, 4194, 3065, 3062, 3101, 3063, 3065, 3072, + 3079, 3077, 3106, 6183, 3115, 4159, 3091, 3076, 0, 3079, + 0, 3084, 3088, 0, 3104, 3106, 3156, 3110, 0, 3113, + + 3105, 3121, 3105, 3110, 0, 3114, 0, 3176, 3130, 3126, + 4157, 3131, 3148, 3186, 4222, 4150, 3139, 3150, 0, 3158, + 3150, 4148, 3165, 3164, 0, 3170, 0, 3173, 3173, 3159, + 3205, 3175, 3179, 3181, 3181, 3179, 4154, 3174, 4147, 3182, + 3201, 3191, 3194, 3190, 3195, 3212, 3213, 4117, 3200, 0, + 3214, 3200, 0, 3247, 3223, 0, 3218, 3226, 0, 3214, + 3221, 0, 3247, 3253, 3255, 3261, 4171, 3259, 3268, 4152, + 3262, 3245, 3250, 3255, 3252, 0, 3261, 3271, 0, 3265, + 0, 3267, 3272, 3266, 3268, 3269, 3309, 3334, 3336, 3288, + 3283, 3291, 3295, 3292, 3296, 3344, 3350, 3364, 3378, 3384, + + 3416, 3371, 4076, 0, 3317, 3315, 6183, 3327, 3338, 3350, + 3369, 3364, 3366, 0, 3366, 3366, 4074, 0, 3365, 0, + 3377, 3382, 0, 0, 3443, 4142, 4071, 3384, 0, 0, + 3387, 3383, 3402, 3401, 3450, 4139, 4057, 3409, 3410, 3412, + 3420, 2969, 4051, 0, 0, 3422, 3423, 0, 0, 3472, + 3430, 0, 3428, 3483, 4119, 4043, 3443, 3443, 3431, 3440, + 3439, 3458, 3463, 3451, 3453, 3457, 3474, 3460, 3480, 3467, + 3474, 3478, 3489, 3474, 3473, 3474, 3490, 3547, 4107, 4024, + 0, 3495, 3483, 3494, 3492, 3515, 2983, 3538, 3521, 3544, + 3546, 3545, 3547, 3489, 3497, 4045, 3537, 3546, 3551, 3549, + + 3550, 3551, 3544, 3545, 3551, 3558, 3545, 3546, 3562, 3551, + 3550, 3565, 3619, 3625, 3634, 3641, 3996, 3548, 3558, 3591, + 3588, 3597, 3600, 3603, 3609, 0, 3612, 3623, 3615, 3617, + 3113, 3980, 0, 3622, 3611, 3624, 3624, 3580, 3927, 3639, + 3618, 3638, 3628, 3585, 3021, 3670, 3639, 3630, 3706, 3923, + 3852, 0, 3641, 3649, 3664, 3651, 3688, 3841, 3864, 3654, + 3847, 3661, 3687, 3659, 3670, 3687, 3672, 3692, 3699, 3700, + 3685, 3703, 3707, 3710, 3700, 3699, 0, 3705, 0, 3727, + 3816, 3700, 0, 3707, 3703, 3342, 3735, 3863, 3856, 3841, + 3733, 3835, 3828, 0, 3790, 3714, 0, 0, 3715, 0, + + 0, 0, 3727, 0, 3777, 3724, 3725, 3731, 3743, 3741, + 3730, 3742, 3806, 3812, 3821, 3828, 3843, 3858, 3873, 3880, + 3917, 3924, 3739, 3753, 3739, 3741, 3764, 3760, 3789, 3799, + 3805, 3791, 0, 3803, 3808, 3767, 3784, 3902, 0, 3831, + 3860, 3852, 3773, 3836, 3908, 3871, 3860, 3871, 3884, 3947, + 3951, 3903, 3737, 0, 3892, 3782, 3724, 0, 3917, 3903, + 3909, 3878, 3842, 3954, 3918, 3918, 3925, 3929, 3944, 3944, + 3939, 3976, 3957, 3945, 3960, 3943, 3966, 3949, 3970, 3969, + 3950, 3956, 3977, 3971, 3970, 4005, 4008, 4039, 0, 3990, + 3990, 4010, 4016, 4018, 3986, 4003, 4010, 4002, 4011, 4020, + + 4013, 4053, 3750, 4007, 4011, 6183, 4083, 4089, 4099, 4105, + 4119, 4146, 4156, 4179, 4193, 4207, 4213, 4241, 4256, 4279, + 4288, 4017, 0, 4022, 4024, 0, 4033, 4126, 4037, 4035, + 0, 0, 4163, 4186, 4056, 3706, 0, 0, 4082, 4175, + 4264, 4083, 3543, 4105, 4100, 4129, 4157, 4222, 3534, 4166, + 3460, 6183, 4051, 4055, 4273, 4183, 4226, 4225, 4307, 4315, + 4252, 3439, 4268, 4267, 4272, 4258, 4268, 4271, 4276, 4133, + 3401, 4295, 4286, 4303, 4303, 4312, 4305, 4303, 4300, 4375, + 4318, 4305, 4312, 4315, 4381, 4384, 4329, 3390, 4324, 4327, + 4362, 3452, 3412, 4327, 0, 0, 4344, 4365, 0, 4362, + + 4423, 4430, 3335, 4368, 4449, 0, 4454, 4461, 4476, 4491, + 4498, 4535, 4365, 0, 4367, 4381, 4518, 4521, 3321, 4527, + 4388, 4541, 3319, 4382, 3302, 4437, 4549, 3296, 4447, 3283, + 4445, 4470, 3310, 4458, 3275, 4468, 4557, 4565, 4494, 3269, + 4512, 4538, 4516, 4578, 3229, 4532, 3223, 4548, 4537, 4560, + 4537, 4558, 4401, 4586, 3132, 3101, 4563, 3048, 4606, 4553, + 4574, 4560, 4568, 4592, 4574, 4588, 4589, 4590, 4605, 4581, + 4591, 4600, 6183, 4605, 0, 4652, 3002, 4599, 2989, 4593, + 4595, 4634, 4619, 0, 4596, 4617, 4673, 4625, 4631, 2965, + 2938, 4642, 4679, 4682, 2919, 4709, 4700, 2933, 2926, 4645, + + 4717, 4648, 2893, 2885, 4692, 4729, 2796, 4658, 4676, 2785, + 4662, 4661, 2772, 4679, 4696, 4686, 4707, 0, 4746, 4758, + 2737, 4703, 2733, 4704, 4719, 2697, 2665, 4707, 4725, 4717, + 4728, 2679, 4721, 4785, 2601, 4794, 2561, 4737, 4797, 4737, + 4804, 2545, 4752, 2566, 4754, 4754, 4746, 4756, 4772, 4767, + 4765, 4768, 4768, 4771, 6183, 4791, 4792, 4778, 4777, 6183, + 2533, 4778, 4780, 4796, 4811, 4784, 0, 4856, 4790, 2559, + 2511, 4799, 4863, 6183, 2327, 2318, 4872, 4806, 4808, 4889, + 4812, 2267, 4892, 6183, 2208, 2164, 4831, 4829, 4900, 0, + 4903, 4843, 4845, 4861, 4910, 2069, 2024, 4853, 4851, 4865, + + 4863, 4917, 2039, 4920, 2001, 4874, 4928, 1938, 1918, 4886, + 4931, 1916, 4889, 4914, 1904, 4898, 4907, 4900, 4911, 4904, + 4921, 4909, 4914, 6183, 4916, 4915, 4918, 4918, 4936, 6183, + 6183, 4962, 0, 4933, 4963, 4938, 4948, 4951, 4945, 1897, + 4967, 4969, 4959, 1884, 4969, 4962, 5007, 1870, 5016, 1863, + 4959, 4971, 0, 1859, 5032, 4964, 4986, 4992, 5036, 1845, + 4998, 5043, 5053, 1821, 4990, 4993, 5060, 1782, 5003, 1773, + 5007, 5023, 6183, 5005, 5024, 5010, 5017, 5021, 5033, 5038, + 6183, 5027, 5026, 5046, 5043, 6183, 5048, 5081, 1769, 5052, + 5058, 5043, 5060, 5059, 6183, 0, 0, 5063, 5058, 5054, + + 1762, 1758, 5074, 5070, 5108, 1742, 1760, 1362, 5071, 1723, + 5075, 5131, 5097, 5076, 1704, 1692, 5087, 5082, 5102, 1688, + 5094, 5091, 5108, 5116, 6183, 5114, 5104, 5120, 5119, 5108, + 5106, 5115, 6183, 5119, 6183, 1685, 5110, 1713, 0, 5122, + 5137, 5124, 5133, 5126, 5132, 5134, 1646, 5139, 1340, 1635, + 5139, 5145, 1661, 1611, 1530, 5149, 5145, 5151, 5163, 5203, + 5169, 5160, 5164, 6183, 5166, 6183, 5165, 6183, 5186, 5177, + 0, 5179, 5191, 5183, 6183, 5194, 5184, 6183, 5184, 6183, + 1509, 1503, 5193, 5185, 5187, 5188, 5184, 5205, 5237, 5192, + 5263, 1491, 6183, 5200, 5208, 6183, 5214, 5224, 6183, 5217, + + 1518, 5219, 5232, 6183, 5223, 1454, 5230, 5237, 5242, 5244, + 5275, 1425, 5284, 1394, 5230, 1371, 5247, 6183, 5242, 6183, + 5251, 5252, 5259, 6183, 5249, 5257, 5253, 5276, 5279, 5280, + 1326, 5277, 5268, 5270, 5267, 5274, 5270, 5286, 5343, 5275, + 5298, 5299, 5300, 5291, 5293, 6183, 5360, 5302, 5369, 1309, + 5373, 1749, 1263, 1239, 5310, 1257, 1149, 5322, 5324, 6183, + 5303, 5334, 6183, 1110, 1107, 1078, 5327, 5339, 5330, 5332, + 5329, 6183, 5356, 1004, 5358, 5347, 5349, 6183, 5314, 5361, + 5341, 5354, 5357, 5402, 1000, 5354, 5406, 5354, 5350, 989, + 6183, 5369, 5430, 5368, 5368, 5391, 5385, 5397, 5378, 5433, + + 6183, 5519, 5529, 5539, 5549, 5559, 5569, 5579, 5589, 5599, + 5609, 5619, 5629, 5639, 5649, 5659, 5669, 5676, 5681, 946, + 5687, 5689, 943, 5698, 5703, 936, 5704, 5706, 5712, 906, + 5719, 5729, 5739, 5749, 5759, 5769, 5778, 5787, 898, 5797, + 5807, 876, 857, 5817, 752, 5827, 5834, 5839, 749, 5845, + 5847, 742, 5856, 5861, 725, 5862, 5864, 5870, 703, 5877, + 5887, 5897, 5907, 5917, 5927, 5936, 5945, 695, 5955, 5965, + 662, 654, 5975, 652, 5985, 5995, 546, 498, 6001, 480, + 6005, 478, 6012, 6022, 6032, 6042, 6052, 6062, 6072, 6082, + 6092, 6102, 6112, 6122, 6132, 6142, 6152, 6162, 6172 + } ; -static yyconst flex_int16_t yy_def[2171] = +static yyconst flex_int16_t yy_def[2200] = { 0, - 2072, 1, 2072, 3, 2073, 2073, 2072, 7, 2072, 9, - 2072, 11, 2074, 2074, 2075, 2075, 2076, 2076, 2072, 19, - 2076, 2076, 2077, 2077, 2078, 2078, 2079, 2079, 2080, 2080, - 2081, 2081, 2082, 2082, 2083, 2083, 2072, 37, 2084, 2084, - 2085, 2085, 2072, 43, 2086, 2086, 2087, 2087, 2072, 49, - 2088, 2088, 2072, 2072, 2072, 2072, 2072, 2089, 2090, 2091, - 2092, 2072, 2072, 2072, 2072, 2072, 2072, 2091, 2072, 2091, - 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, - 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, 2091, - 2091, 2091, 2093, 2072, 2072, 2072, 2072, 2094, 2072, 2072, - - 2072, 2094, 2094, 2094, 2094, 2094, 2094, 2094, 2072, 2072, - 2072, 2072, 2072, 2095, 2095, 2095, 2072, 2095, 2095, 2072, - 2095, 2095, 2095, 2095, 2095, 2095, 2095, 2072, 2072, 2096, - 2097, 2098, 2072, 2097, 2097, 2099, 2072, 2072, 2072, 2100, - 2072, 2072, 2100, 2072, 2100, 2072, 2072, 2100, 2100, 2100, - 2100, 2100, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2101, 2072, 2072, 2072, 2101, 2072, 2101, 2101, - 2101, 2101, 2101, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2102, 2072, 2103, 2104, 2072, - 2105, 2072, 2072, 2106, 2107, 2072, 2108, 2072, 2072, 2072, - 2072, 2072, 2109, 2110, 2072, 2072, 2072, 2072, 2072, 2110, - 2110, 2110, 2110, 2110, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2111, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2112, 2113, 2072, 2072, 2072, - 2072, 2113, 2113, 2113, 2072, 2072, 2072, 2114, 2072, 2114, - 2072, 2072, 2072, 2072, 2072, 2072, 2115, 2116, 2072, 2072, - 2072, 2072, 2072, 2116, 2072, 2072, 2072, 2116, 2116, 2116, - 2072, 2072, 2072, 2072, 2072, 2072, 2117, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2118, 2119, 2120, 2121, 2072, - 2072, 2072, 2072, 2072, 2072, 2120, 2120, 2120, 2120, 2120, - 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, + 2101, 1, 2101, 3, 2102, 2102, 2101, 7, 2101, 9, + 2101, 11, 2103, 2103, 2104, 2104, 2105, 2105, 2101, 19, + 2105, 2105, 2106, 2106, 2107, 2107, 2108, 2108, 2109, 2109, + 2110, 2110, 2111, 2111, 2112, 2112, 2101, 37, 2113, 2113, + 2114, 2114, 2101, 43, 2115, 2115, 2116, 2116, 2101, 49, + 2117, 2117, 2101, 2101, 2101, 2101, 2101, 2118, 2119, 2120, + 2121, 2101, 2101, 2101, 2101, 2101, 2101, 2120, 2101, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, - 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, - 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, 2120, - 2120, 2120, 2122, 2072, 2072, 2072, 2123, 2072, 2072, 2072, - 2123, 2123, 2123, 2123, 2123, 2123, 2123, 2072, 2072, 2072, - 2124, 2124, 2124, 2072, 2124, 2124, 2124, 2124, 2124, 2124, - - 2072, 2072, 2072, 2124, 2124, 400, 2124, 2124, 400, 2124, - 2124, 2124, 2124, 400, 2072, 2072, 2125, 2126, 2127, 2072, - 2072, 2072, 2126, 2126, 2128, 2072, 2129, 2072, 2072, 2129, - 2129, 2129, 2129, 2129, 2129, 2129, 2129, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2130, 2130, 2130, - 2072, 2072, 2130, 2130, 2130, 2130, 2130, 2130, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2131, 2072, 2132, 2072, 2072, 2072, 2072, 2133, 2072, 2134, - 2072, 2072, 2072, 2072, 2072, 2072, 2135, 2072, 2072, 2072, - 2136, 2072, 2137, 2072, 2072, 2072, 2138, 2139, 2072, 2072, - - 2139, 2139, 2139, 2139, 2139, 2139, 2072, 2072, 2072, 2072, - 2072, 2140, 2072, 2072, 2072, 2072, 2072, 2072, 2141, 2142, - 2072, 2142, 2142, 2142, 2072, 2072, 2143, 2143, 2072, 2072, - 2072, 2144, 2145, 2072, 2072, 2072, 2072, 2145, 2145, 2072, - 2145, 2145, 2145, 2072, 2072, 2072, 2072, 2146, 2072, 2072, - 2072, 2072, 2147, 2072, 2072, 2072, 2072, 2072, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2072, 2149, 2149, - 2149, 2149, 2149, 2149, 2072, 2072, 2150, 2150, 2150, 2150, - 2150, 2150, 2150, 2150, 2150, 2150, 2150, 2151, 2151, 2152, - 2152, 2152, 2152, 2152, 2152, 2152, 2152, 2072, 2072, 2072, - 2072, 2072, 2153, 2153, 2153, 2153, 2153, 2153, 2153, 2153, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2132, 2132, 2072, 2072, 2072, 2134, 2134, 2072, 2072, 2072, - - 2135, 2135, 2072, 2072, 2154, 2072, 2072, 2072, 2072, 2139, - 2139, 2139, 2139, 2139, 2139, 2072, 2072, 2155, 2072, 2072, - 2072, 2072, 2156, 2072, 2142, 2142, 2142, 2072, 2072, 2143, - 2072, 2157, 2072, 2072, 2072, 2072, 2072, 2145, 2072, 2072, - 2145, 2145, 2145, 2072, 2158, 2072, 2072, 2072, 2147, 2147, - 2147, 2072, 2072, 2072, 2072, 2072, 2072, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2149, 2149, 2149, 2149, - 2149, 2149, 2150, 2150, 2150, 2150, 2072, 2150, 2150, 2150, - 2150, 2150, 2150, 2151, 2151, 2152, 2152, 2152, 2152, 2152, - 2152, 2152, 2152, 2153, 2153, 2153, 2153, 2153, 2153, 2153, - 2153, 2159, 2072, 2160, 2072, 2161, 2072, 2154, 2154, 2154, - 2072, 2072, 2072, 2072, 2139, 2139, 2139, 2139, 2139, 2139, - 2072, 2155, 2155, 2155, 2072, 2072, 2156, 2156, 2156, 2072, - - 2142, 2142, 2142, 2143, 2072, 2157, 2157, 2157, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2145, 2145, 2145, 2158, - 2158, 2158, 2072, 2072, 2072, 2147, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2072, 2072, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - - 2148, 2148, 2148, 2149, 2149, 2149, 2149, 2149, 2072, 2150, - 2150, 2150, 2150, 2150, 2150, 2150, 2150, 2150, 2151, 2151, - 2152, 2152, 2152, 2152, 2152, 2152, 2152, 2152, 2153, 2153, - 2153, 2153, 2153, 2153, 2153, 2159, 2159, 2159, 2159, 2072, - 2160, 2160, 2160, 2160, 2072, 2161, 2161, 2161, 2161, 2072, - 2154, 2072, 2072, 2072, 2139, 2139, 2072, 2162, 2072, 2072, - 2139, 2139, 2139, 2072, 2155, 2072, 2156, 2142, 2142, 2142, - 2157, 2072, 2072, 2145, 2145, 2145, 2158, 2072, 2072, 2072, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - - 2148, 2148, 2072, 2072, 2072, 2148, 2148, 2072, 2072, 2072, - 2148, 2148, 2148, 2148, 2072, 2072, 2148, 2148, 2148, 2148, - 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2148, - 2148, 2149, 2149, 2149, 2149, 2150, 2150, 2150, 2150, 2150, - 2150, 2150, 2150, 2150, 2151, 2152, 2152, 2152, 2152, 2152, - 2152, 2152, 2152, 2153, 2153, 2153, 2153, 2153, 2153, 2159, - 2160, 2161, 2072, 2072, 2072, 2139, 2139, 2072, 2162, 2162, - 2162, 2162, 2162, 2162, 2072, 2072, 2139, 2139, 2139, 2072, - - 2142, 2142, 2142, 2072, 2072, 2145, 2145, 2145, 2072, 2072, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2072, - 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2072, - 2072, 2072, 2148, 2148, 2163, 2072, 2148, 2148, 2148, 2072, - 2148, 2148, 2148, 2148, 2148, 2148, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2148, 2148, 2148, - 2072, 2072, 2072, 2148, 2148, 2148, 2149, 2149, 2150, 2150, - 2150, 2150, 2150, 2150, 2150, 2150, 2151, 2152, 2152, 2152, - 2152, 2152, 2152, 2152, 2153, 2153, 2153, 2153, 2153, 2072, - - 2072, 2072, 2139, 2139, 2072, 2162, 2162, 2162, 2164, 2072, - 2139, 2139, 2142, 2142, 2142, 2072, 2072, 2145, 2145, 2072, - 2148, 2148, 2148, 2165, 2072, 2148, 2148, 2148, 2148, 2148, - 2166, 2072, 2072, 2072, 2148, 2148, 2163, 2163, 2072, 2148, - 2072, 2072, 2072, 2072, 2148, 2148, 2072, 2072, 2167, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2148, 2148, - 2148, 2168, 2072, 2148, 2148, 2149, 2149, 2150, 2150, 2150, - 2150, 2150, 2150, 2150, 2150, 2151, 2152, 2152, 2152, 2152, - 2152, 2152, 2152, 2153, 2153, 2153, 2153, 2153, 2072, 2072, - - 2072, 2139, 2139, 2072, 2169, 2164, 2162, 2162, 2164, 2164, - 2169, 2164, 2164, 2164, 2072, 2139, 2139, 2142, 2142, 2142, - 2072, 2072, 2145, 2072, 2148, 2148, 2148, 2165, 2165, 2072, - 2148, 2148, 2148, 2148, 2166, 2166, 2072, 2072, 2072, 2148, - 2148, 2072, 2072, 2072, 2072, 2148, 2072, 2170, 2072, 2148, - 2072, 2072, 2167, 2167, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2148, 2168, 2168, 2072, 2148, - 2149, 2149, 2150, 2150, 2150, 2152, 2152, 2152, 2153, 2153, - 2153, 2072, 2072, 2072, 2139, 2139, 2072, 2169, 2169, 2169, - - 2169, 2169, 2169, 2162, 2162, 2164, 2164, 2169, 2169, 2164, - 2164, 2164, 2139, 2139, 2142, 2142, 2142, 2072, 2072, 2145, - 2072, 2148, 2148, 2072, 2072, 2072, 2072, 2148, 2148, 2148, - 2072, 2072, 2072, 2072, 2072, 2072, 2148, 2148, 2072, 2072, - 2072, 2072, 2072, 2170, 2170, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2148, 2072, 2072, 2072, 2072, 2149, 2149, - 2150, 2150, 2150, 2152, 2152, 2152, 2153, 2153, 2153, 2072, - 2072, 2072, 2072, 2072, 2139, 2139, 2169, 2169, 2169, 2164, - - 2169, 2164, 2139, 2142, 2142, 2072, 2072, 2072, 2072, 2145, - 2072, 2072, 2072, 2072, 2072, 2148, 2072, 2072, 2072, 2072, - 2072, 2072, 2148, 2148, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2148, 2072, 2072, 2072, 2072, 2149, 2149, - 2150, 2152, 2153, 2153, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2169, 2164, 2139, 2142, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2148, 2072, 2072, 2072, 2072, 2148, 2148, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2149, 2149, 2150, 2152, 2153, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2169, 2139, 2142, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2148, 2072, 2072, - 2072, 2148, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2149, - 2149, 2150, 2152, 2072, 2072, 2072, 2072, 2139, 2142, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2148, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2149, 2150, 2152, 2072, 2072, - 2072, 2139, 2142, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2149, 2150, 2152, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2149, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 0, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072 + 2120, 2120, 2122, 2101, 2101, 2101, 2101, 2123, 2101, 2101, + + 2101, 2123, 2123, 2123, 2123, 2123, 2123, 2123, 2101, 2101, + 2101, 2101, 2101, 2124, 2124, 2124, 2101, 2124, 2124, 2101, + 2124, 2124, 2124, 2124, 2124, 2124, 2124, 2101, 2101, 2125, + 2126, 2127, 2101, 2126, 2126, 2128, 2101, 2101, 2101, 2129, + 2101, 2101, 2129, 2101, 2129, 2101, 2101, 2129, 2129, 2129, + 2129, 2129, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2130, 2101, 2101, 2101, 2130, 2101, 2130, 2130, + 2130, 2130, 2130, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2131, 2101, 2132, 2133, 2101, + 2134, 2101, 2101, 2135, 2136, 2101, 2137, 2101, 2101, 2101, + 2101, 2101, 2138, 2139, 2101, 2101, 2101, 2101, 2101, 2139, + 2139, 2139, 2139, 2139, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2140, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2141, 2142, 2101, 2101, 2101, + 2101, 2142, 2142, 2142, 2101, 2101, 2101, 2143, 2101, 2143, + 2101, 2101, 2101, 2101, 2101, 2101, 2144, 2145, 2101, 2101, + 2101, 2101, 2101, 2145, 2101, 2101, 2101, 2145, 2145, 2145, + 2101, 2101, 2101, 2101, 2101, 2101, 2146, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2147, 2148, 2149, 2150, 2101, + 2101, 2101, 2101, 2101, 2101, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, 2149, + 2149, 2149, 2149, 2151, 2101, 2101, 2101, 2152, 2101, 2101, + 2101, 2152, 2152, 2152, 2152, 2152, 2152, 2152, 2101, 2101, + 2101, 2153, 2153, 2153, 2101, 2153, 2153, 2153, 2153, 2153, + + 2153, 2101, 2101, 2101, 2153, 2153, 401, 2153, 2153, 401, + 2153, 2153, 2153, 2153, 401, 2101, 2101, 2154, 2155, 2156, + 2101, 2101, 2101, 2155, 2155, 2157, 2101, 2158, 2101, 2101, + 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2159, 2159, + 2159, 2101, 2101, 2159, 2159, 2159, 2159, 2159, 2159, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2160, 2101, 2161, 2101, 2101, 2101, 2101, 2162, 2101, + 2163, 2101, 2101, 2101, 2101, 2101, 2101, 2164, 2101, 2101, + 2101, 2165, 2101, 2166, 2101, 2101, 2101, 2167, 2168, 2101, + + 2101, 2168, 2168, 2168, 2168, 2168, 2168, 2101, 2101, 2101, + 2101, 2101, 2169, 2101, 2101, 2101, 2101, 2101, 2101, 2170, + 2171, 2101, 2171, 2171, 2171, 2101, 2101, 2172, 2172, 2101, + 2101, 2101, 2173, 2174, 2101, 2101, 2101, 2101, 2174, 2174, + 2101, 2174, 2174, 2174, 2101, 2101, 2101, 2101, 2175, 2101, + 2101, 2101, 2101, 2176, 2101, 2101, 2101, 2101, 2101, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2101, 2101, 2101, + 2178, 2178, 2178, 2178, 2178, 2178, 2101, 2101, 2179, 2179, + 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2180, + 2180, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2101, + 2101, 2101, 2101, 2101, 2182, 2182, 2182, 2182, 2182, 2182, + 2182, 2182, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2161, 2161, 2101, 2101, 2101, 2163, 2163, 2101, + + 2101, 2101, 2164, 2164, 2101, 2101, 2183, 2101, 2101, 2101, + 2101, 2168, 2168, 2168, 2168, 2168, 2168, 2101, 2101, 2184, + 2101, 2101, 2101, 2101, 2185, 2101, 2171, 2171, 2171, 2101, + 2101, 2172, 2101, 2186, 2101, 2101, 2101, 2101, 2101, 2174, + 2101, 2101, 2174, 2174, 2174, 2101, 2187, 2101, 2101, 2101, + 2176, 2176, 2176, 2101, 2101, 2101, 2101, 2101, 2101, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2101, 2101, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2178, 2178, 2178, 2178, 2178, 2178, 2179, 2179, 2179, 2179, + 2101, 2179, 2179, 2179, 2179, 2179, 2179, 2180, 2180, 2181, + 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2182, 2182, 2182, + 2182, 2182, 2182, 2182, 2182, 2188, 2101, 2189, 2101, 2190, + 2101, 2183, 2183, 2183, 2101, 2101, 2101, 2101, 2168, 2168, + 2168, 2168, 2168, 2168, 2101, 2184, 2184, 2184, 2101, 2101, + + 2185, 2185, 2185, 2101, 2171, 2171, 2171, 2172, 2101, 2186, + 2186, 2186, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2174, 2174, 2174, 2187, 2187, 2187, 2101, 2101, 2101, 2176, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2101, 2101, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2101, + 2101, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2178, + 2178, 2178, 2178, 2178, 2101, 2179, 2179, 2179, 2179, 2179, + 2179, 2179, 2179, 2179, 2180, 2180, 2181, 2181, 2181, 2181, + 2181, 2181, 2181, 2181, 2182, 2182, 2182, 2182, 2182, 2182, + 2182, 2188, 2188, 2188, 2188, 2101, 2189, 2189, 2189, 2189, + 2101, 2190, 2190, 2190, 2190, 2101, 2183, 2101, 2101, 2101, + 2168, 2168, 2101, 2191, 2101, 2101, 2168, 2168, 2168, 2101, + 2184, 2101, 2185, 2171, 2171, 2171, 2186, 2101, 2101, 2174, + 2174, 2174, 2187, 2101, 2101, 2101, 2177, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2101, 2101, + 2101, 2177, 2177, 2101, 2101, 2101, 2177, 2177, 2177, 2177, + 2101, 2101, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, + 2177, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2177, 2177, + 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2177, 2178, 2178, + 2178, 2178, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, + 2179, 2180, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, + 2182, 2182, 2182, 2182, 2182, 2182, 2188, 2189, 2190, 2101, + 2101, 2101, 2168, 2168, 2101, 2191, 2191, 2191, 2191, 2191, + + 2191, 2101, 2101, 2168, 2168, 2168, 2101, 2171, 2171, 2171, + 2101, 2101, 2174, 2174, 2174, 2101, 2101, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2101, 2101, 2101, 2177, 2177, 2177, + 2177, 2177, 2177, 2177, 2101, 2101, 2101, 2101, 2101, 2177, + 2177, 2192, 2101, 2177, 2177, 2177, 2101, 2177, 2177, 2177, + 2177, 2177, 2177, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2177, 2177, 2177, 2101, 2101, 2101, + 2177, 2177, 2177, 2178, 2178, 2179, 2179, 2179, 2179, 2179, + 2179, 2179, 2179, 2180, 2181, 2181, 2181, 2181, 2181, 2181, + + 2181, 2182, 2182, 2182, 2182, 2182, 2101, 2101, 2101, 2168, + 2168, 2101, 2191, 2191, 2191, 2193, 2101, 2168, 2168, 2171, + 2171, 2171, 2101, 2101, 2174, 2174, 2101, 2177, 2177, 2177, + 2194, 2101, 2177, 2177, 2177, 2177, 2177, 2195, 2101, 2101, + 2101, 2177, 2177, 2192, 2192, 2101, 2177, 2101, 2101, 2101, + 2101, 2177, 2177, 2101, 2101, 2101, 2196, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2177, 2177, 2177, 2197, + 2101, 2177, 2177, 2178, 2178, 2179, 2179, 2179, 2179, 2179, + 2179, 2179, 2179, 2180, 2181, 2181, 2181, 2181, 2181, 2181, + + 2181, 2182, 2182, 2182, 2182, 2182, 2101, 2101, 2101, 2168, + 2168, 2101, 2198, 2193, 2191, 2191, 2193, 2193, 2198, 2193, + 2193, 2193, 2101, 2168, 2168, 2171, 2171, 2171, 2101, 2101, + 2174, 2101, 2177, 2177, 2177, 2194, 2194, 2101, 2177, 2177, + 2177, 2177, 2195, 2195, 2101, 2101, 2101, 2177, 2177, 2101, + 2101, 2101, 2101, 2177, 2101, 2199, 2101, 2177, 2101, 2101, + 2101, 2196, 2196, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2177, 2197, 2197, 2101, 2177, 2178, + 2178, 2179, 2179, 2179, 2181, 2181, 2181, 2182, 2182, 2182, + + 2101, 2101, 2101, 2168, 2168, 2101, 2198, 2198, 2198, 2198, + 2198, 2198, 2191, 2191, 2193, 2193, 2198, 2198, 2193, 2193, + 2193, 2168, 2168, 2171, 2171, 2171, 2101, 2101, 2174, 2101, + 2177, 2177, 2101, 2101, 2101, 2101, 2177, 2177, 2177, 2101, + 2101, 2101, 2101, 2101, 2101, 2177, 2177, 2101, 2101, 2101, + 2101, 2101, 2199, 2199, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2177, 2101, 2101, 2101, 2101, 2178, 2178, + 2179, 2179, 2179, 2181, 2181, 2181, 2182, 2182, 2182, 2101, + + 2101, 2101, 2101, 2101, 2168, 2168, 2198, 2198, 2198, 2193, + 2198, 2193, 2168, 2171, 2171, 2101, 2101, 2101, 2101, 2174, + 2101, 2101, 2101, 2101, 2101, 2177, 2101, 2101, 2101, 2101, + 2101, 2101, 2177, 2177, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2177, 2101, 2101, 2101, 2101, 2178, + 2178, 2179, 2181, 2182, 2182, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2198, 2193, 2168, 2171, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2177, 2101, 2101, 2101, 2101, 2177, 2177, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2178, 2178, 2179, 2181, 2182, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2198, 2168, 2171, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2177, + 2101, 2101, 2101, 2177, 2101, 2101, 2101, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2178, 2178, 2179, 2181, 2101, 2101, 2101, 2101, + 2168, 2171, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2177, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2178, + 2179, 2181, 2101, 2101, 2101, 2168, 2171, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2178, 2179, 2181, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2178, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + + 0, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101 + } ; -static yyconst flex_int16_t yy_nxt[6218] = +static yyconst flex_int16_t yy_nxt[6271] = { 0, 54, 55, 56, 55, 57, 58, 59, 60, 61, 62, 62, 57, 63, 64, 65, 66, 67, 68, 68, 57, @@ -1172,14 +1184,14 @@ static yyconst flex_int16_t yy_nxt[6218] = 156, 157, 156, 158, 207, 210, 159, 158, 160, 210, 159, 213, 160, 164, 165, 164, 213, 164, 165, 164, 166, 97, 97, 167, 166, 166, 335, 167, 97, 166, - 97, 378, 448, 342, 418, 315, 379, 316, 316, 323, + 97, 379, 449, 342, 419, 315, 380, 316, 316, 323, 346, 208, 347, 324, 239, 239, 208, 211, 317, 356, - 328, 211, 377, 325, 329, 335, 214, 318, 272, 319, + 328, 211, 378, 325, 329, 335, 214, 318, 272, 319, 272, 214, 342, 330, 320, 321, 322, 326, 323, 346, - 327, 347, 324, 239, 239, 380, 380, 317, 356, 328, + 327, 347, 324, 239, 239, 381, 381, 317, 356, 328, 161, 162, 325, 329, 161, 162, 318, 272, 319, 272, - 381, 381, 330, 320, 321, 322, 326, 168, 169, 327, + 382, 382, 330, 320, 321, 322, 326, 168, 169, 327, 308, 168, 169, 170, 171, 172, 171, 170, 170, 170, 173, 174, 175, 175, 176, 177, 178, 177, 170, 170, 179, 179, 170, 170, 175, 170, 170, 179, 179, 179, @@ -1190,20 +1202,20 @@ static yyconst flex_int16_t yy_nxt[6218] = 179, 179, 180, 179, 179, 179, 179, 179, 179, 179, 179, 181, 179, 179, 182, 183, 179, 179, 179, 179, 179, 179, 175, 170, 170, 170, 170, 170, 184, 185, - 186, 186, 186, 186, 186, 186, 534, 534, 187, 368, - 389, 187, 191, 192, 191, 390, 533, 193, 527, 194, - 372, 195, 196, 191, 192, 191, 520, 382, 193, 383, - 194, 331, 195, 196, 200, 201, 200, 332, 368, 202, - 384, 202, 202, 202, 203, 200, 201, 200, 333, 372, - 202, 334, 202, 202, 202, 203, 382, 336, 383, 498, - - 331, 337, 343, 402, 385, 338, 332, 448, 403, 384, - 344, 339, 357, 386, 340, 352, 345, 333, 536, 536, - 334, 341, 358, 353, 188, 189, 336, 188, 189, 418, - 337, 343, 354, 385, 338, 355, 197, 198, 387, 344, - 339, 357, 386, 340, 352, 345, 377, 197, 198, 421, - 341, 358, 353, 308, 422, 428, 533, 440, 204, 205, - 429, 354, 441, 444, 355, 311, 311, 387, 445, 204, + 186, 186, 186, 186, 186, 186, 535, 535, 187, 369, + 390, 187, 191, 192, 191, 391, 534, 193, 528, 194, + 373, 195, 196, 191, 192, 191, 521, 383, 193, 384, + 194, 331, 195, 196, 200, 201, 200, 332, 369, 202, + 385, 202, 202, 202, 203, 200, 201, 200, 333, 373, + 202, 334, 202, 202, 202, 203, 383, 336, 384, 499, + + 331, 337, 343, 403, 386, 338, 332, 449, 404, 385, + 344, 339, 357, 387, 340, 352, 345, 333, 537, 537, + 334, 341, 358, 353, 188, 189, 336, 188, 189, 419, + 337, 343, 354, 386, 338, 355, 197, 198, 388, 344, + 339, 357, 387, 340, 352, 345, 378, 197, 198, 422, + 341, 358, 353, 308, 423, 429, 534, 441, 204, 205, + 430, 354, 442, 445, 355, 311, 311, 388, 446, 204, 205, 219, 220, 221, 220, 219, 222, 223, 224, 225, 226, 219, 219, 219, 219, 227, 219, 219, 224, 224, 219, 228, 219, 219, 229, 230, 224, 224, 224, 224, @@ -1214,17 +1226,17 @@ static yyconst flex_int16_t yy_nxt[6218] = 232, 224, 224, 224, 224, 233, 224, 224, 224, 224, 224, 234, 224, 224, 224, 224, 224, 224, 224, 224, 219, 219, 219, 219, 219, 219, 236, 237, 241, 242, - 241, 527, 243, 244, 520, 245, 246, 247, 241, 242, + 241, 528, 243, 244, 424, 245, 246, 247, 241, 242, 241, 248, 243, 244, 348, 245, 246, 247, 349, 359, - 498, 248, 360, 363, 423, 424, 350, 365, 361, 366, - 351, 430, 367, 364, 362, 431, 432, 369, 433, 249, - - 449, 450, 448, 348, 370, 371, 453, 349, 359, 249, - 454, 360, 363, 423, 424, 350, 365, 361, 366, 351, - 430, 367, 364, 362, 431, 432, 369, 433, 249, 449, - 450, 451, 463, 370, 371, 453, 452, 464, 249, 454, - 467, 378, 250, 251, 535, 468, 379, 534, 534, 636, - 637, 638, 250, 251, 252, 253, 254, 253, 252, 255, + 521, 248, 360, 425, 363, 431, 350, 366, 361, 367, + 351, 364, 368, 424, 362, 432, 433, 370, 434, 249, + + 454, 365, 499, 348, 371, 372, 455, 349, 359, 249, + 449, 360, 425, 363, 431, 350, 366, 361, 367, 351, + 364, 368, 502, 362, 432, 433, 370, 434, 249, 454, + 365, 452, 464, 371, 372, 455, 453, 465, 249, 468, + 419, 379, 250, 251, 469, 536, 380, 378, 535, 535, + 308, 502, 250, 251, 252, 253, 254, 253, 252, 255, 256, 257, 258, 259, 260, 252, 252, 252, 261, 252, 252, 257, 257, 252, 252, 252, 252, 252, 257, 257, 257, 257, 257, 262, 257, 257, 257, 257, 257, 263, @@ -1234,25 +1246,25 @@ static yyconst flex_int16_t yy_nxt[6218] = 257, 257, 262, 257, 257, 257, 257, 257, 263, 257, 257, 257, 257, 257, 264, 257, 257, 257, 257, 257, 257, 257, 257, 252, 252, 252, 252, 252, 252, 265, - 266, 97, 97, 97, 97, 97, 267, 97, 501, 269, - 97, 97, 97, 97, 97, 97, 97, 97, 380, 380, - 97, 97, 97, 97, 97, 381, 381, 504, 457, 418, - 270, 303, 303, 303, 303, 303, 303, 501, 505, 455, - 377, 2072, 2072, 2072, 456, 458, 308, 2072, 2072, 2072, - 506, 97, 2072, 97, 2072, 394, 504, 457, 2072, 270, - - 2072, 394, 645, 646, 2072, 2072, 2072, 505, 455, 495, - 495, 495, 2055, 456, 458, 2072, 509, 2072, 394, 506, + 266, 97, 97, 97, 97, 97, 267, 97, 505, 269, + 97, 97, 97, 97, 97, 97, 97, 97, 381, 381, + 97, 97, 97, 97, 97, 382, 382, 450, 451, 2084, + 270, 303, 303, 303, 303, 303, 303, 505, 506, 456, + 2090, 2101, 2101, 2101, 457, 507, 2063, 2101, 2101, 2101, + 510, 97, 2101, 97, 2101, 395, 450, 451, 2101, 270, + + 2101, 395, 647, 648, 2101, 2101, 2101, 506, 456, 496, + 496, 496, 395, 457, 507, 2101, 851, 2101, 395, 510, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, - 97, 97, 97, 267, 97, 405, 269, 97, 97, 97, - 97, 97, 97, 97, 97, 509, 514, 97, 97, 97, - 97, 97, 522, 523, 524, 304, 434, 270, 304, 2061, - 435, 528, 529, 502, 405, 2072, 503, 541, 542, 436, - 543, 2072, 550, 437, 551, 514, 562, 563, 97, 389, - 97, 522, 523, 524, 390, 434, 270, 2041, 2072, 435, - 528, 529, 502, 496, 2040, 503, 541, 542, 436, 543, - - 394, 550, 437, 551, 847, 562, 563, 97, 97, 97, + 97, 97, 97, 267, 97, 406, 269, 97, 97, 97, + 97, 97, 97, 97, 97, 458, 515, 97, 97, 97, + 97, 97, 523, 524, 525, 304, 435, 270, 304, 2052, + 436, 529, 459, 503, 406, 2101, 504, 530, 542, 437, + 543, 2101, 544, 438, 458, 515, 551, 552, 97, 390, + 97, 523, 524, 525, 391, 435, 270, 2051, 2101, 436, + 529, 459, 503, 497, 2074, 504, 530, 542, 437, 543, + + 2069, 544, 438, 469, 469, 551, 552, 97, 97, 97, 97, 97, 97, 97, 97, 273, 274, 275, 274, 273, 276, 277, 278, 279, 273, 273, 273, 280, 281, 282, 283, 273, 284, 284, 273, 285, 286, 273, 287, 288, @@ -1263,550 +1275,555 @@ static yyconst flex_int16_t yy_nxt[6218] = 278, 278, 278, 278, 278, 278, 290, 278, 278, 278, 278, 278, 278, 278, 273, 273, 273, 273, 273, 273, - 292, 293, 295, 296, 295, 313, 313, 297, 2038, 295, - 296, 295, 564, 565, 297, 298, 555, 566, 2072, 2072, - 2072, 299, 298, 1995, 567, 495, 495, 495, 299, 2072, - 300, 2072, 394, 510, 510, 510, 571, 300, 2072, 2072, - 2072, 564, 565, 402, 404, 555, 566, 301, 403, 2072, - 421, 2072, 394, 567, 301, 422, 2072, 2072, 2072, 300, - 1974, 537, 394, 538, 538, 571, 300, 2072, 572, 2072, - 394, 2012, 406, 404, 539, 575, 301, 573, 2072, 2072, - 2072, 394, 408, 301, 574, 1009, 302, 468, 468, 2072, - 576, 2072, 394, 302, 391, 391, 391, 572, 849, 556, - - 556, 406, 2072, 539, 575, 393, 573, 393, 394, 496, - 557, 408, 394, 574, 2072, 2072, 2072, 511, 407, 576, - 395, 396, 2072, 397, 577, 2072, 2011, 2072, 394, 510, - 510, 510, 398, 428, 399, 580, 400, 409, 429, 557, - 2072, 517, 517, 517, 2072, 2072, 2072, 407, 852, 395, - 396, 1964, 397, 577, 558, 2072, 558, 2072, 394, 559, - 559, 398, 2072, 399, 580, 400, 409, 517, 517, 517, - 472, 472, 472, 585, 2004, 473, 1998, 474, 401, 391, - 391, 391, 410, 475, 479, 479, 479, 721, 722, 480, - 393, 481, 393, 394, 682, 683, 684, 482, 2072, 485, - - 485, 485, 585, 486, 487, 395, 396, 315, 397, 316, - 316, 410, 488, 511, 530, 530, 530, 398, 586, 399, - 317, 400, 530, 530, 530, 518, 1934, 440, 2072, 546, - 546, 546, 441, 587, 395, 396, 1989, 397, 546, 546, - 546, 303, 303, 303, 560, 588, 398, 586, 399, 317, - 400, 518, 561, 589, 476, 553, 553, 553, 1897, 568, - 569, 578, 587, 401, 2072, 2072, 2072, 570, 483, 1969, - 590, 1968, 579, 560, 588, 2072, 581, 2072, 394, 582, - 583, 561, 589, 489, 1966, 591, 592, 411, 568, 569, - 578, 412, 593, 1891, 584, 594, 570, 394, 531, 590, - - 413, 579, 597, 598, 414, 581, 531, 599, 582, 583, - 1874, 600, 601, 547, 591, 592, 411, 602, 603, 1853, - 412, 593, 547, 584, 594, 304, 595, 604, 596, 413, - 605, 597, 598, 414, 606, 607, 599, 1848, 608, 554, - 600, 601, 609, 615, 620, 610, 602, 603, 2072, 621, - 611, 622, 616, 618, 617, 595, 604, 596, 612, 605, - 613, 619, 623, 606, 607, 624, 614, 608, 626, 631, - 633, 609, 615, 620, 610, 632, 634, 635, 621, 611, - 622, 616, 618, 617, 629, 625, 639, 612, 630, 613, - 619, 623, 640, 627, 624, 614, 628, 626, 631, 633, - - 641, 642, 643, 644, 632, 634, 635, 658, 1939, 659, - 728, 729, 1845, 629, 625, 639, 660, 630, 444, 661, - 662, 640, 627, 445, 663, 628, 2072, 2072, 2072, 641, - 642, 643, 644, 391, 391, 391, 658, 2072, 659, 2072, - 394, 2072, 2072, 2072, 393, 660, 393, 394, 661, 662, - 534, 534, 2072, 663, 2072, 394, 2072, 2072, 2072, 395, - 396, 1934, 397, 2072, 2072, 2072, 664, 2072, 1933, 2072, - 394, 398, 665, 399, 2072, 400, 2072, 394, 1932, 2072, - 2072, 2072, 666, 667, 673, 674, 675, 405, 395, 396, - 2072, 397, 2072, 394, 451, 664, 1836, 406, 676, 452, - - 398, 665, 399, 677, 400, 678, 1834, 2072, 2072, 2072, - 2072, 666, 667, 673, 674, 675, 405, 401, 2072, 407, - 2072, 394, 2072, 2072, 2072, 2072, 406, 676, 757, 757, - 409, 1921, 677, 2072, 678, 2072, 394, 679, 1935, 680, - 2072, 2072, 2072, 2072, 463, 467, 1798, 2072, 407, 464, - 468, 710, 2072, 711, 2072, 394, 2072, 2072, 2072, 409, - 410, 712, 1905, 2072, 713, 647, 679, 2072, 680, 2072, - 394, 649, 472, 472, 472, 392, 479, 479, 479, 690, - 710, 714, 711, 695, 392, 2072, 2072, 2072, 392, 410, - 712, 2072, 1901, 713, 647, 715, 2072, 648, 2072, 394, - - 649, 2072, 2072, 2072, 392, 1895, 2072, 2072, 2072, 2072, - 714, 1338, 2072, 392, 2072, 394, 1442, 392, 2072, 392, - 2072, 394, 650, 392, 715, 2072, 648, 1783, 717, 720, - 411, 1888, 392, 725, 412, 1887, 652, 559, 559, 651, - 2072, 726, 727, 413, 495, 495, 495, 414, 392, 914, - 914, 650, 392, 2072, 2072, 2072, 476, 717, 720, 411, - 483, 392, 725, 412, 2072, 652, 2072, 394, 651, 2072, - 726, 727, 413, 2072, 2072, 2072, 414, 1771, 653, 2072, - 2072, 2072, 738, 738, 2072, 2072, 2072, 394, 2072, 2072, - 2072, 2072, 2072, 394, 705, 705, 705, 654, 2072, 2072, - - 2072, 657, 485, 485, 485, 392, 700, 653, 1761, 2072, - 730, 2072, 394, 707, 392, 510, 510, 510, 392, 731, - 655, 718, 718, 718, 754, 754, 654, 1729, 496, 741, - 657, 708, 709, 742, 392, 754, 754, 2072, 739, 730, - 656, 1856, 707, 392, 517, 517, 517, 392, 731, 655, - 723, 723, 723, 530, 530, 530, 740, 2072, 741, 394, - 708, 709, 742, 2072, 732, 732, 732, 739, 743, 656, - 536, 536, 735, 735, 537, 747, 538, 538, 706, 1429, - 748, 734, 2072, 736, 1524, 740, 489, 539, 737, 758, - 737, 756, 756, 738, 738, 1013, 759, 743, 760, 511, - - 546, 546, 546, 761, 747, 719, 745, 745, 745, 748, - 734, 762, 736, 750, 553, 553, 539, 753, 758, 753, - 556, 556, 754, 754, 755, 759, 755, 760, 518, 756, - 756, 557, 761, 763, 724, 764, 765, 531, 766, 768, - 762, 769, 770, 767, 773, 771, 774, 775, 733, 772, - 776, 777, 778, 779, 780, 1724, 781, 782, 783, 784, - 557, 1722, 763, 785, 764, 765, 786, 766, 768, 787, - 769, 770, 767, 773, 771, 774, 775, 788, 772, 776, - 777, 778, 779, 780, 547, 781, 782, 783, 784, 789, - 746, 790, 785, 791, 792, 786, 793, 751, 787, 795, - - 796, 797, 798, 799, 800, 801, 788, 802, 803, 804, - 805, 806, 807, 808, 809, 810, 811, 812, 789, 813, - 790, 815, 791, 792, 1851, 793, 818, 819, 795, 796, - 797, 798, 799, 800, 801, 822, 802, 803, 804, 805, - 806, 807, 808, 809, 810, 811, 812, 820, 813, 821, - 815, 816, 816, 816, 823, 818, 819, 825, 826, 827, - 828, 829, 830, 831, 822, 824, 832, 833, 834, 836, - 837, 838, 835, 839, 840, 841, 820, 842, 821, 394, - 394, 394, 394, 823, 394, 394, 825, 826, 827, 828, - 829, 830, 831, 844, 824, 832, 833, 834, 836, 837, - - 838, 835, 839, 840, 841, 843, 842, 845, 846, 394, - 848, 854, 394, 851, 855, 856, 857, 858, 860, 861, - 863, 864, 844, 853, 865, 866, 867, 868, 869, 850, - 870, 871, 756, 756, 843, 817, 845, 846, 1847, 848, - 854, 1709, 851, 855, 856, 857, 858, 860, 861, 863, - 864, 882, 853, 865, 866, 867, 868, 869, 850, 870, - 871, 872, 872, 872, 874, 874, 874, 876, 876, 876, - 879, 705, 705, 883, 884, 885, 886, 887, 888, 889, - 882, 891, 893, 718, 718, 896, 898, 723, 723, 901, - 902, 904, 905, 907, 732, 732, 915, 916, 735, 735, - - 917, 918, 883, 884, 885, 886, 887, 888, 889, 736, - 891, 910, 919, 910, 896, 924, 911, 911, 901, 902, - 904, 905, 912, 925, 912, 915, 916, 913, 913, 917, - 918, 921, 745, 745, 750, 553, 553, 1841, 736, 2072, - 2072, 919, 757, 757, 924, 873, 927, 928, 875, 929, - 930, 877, 925, 931, 880, 932, 933, 935, 936, 937, - 934, 938, 939, 940, 941, 942, 894, 943, 944, 945, - 899, 946, 947, 948, 949, 927, 928, 908, 929, 930, - 950, 952, 931, 953, 932, 933, 935, 936, 937, 934, - 938, 939, 940, 941, 942, 954, 943, 944, 945, 958, - - 946, 947, 948, 949, 955, 955, 955, 959, 960, 950, - 952, 961, 953, 962, 963, 922, 964, 926, 751, 965, - 966, 966, 966, 968, 954, 969, 970, 971, 958, 972, - 973, 974, 975, 976, 977, 978, 959, 960, 979, 990, - 961, 991, 962, 963, 992, 964, 993, 994, 965, 988, - 988, 988, 968, 995, 969, 970, 971, 996, 972, 973, - 974, 975, 976, 977, 978, 997, 998, 979, 990, 999, - 991, 1000, 1001, 992, 1002, 993, 994, 394, 394, 1003, - 1004, 1005, 995, 1006, 1008, 1020, 996, 394, 956, 394, - 1694, 1015, 1831, 394, 997, 998, 1021, 1016, 999, 1690, - - 1000, 1001, 394, 1002, 967, 816, 816, 816, 1003, 1004, - 1005, 394, 1006, 1008, 1020, 1011, 1010, 1012, 1682, 1014, - 1015, 394, 1022, 1023, 1024, 1021, 1016, 980, 1017, 1025, - 981, 982, 1026, 989, 1018, 1027, 1028, 1029, 1030, 1031, - 1033, 983, 984, 985, 1011, 1010, 1012, 986, 1014, 1034, - 1035, 1022, 1023, 1024, 2072, 2072, 1827, 1017, 1025, 981, - 982, 1026, 1676, 1018, 1027, 1028, 1029, 1030, 1031, 1033, - 983, 984, 985, 1037, 872, 872, 986, 1052, 1034, 1035, - 1038, 1042, 874, 874, 879, 705, 705, 1053, 1043, 817, - 1047, 876, 876, 1055, 1048, 2072, 2072, 1056, 1057, 1057, - - 1057, 1061, 1063, 1058, 1064, 1066, 1052, 2072, 2072, 2072, - 2072, 1059, 893, 718, 718, 1068, 1053, 2072, 2072, 898, - 723, 723, 1055, 2072, 2072, 1070, 1056, 2072, 2072, 1072, - 1061, 1063, 1073, 1064, 1066, 907, 732, 732, 2072, 2072, - 911, 911, 911, 911, 1068, 913, 913, 913, 913, 914, - 914, 1074, 1075, 1825, 1070, 1076, 1078, 1039, 1072, 2072, - 2072, 1073, 921, 745, 745, 1044, 2072, 2072, 880, 2072, - 2072, 1081, 1665, 1051, 1049, 1079, 1079, 1079, 1082, 1083, - 1074, 1075, 1060, 1084, 1076, 1078, 1085, 1065, 1086, 1087, - 1804, 1088, 1089, 1090, 1091, 1092, 894, 1093, 1094, 1095, - - 1081, 1067, 1096, 899, 1099, 1100, 1101, 1082, 1083, 1102, - 1338, 1339, 1084, 1106, 1107, 1085, 1071, 1086, 1087, 908, - 1088, 1089, 1090, 1091, 1092, 1111, 1093, 1094, 1095, 1112, - 1802, 1096, 1113, 1099, 1100, 1101, 1114, 1117, 1102, 955, - 955, 955, 1106, 1107, 1077, 1118, 922, 750, 1108, 1108, - 1108, 966, 966, 966, 1111, 1119, 1120, 1121, 1112, 1080, - 1126, 1113, 1122, 1109, 1123, 1114, 1117, 1124, 1103, 1125, - 1128, 1129, 1130, 1131, 1118, 1127, 1132, 1104, 1133, 1143, - 1144, 1145, 1146, 1147, 1119, 1120, 1121, 1148, 1149, 1115, - 1150, 1122, 1151, 1123, 394, 1152, 1124, 1103, 1125, 1128, - - 1129, 1130, 1131, 394, 1127, 1132, 1104, 1133, 1143, 1144, - 1145, 1146, 1147, 1154, 1155, 394, 1148, 1149, 1115, 1150, - 1157, 1151, 1156, 956, 1152, 394, 1165, 1160, 394, 394, - 394, 394, 1110, 394, 1796, 967, 988, 988, 988, 1166, - 1436, 1163, 1154, 1155, 1159, 1531, 1167, 1169, 1795, 1157, - 1171, 1156, 1793, 1172, 1162, 1165, 1160, 1174, 1176, 1177, - 1178, 1134, 1135, 1136, 1158, 1137, 1161, 1633, 1166, 1164, - 1163, 2072, 2072, 1159, 1138, 1167, 1169, 1139, 1140, 1171, - 1454, 1141, 1172, 1162, 1183, 1549, 1174, 1176, 1177, 1178, - 1134, 1135, 1136, 1628, 1137, 2072, 2072, 1037, 872, 872, - - 1478, 1038, 1785, 1138, 1038, 1575, 1139, 1140, 2072, 2072, - 1141, 2072, 2072, 1183, 1038, 2072, 2072, 1038, 2072, 2072, - 989, 1043, 1042, 874, 874, 2072, 2072, 2072, 2072, 1043, - 1184, 1043, 1185, 1043, 2072, 2072, 1186, 1048, 1047, 876, - 876, 1197, 1048, 2072, 2072, 1198, 1048, 2072, 2072, 879, - 1048, 1199, 1617, 1057, 1057, 1057, 1200, 1202, 1058, 1184, - 1612, 1185, 1190, 1057, 1057, 1186, 1059, 1191, 1203, 1204, - 1197, 2072, 2072, 1205, 1198, 1192, 2072, 2072, 1206, 1188, - 1199, 1039, 1207, 1208, 1211, 1200, 1202, 1212, 1193, 1180, - 2072, 2072, 1079, 1079, 1079, 1213, 893, 1203, 1204, 1214, - - 1215, 1774, 1205, 1216, 1217, 1181, 1044, 1206, 1188, 1221, - 1222, 1207, 1208, 1211, 1223, 1224, 1212, 1193, 1225, 1218, - 1218, 1218, 1049, 1226, 1213, 1182, 1227, 1607, 1214, 1215, - 1209, 1231, 1216, 1217, 1219, 1232, 1233, 1060, 1221, 1222, - 1234, 1237, 1770, 1223, 1224, 1767, 1194, 1225, 1238, 898, - 1239, 1240, 1226, 1241, 907, 1227, 1228, 1228, 1228, 1209, - 1231, 1242, 1243, 1244, 1232, 1233, 1245, 1246, 921, 1234, - 1237, 1229, 1108, 1108, 1108, 1253, 1080, 1238, 1254, 1239, - 1240, 1255, 1241, 1247, 1247, 1247, 1256, 1109, 1258, 1250, - 1242, 1243, 1244, 1251, 1252, 1245, 1246, 1260, 1248, 1261, - - 1262, 1263, 1264, 1220, 1253, 1265, 1266, 1254, 1267, 1268, - 1255, 1545, 1269, 1270, 1766, 1256, 1627, 1258, 1250, 1274, - 1275, 1276, 1251, 1252, 1277, 1278, 1260, 1764, 1261, 1262, - 1263, 1264, 1591, 394, 1265, 1266, 394, 1267, 1268, 394, - 1230, 1269, 1270, 1271, 1271, 1271, 1279, 1280, 1274, 1275, - 1276, 394, 1282, 1277, 1278, 394, 1110, 394, 1272, 1287, - 1288, 394, 1289, 1290, 1291, 1293, 1285, 1249, 1281, 1292, - 1294, 1295, 1283, 1296, 1297, 1279, 1280, 1284, 1286, 1298, - 1299, 1282, 1429, 1430, 2072, 2072, 1300, 1301, 1287, 1288, - 1038, 1289, 1290, 1291, 1293, 1285, 1302, 1281, 1292, 1294, - - 1295, 1283, 1296, 1297, 2072, 2072, 1284, 1286, 1298, 1299, - 1043, 2072, 2072, 1303, 1048, 1300, 1301, 1304, 1305, 1190, - 1057, 1057, 1436, 1437, 1191, 1302, 1964, 1273, 1190, 1057, - 1057, 1311, 1192, 1191, 1759, 1190, 1057, 1057, 1576, 1312, - 1191, 1192, 1303, 1313, 1314, 1193, 1304, 1305, 1192, 1749, - 1190, 1057, 1057, 1728, 1193, 1191, 1309, 1309, 1309, 1315, - 1311, 1193, 1037, 1306, 1316, 1190, 1057, 1057, 1312, 1726, - 1191, 1317, 1313, 1314, 1193, 1318, 1193, 1319, 1192, 1338, - 2072, 1320, 1042, 1193, 1321, 1322, 1323, 1560, 1315, 1047, - 1193, 1193, 1326, 1316, 1327, 1328, 1190, 1057, 1057, 1329, - - 1317, 1191, 1965, 1194, 1318, 1193, 1319, 1330, 1307, 1192, - 1320, 1550, 1194, 1321, 1322, 1323, 1218, 1218, 1218, 1194, - 1193, 1326, 1193, 1327, 1328, 1333, 1334, 1715, 1329, 1335, - 1336, 1219, 1340, 1341, 1194, 1345, 1330, 1307, 1346, 1712, - 1310, 1228, 1228, 1228, 1351, 1342, 1342, 1342, 1352, 1194, - 1353, 1193, 1354, 1355, 1333, 1334, 1229, 1356, 1335, 1336, - 1343, 1340, 1341, 1357, 1345, 1539, 1358, 1346, 1247, 1247, - 1247, 1706, 1359, 1351, 1360, 1308, 1361, 1352, 1362, 1353, - 1194, 1354, 1355, 1248, 1363, 1364, 1356, 1365, 1366, 1367, - 1347, 1368, 1357, 1369, 1348, 1358, 394, 1370, 1371, 1374, - - 1220, 1359, 1375, 1360, 1376, 1361, 1377, 1362, 394, 1271, - 1271, 1271, 394, 1363, 1364, 1386, 1365, 1366, 1367, 1532, - 1368, 1702, 1369, 1348, 1272, 1230, 1370, 1371, 1374, 1344, - 394, 1375, 1379, 1376, 1378, 1377, 394, 394, 394, 394, - 1387, 1389, 1380, 1390, 1386, 1381, 1383, 1391, 1382, 1392, - 1384, 1385, 1249, 1393, 1394, 1395, 1396, 1397, 1398, 1399, - 1400, 1401, 1402, 1378, 1525, 1403, 1404, 1454, 1455, 1387, - 1389, 1380, 1390, 1699, 1381, 1383, 1391, 1382, 1392, 1384, - 1385, 1416, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, - 1401, 1402, 1417, 1273, 1403, 1404, 1405, 1406, 1406, 1693, - - 1418, 1191, 1190, 1057, 1057, 1419, 1420, 1191, 1680, 1306, - 1416, 1190, 1057, 1057, 1421, 1192, 1191, 1422, 1410, 1406, - 1406, 1417, 1193, 1411, 1192, 1408, 1423, 1424, 1193, 1418, - 1425, 1412, 1426, 1427, 1419, 1420, 1431, 1193, 1432, 1433, - 1434, 1438, 1439, 1421, 1413, 1440, 1422, 1441, 1443, 1443, - 1443, 1193, 1446, 1447, 1408, 1423, 1424, 1193, 1450, 1425, - 1451, 1426, 1427, 1452, 1457, 1431, 1193, 1432, 1433, 1434, - 1438, 1439, 1459, 1413, 1440, 1462, 1441, 1463, 1444, 1460, - 1407, 1446, 1447, 1342, 1342, 1342, 1194, 1450, 1464, 1451, - 1190, 1465, 1452, 1457, 1466, 1194, 1467, 1461, 1343, 1468, - - 1469, 1459, 1414, 394, 1462, 1470, 1463, 1444, 1460, 1471, - 1473, 1474, 1475, 1476, 1478, 1479, 1480, 1464, 1481, 1482, - 1465, 394, 1487, 1466, 1485, 1467, 1461, 394, 1468, 1469, - 1472, 1488, 1445, 1489, 1470, 1491, 1492, 1493, 1471, 1473, - 1474, 1475, 1476, 1494, 1484, 1480, 1495, 1481, 1482, 1496, - 1497, 1487, 1513, 1485, 1429, 2072, 394, 1483, 1514, 1472, - 1488, 394, 1489, 1515, 1491, 1492, 1493, 1344, 1516, 1499, - 1406, 1406, 1494, 1484, 1500, 1495, 1436, 2072, 1496, 1497, - 1668, 1513, 1501, 1410, 1406, 1406, 1644, 1514, 1411, 1190, - 1057, 1057, 1515, 1636, 1191, 1502, 1412, 1516, 1517, 1190, - - 1057, 1057, 1192, 1518, 1191, 1506, 1057, 1057, 1443, 1413, - 1411, 1625, 1192, 1519, 1520, 1193, 1521, 1522, 1412, 1620, - 1523, 1410, 1406, 1406, 1502, 1193, 1411, 1517, 1615, 1528, - 1529, 1413, 1518, 1505, 1412, 1508, 1057, 1057, 1413, 1594, - 1500, 1342, 1519, 1520, 1193, 1521, 1522, 1413, 1501, 1523, - 1530, 1535, 1536, 1503, 1193, 1525, 1525, 1525, 1528, 1529, - 1413, 1502, 1505, 1537, 1532, 1532, 1532, 1414, 1504, 1506, - 1057, 1057, 1542, 1194, 1411, 1538, 1413, 1545, 1546, 1530, - 1535, 1536, 1510, 1194, 1541, 1526, 1506, 1057, 1057, 1507, - 1502, 1411, 1537, 1309, 1533, 1413, 1506, 1057, 1057, 1412, - - 1490, 1411, 1543, 1547, 1538, 1414, 1539, 1539, 1539, 1412, - 1486, 1548, 1413, 1541, 1526, 1443, 1443, 1443, 1553, 1509, - 1554, 1109, 1413, 1533, 1413, 1454, 2072, 1555, 1556, 1511, - 1557, 1543, 1547, 1550, 1550, 1550, 1558, 1559, 1562, 1527, - 1548, 1413, 1560, 1560, 1560, 1444, 394, 1553, 1534, 1554, - 1563, 1413, 1564, 1507, 1565, 1566, 1555, 1556, 1511, 1557, - 1567, 394, 1568, 1551, 1569, 1558, 1559, 1562, 1570, 1571, - 1507, 1572, 1573, 1574, 1444, 1512, 1478, 2072, 1579, 1563, - 1507, 1564, 1580, 1565, 1566, 1576, 1576, 1576, 394, 1567, - 1540, 1568, 1551, 1569, 394, 1584, 394, 1570, 1571, 1445, - - 1572, 1573, 1574, 394, 1585, 1586, 1587, 1579, 1582, 1588, - 1589, 1580, 1590, 1595, 1596, 1577, 1583, 1552, 1591, 1591, - 1591, 1581, 1545, 2072, 1584, 394, 1561, 1603, 394, 1604, - 1271, 1605, 1592, 1585, 1586, 1587, 1458, 1582, 1588, 1589, - 1456, 1590, 1595, 1596, 1577, 1583, 1508, 1057, 1057, 1247, - 1581, 1500, 1499, 1406, 1406, 1449, 1603, 1500, 1604, 1501, - 1605, 1508, 1057, 1057, 1606, 1501, 1500, 1448, 1228, 1578, - 1610, 1218, 1502, 1415, 1501, 1611, 1388, 1614, 1502, 1508, - 1057, 1057, 1373, 1616, 1500, 1372, 1619, 1502, 1621, 1350, - 1349, 1108, 1597, 1606, 1332, 1508, 1057, 1057, 1331, 1610, - - 1500, 1502, 1593, 1325, 1611, 1502, 1614, 1502, 1501, 1508, - 1057, 1057, 1616, 1622, 1500, 1619, 1502, 1621, 1190, 1057, - 1057, 1502, 1501, 1191, 1324, 1623, 1079, 1190, 1057, 1057, - 1509, 1192, 1191, 1624, 1502, 1502, 1503, 1626, 1598, 1057, - 1192, 394, 1622, 394, 1193, 1509, 1506, 1057, 1057, 1631, - 1502, 1411, 988, 1193, 1623, 1506, 1057, 1057, 1259, 1412, - 1411, 1257, 1624, 1509, 1502, 1632, 1626, 1598, 1412, 1607, - 1607, 1607, 1413, 1193, 1612, 1612, 1612, 966, 1631, 1509, - 1236, 1413, 1193, 1608, 1235, 1525, 1525, 1525, 1599, 1219, - 1508, 1057, 1057, 1509, 1632, 1500, 1635, 1405, 1617, 1617, - - 1617, 1413, 1194, 1501, 1539, 1539, 1539, 1508, 1057, 1057, - 1413, 1194, 1500, 1229, 955, 1526, 1502, 1210, 1637, 1109, - 1501, 1201, 1410, 1406, 1406, 1635, 1196, 1411, 1638, 1195, - 1507, 1187, 876, 1502, 1600, 1510, 1532, 1532, 1532, 1507, - 874, 1506, 1057, 1057, 1526, 1502, 1411, 1637, 1413, 872, - 1506, 1057, 1057, 1609, 1412, 1411, 1179, 1638, 1613, 1628, - 1628, 1628, 1502, 1412, 1602, 1639, 1533, 1413, 1640, 1527, - 1633, 1633, 1633, 1641, 1509, 1642, 1413, 1413, 1550, 1550, - 1550, 1643, 1618, 1645, 1646, 1248, 1601, 1647, 1540, 1629, - 1648, 1509, 1649, 1602, 1639, 1533, 1413, 1640, 1560, 1560, - - 1560, 1650, 1641, 1651, 1642, 1413, 1414, 1652, 1551, 1661, - 1643, 1662, 1645, 1646, 394, 1663, 1647, 1664, 1629, 1648, - 1534, 1649, 1576, 1576, 1576, 1507, 1665, 1665, 1665, 1410, - 1650, 1667, 1651, 1175, 1507, 1669, 1652, 1551, 1661, 1670, - 1662, 1272, 1671, 1630, 1663, 1672, 1664, 1673, 1674, 1675, - 1681, 1687, 1577, 1173, 1634, 1591, 1591, 1591, 1688, 1170, - 1667, 1168, 1552, 1153, 1669, 1676, 1676, 1676, 1670, 1592, - 1142, 1671, 1689, 816, 1672, 1116, 1673, 1674, 1675, 1681, - 1687, 1577, 1561, 1653, 1682, 1682, 1682, 1688, 1677, 1654, - 1499, 1406, 1406, 1655, 1656, 1500, 1105, 1098, 1657, 1658, - - 1659, 1689, 1660, 1597, 1683, 1097, 1578, 1678, 1700, 745, - 1666, 732, 1653, 1069, 1701, 723, 1502, 1677, 1654, 1703, - 718, 1062, 1655, 1656, 1694, 1694, 1694, 1657, 1658, 1659, - 1054, 1660, 705, 1508, 1057, 1057, 1678, 1700, 1500, 1593, - 1607, 1607, 1607, 1701, 1695, 1502, 1501, 1050, 1703, 1679, - 1508, 1057, 1057, 1045, 1608, 1500, 1685, 1040, 1704, 1502, - 1506, 1057, 1057, 1501, 1032, 1411, 1697, 1019, 1684, 1007, - 1508, 1057, 1057, 1412, 1503, 1500, 1502, 1506, 1057, 1057, - 1705, 1698, 1411, 1501, 1707, 1685, 1413, 1704, 1502, 987, - 1412, 1690, 1690, 1690, 1708, 1697, 1502, 1612, 1612, 1612, - - 1628, 1628, 1628, 1413, 957, 1502, 1711, 1713, 1696, 1705, - 1698, 1686, 1219, 1707, 1691, 1413, 1714, 1509, 1617, 1617, - 1617, 951, 553, 1708, 1609, 1502, 1709, 1709, 1709, 1499, - 1629, 1716, 1413, 1229, 1509, 1711, 1713, 1717, 1718, 1506, - 1686, 1343, 1719, 1691, 1507, 1714, 923, 1720, 1721, 1508, - 1633, 1633, 1633, 1727, 1509, 1722, 1722, 1722, 1731, 1629, - 1716, 1507, 1724, 1724, 1724, 1248, 1717, 1718, 1729, 1729, - 1729, 1719, 1732, 909, 1733, 1692, 1720, 1721, 1734, 1737, - 1735, 1613, 1727, 1736, 1630, 1738, 1739, 1731, 1740, 1742, - 1745, 1746, 1747, 1748, 903, 900, 895, 1743, 1741, 1750, - - 1751, 1732, 1618, 1733, 1744, 890, 1752, 1734, 1737, 1735, - 1710, 881, 1736, 1754, 1738, 1739, 1755, 1740, 1742, 1745, - 1746, 1747, 1748, 1665, 1665, 1665, 1743, 1741, 1750, 1751, - 1756, 394, 1757, 1744, 1634, 1752, 1758, 1760, 1272, 1723, - 1753, 485, 1754, 702, 702, 1755, 1725, 1676, 1676, 1676, - 479, 1768, 1730, 1682, 1682, 1682, 1761, 1761, 1761, 1756, - 697, 1757, 1508, 1057, 1057, 1758, 1760, 1500, 1769, 1753, - 1677, 697, 1762, 1683, 472, 1501, 1775, 1506, 1057, 1057, - 1768, 1776, 1411, 1690, 1690, 1690, 1777, 692, 1502, 1678, - 1412, 1694, 1694, 1694, 1778, 1779, 1765, 1769, 1780, 1677, - - 1771, 1771, 1771, 1413, 1781, 1775, 1691, 1666, 692, 1782, - 1776, 1695, 1783, 1783, 1783, 1777, 1772, 1502, 1678, 1709, - 1709, 1709, 1786, 1778, 1779, 1765, 1787, 1780, 1788, 1789, - 1790, 1679, 1413, 1781, 1343, 1691, 1791, 1684, 1782, 1792, - 1763, 1794, 1722, 1722, 1722, 1797, 1509, 1724, 1724, 1724, - 1800, 1786, 1798, 1798, 1798, 1787, 1803, 1788, 1789, 1790, - 1805, 1507, 1729, 1729, 1729, 1791, 1806, 1692, 1792, 1807, - 1794, 1808, 1809, 1810, 1797, 1696, 1811, 1812, 1813, 1800, - 1814, 1815, 1816, 1817, 1773, 1803, 1818, 1819, 1820, 1805, - 1821, 862, 394, 1823, 859, 1806, 1784, 814, 1807, 1824, - - 1808, 1809, 1810, 1710, 1801, 1811, 1812, 1813, 1826, 1814, - 1815, 1816, 1817, 1828, 794, 1818, 1819, 1820, 1829, 1821, - 1822, 752, 1823, 1682, 1682, 1682, 1723, 1830, 1824, 546, - 552, 1725, 1832, 1801, 744, 530, 1799, 1826, 1761, 1761, - 1761, 1833, 1828, 1683, 544, 1838, 1730, 1829, 517, 1822, - 1508, 1057, 1057, 1839, 1762, 1500, 1830, 1694, 1694, 1694, - 1840, 1832, 1842, 1501, 1771, 1771, 1771, 1834, 1834, 1834, - 1833, 1836, 1836, 1836, 1838, 1843, 1502, 1695, 525, 1844, - 1772, 1852, 1839, 1783, 1783, 1783, 1845, 1845, 1845, 1840, - 1855, 1842, 1848, 1848, 1848, 1853, 1853, 1853, 1798, 1798, - - 1798, 1857, 1858, 1859, 1843, 1502, 1849, 1684, 1844, 1861, - 1852, 1862, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1855, - 1869, 1870, 1763, 1871, 1872, 1873, 1874, 1874, 1874, 1876, - 1857, 1858, 1859, 394, 1509, 1878, 1879, 1880, 1861, 1881, - 1862, 1696, 1863, 1864, 1865, 1866, 1867, 1868, 1773, 1869, - 1870, 1835, 1871, 1872, 1873, 1837, 1882, 1883, 1876, 1884, - 1885, 1886, 1877, 510, 1878, 1879, 1880, 1784, 1881, 1889, - 1846, 1834, 1834, 1834, 1890, 1893, 1850, 1894, 1896, 1854, - 1902, 515, 1799, 716, 495, 1882, 1883, 507, 1884, 1885, - 1886, 1877, 1836, 1836, 1836, 1891, 1891, 1891, 1889, 1845, - - 1845, 1845, 1903, 1890, 1893, 1906, 1894, 1896, 1907, 1902, - 1875, 1848, 1848, 1848, 1897, 1897, 1897, 1853, 1853, 1853, - 1908, 1909, 704, 1910, 1911, 1849, 703, 702, 1912, 699, - 1913, 1903, 1914, 1915, 1906, 1916, 1917, 1907, 1918, 1919, - 1920, 1874, 1874, 1874, 1922, 698, 1924, 394, 1925, 1908, - 1909, 1898, 1910, 1911, 697, 1835, 1899, 1912, 1904, 1913, - 1926, 1914, 1915, 1927, 1916, 1917, 694, 1918, 1919, 1920, - 1928, 1929, 693, 1922, 1923, 1924, 1837, 1925, 1930, 1892, - 1898, 1931, 1936, 1846, 692, 1899, 1937, 1904, 1938, 1926, - 1940, 1941, 1927, 1942, 1943, 1850, 689, 1944, 1900, 1928, - - 1929, 1854, 1945, 1923, 1891, 1891, 1891, 1930, 1946, 1947, - 1931, 1936, 1897, 1897, 1897, 1937, 1948, 1938, 1949, 1940, - 1941, 1950, 1942, 1943, 1951, 1875, 1944, 1952, 1953, 1954, - 1955, 1945, 1956, 1957, 1958, 1959, 1960, 1946, 1947, 1961, - 1962, 1963, 1967, 1970, 1971, 1948, 1972, 1949, 1973, 1898, - 1950, 688, 687, 1951, 1899, 1976, 1952, 1953, 1954, 1955, - 1977, 1956, 1957, 1958, 1959, 1960, 1978, 686, 1961, 1962, - 1963, 1967, 1970, 1971, 1979, 1972, 1980, 1973, 1898, 1974, - 1974, 1974, 1981, 1899, 1976, 1982, 1983, 1984, 1892, 1977, - 1985, 1986, 1987, 1988, 1990, 1978, 1900, 1991, 1992, 1993, - - 1994, 1997, 685, 1979, 681, 1980, 1995, 1995, 1995, 1999, - 2000, 1981, 2001, 2002, 1982, 1983, 1984, 2003, 2005, 1985, - 1986, 1987, 1988, 1990, 2006, 2007, 1991, 1992, 1993, 1994, - 1997, 1974, 1974, 1974, 445, 2008, 2009, 2010, 1999, 2000, - 2013, 2001, 2002, 1974, 1974, 1974, 2003, 2005, 1995, 1995, - 1995, 2014, 2015, 2006, 2007, 2016, 2017, 2018, 2019, 2020, - 2021, 672, 2022, 1975, 2008, 2009, 2010, 2023, 2024, 2013, - 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, - 2014, 2015, 2035, 2036, 2016, 2017, 2018, 2019, 2020, 2021, - 1996, 2022, 2037, 1682, 1682, 1682, 2023, 2024, 671, 2025, - - 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2039, - 2042, 2035, 2036, 1683, 2043, 1975, 1694, 1694, 1694, 2044, - 2045, 2037, 2046, 2047, 2048, 2049, 2050, 1975, 2051, 2052, - 2053, 2054, 1996, 2055, 2055, 2055, 1695, 2057, 2039, 2042, - 2058, 2059, 2060, 2043, 2055, 2055, 2055, 2062, 2044, 2045, - 2063, 2046, 2047, 2048, 2049, 2050, 2064, 2051, 2052, 2053, - 2054, 2055, 2055, 2055, 2065, 2066, 2057, 2067, 2068, 2058, - 2059, 2060, 1974, 1974, 1974, 2069, 2062, 1684, 2070, 2063, - 2071, 2055, 2055, 2055, 670, 2064, 669, 668, 426, 420, - 416, 391, 415, 2065, 2066, 303, 2067, 2068, 374, 310, - - 1696, 305, 375, 552, 2069, 549, 545, 2070, 544, 2071, - 540, 526, 525, 521, 516, 515, 513, 2056, 508, 507, - 500, 499, 494, 492, 470, 469, 466, 465, 2056, 462, - 461, 460, 459, 447, 446, 443, 442, 439, 438, 426, - 420, 416, 415, 388, 376, 2056, 375, 374, 314, 310, - 305, 2072, 218, 218, 216, 216, 1975, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2056, 110, 110, 110, 110, - 110, 110, 110, 110, 110, 110, 155, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 163, 163, 163, 163, - 163, 163, 163, 163, 163, 163, 97, 97, 97, 97, - - 97, 97, 97, 97, 97, 97, 190, 190, 190, 190, - 190, 190, 190, 190, 190, 190, 199, 199, 199, 199, - 199, 199, 199, 199, 199, 199, 206, 206, 206, 206, - 206, 206, 206, 206, 206, 206, 209, 209, 209, 209, - 209, 209, 209, 209, 209, 209, 212, 212, 212, 212, - 212, 212, 212, 212, 212, 212, 215, 215, 215, 215, - 215, 215, 215, 215, 215, 215, 217, 217, 217, 217, - 217, 217, 217, 217, 217, 217, 238, 238, 238, 238, - 238, 238, 238, 238, 238, 238, 240, 240, 240, 240, - 240, 240, 240, 240, 240, 240, 268, 268, 268, 268, - - 268, 268, 268, 268, 268, 268, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 294, 294, 294, 294, - 294, 294, 294, 294, 294, 294, 306, 306, 307, 307, - 2072, 307, 307, 307, 307, 307, 307, 307, 309, 309, - 373, 2072, 2072, 2072, 2072, 373, 392, 392, 2072, 392, - 2072, 392, 392, 417, 417, 419, 419, 425, 2072, 2072, - 2072, 2072, 425, 427, 2072, 427, 471, 471, 471, 471, - 471, 2072, 471, 471, 2072, 471, 477, 477, 477, 477, - 477, 477, 477, 477, 477, 477, 478, 478, 478, 478, - 478, 2072, 478, 478, 2072, 478, 484, 484, 484, 484, - - 484, 484, 484, 484, 484, 484, 490, 490, 490, 490, - 490, 490, 490, 490, 490, 490, 491, 491, 491, 491, - 491, 491, 491, 491, 491, 493, 493, 493, 493, 493, - 493, 493, 493, 493, 497, 497, 2072, 497, 497, 497, - 497, 497, 497, 497, 512, 512, 2072, 512, 512, 512, - 512, 512, 512, 512, 519, 519, 2072, 519, 519, 519, - 519, 519, 519, 519, 532, 532, 2072, 532, 532, 532, - 532, 532, 532, 532, 548, 548, 2072, 548, 548, 548, - 548, 548, 548, 548, 306, 306, 307, 307, 2072, 307, - 307, 307, 307, 307, 307, 307, 309, 309, 373, 2072, - - 2072, 2072, 2072, 373, 392, 392, 2072, 392, 2072, 392, - 392, 417, 417, 419, 419, 425, 2072, 2072, 2072, 2072, - 425, 427, 2072, 427, 471, 471, 471, 471, 471, 2072, - 471, 471, 2072, 471, 691, 691, 2072, 691, 691, 691, - 691, 691, 691, 691, 478, 478, 478, 478, 478, 2072, - 478, 478, 2072, 478, 696, 696, 2072, 696, 696, 696, - 696, 696, 696, 696, 701, 701, 2072, 701, 701, 701, - 701, 701, 701, 701, 491, 491, 491, 491, 491, 491, - 491, 491, 491, 493, 493, 493, 493, 493, 493, 493, - 493, 493, 497, 497, 2072, 497, 497, 497, 497, 497, - - 497, 497, 512, 512, 2072, 512, 512, 512, 512, 512, - 512, 512, 519, 519, 2072, 519, 519, 519, 519, 519, - 519, 519, 532, 532, 2072, 532, 532, 532, 532, 532, - 532, 532, 548, 548, 2072, 548, 548, 548, 548, 548, - 548, 548, 749, 749, 749, 749, 749, 749, 749, 749, - 749, 749, 392, 2072, 2072, 392, 427, 2072, 427, 878, - 878, 878, 878, 878, 878, 878, 878, 878, 878, 892, - 892, 892, 892, 892, 892, 892, 892, 892, 892, 897, - 897, 897, 897, 897, 897, 897, 897, 897, 897, 906, - 906, 906, 906, 906, 906, 906, 906, 906, 906, 920, - - 920, 920, 920, 920, 920, 920, 920, 920, 920, 1036, - 1036, 1036, 1036, 1036, 1036, 1036, 1036, 1036, 1036, 1041, - 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1041, 1046, - 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1189, - 1189, 1189, 1189, 1189, 1189, 1189, 1189, 1189, 1189, 1337, - 1337, 1337, 1337, 1337, 1337, 1337, 1337, 1337, 1337, 1409, - 1409, 1409, 1409, 1409, 1409, 1409, 1409, 1409, 1409, 1428, - 1428, 1428, 1428, 1428, 1428, 1428, 1428, 1428, 1428, 1435, - 1435, 1435, 1435, 1435, 1435, 1435, 1435, 1435, 1435, 1453, - 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1453, 1477, - - 1477, 1477, 1477, 1477, 1477, 1477, 1477, 1477, 1477, 1498, - 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1498, 1544, - 1544, 1544, 1544, 1544, 1544, 1544, 1544, 1544, 1544, 53, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072 + 292, 293, 295, 296, 295, 313, 313, 297, 2068, 295, + 296, 295, 563, 564, 297, 298, 556, 565, 2101, 2101, + 2101, 299, 298, 2066, 566, 496, 496, 496, 299, 2101, + 300, 2101, 395, 511, 511, 511, 567, 300, 2101, 2101, + 2101, 563, 564, 2065, 405, 556, 565, 301, 568, 2101, + 1980, 2101, 395, 566, 301, 572, 2101, 2101, 2101, 300, + 2062, 538, 395, 539, 539, 567, 300, 2101, 576, 2101, + 395, 1949, 407, 405, 540, 577, 301, 568, 2101, 2101, + 2101, 395, 409, 301, 572, 1015, 302, 723, 724, 2101, + 578, 2101, 395, 302, 392, 392, 392, 576, 853, 557, + + 557, 407, 2101, 540, 577, 394, 2013, 394, 395, 497, + 558, 409, 730, 731, 2101, 2101, 2101, 512, 408, 578, + 396, 397, 2101, 398, 579, 2101, 1981, 2101, 395, 511, + 511, 511, 399, 403, 400, 580, 401, 410, 404, 558, + 2101, 518, 518, 518, 2101, 2101, 2101, 408, 1950, 396, + 397, 1991, 398, 579, 559, 2101, 559, 2101, 395, 560, + 560, 399, 2101, 400, 580, 401, 410, 518, 518, 518, + 473, 473, 473, 581, 2031, 474, 2030, 475, 402, 392, + 392, 392, 411, 476, 480, 480, 480, 535, 535, 481, + 394, 482, 394, 395, 638, 639, 640, 483, 2101, 486, + + 486, 486, 581, 487, 488, 396, 397, 315, 398, 316, + 316, 411, 489, 512, 531, 531, 531, 399, 584, 400, + 317, 401, 531, 531, 531, 519, 759, 759, 2101, 547, + 547, 547, 585, 586, 396, 397, 1980, 398, 547, 547, + 547, 303, 303, 303, 561, 587, 399, 584, 400, 317, + 401, 519, 562, 422, 477, 554, 554, 554, 423, 569, + 570, 585, 586, 402, 2101, 2101, 2101, 571, 484, 2022, + 573, 2016, 588, 561, 587, 2101, 589, 2101, 395, 574, + 590, 562, 582, 490, 1949, 583, 575, 412, 569, 570, + 594, 413, 595, 2006, 598, 596, 571, 597, 532, 573, + + 414, 588, 591, 599, 415, 589, 532, 600, 574, 590, + 1912, 582, 601, 548, 583, 575, 412, 592, 593, 594, + 413, 595, 548, 598, 596, 304, 597, 602, 603, 414, + 604, 591, 599, 415, 605, 606, 600, 607, 608, 555, + 609, 601, 616, 621, 610, 622, 592, 593, 2101, 623, + 619, 617, 624, 618, 625, 611, 602, 603, 620, 604, + 612, 628, 1986, 605, 606, 626, 607, 608, 613, 609, + 614, 616, 621, 610, 622, 635, 615, 636, 623, 619, + 617, 624, 618, 625, 611, 627, 637, 620, 629, 612, + 628, 630, 631, 641, 626, 633, 632, 613, 642, 614, + + 643, 634, 644, 645, 635, 615, 636, 646, 429, 560, + 560, 660, 1985, 430, 627, 637, 661, 629, 441, 1982, + 630, 631, 641, 442, 633, 632, 1905, 642, 395, 643, + 634, 644, 645, 2101, 2101, 2101, 646, 392, 392, 392, + 660, 2101, 2101, 2101, 2101, 661, 2101, 395, 394, 662, + 394, 395, 2101, 663, 2101, 395, 2101, 2101, 2101, 2063, + 2101, 2101, 2101, 396, 397, 1888, 398, 2101, 1867, 2101, + 395, 2101, 1862, 2101, 395, 399, 664, 400, 662, 401, + 445, 452, 663, 665, 1955, 446, 453, 406, 2101, 2101, + 2101, 666, 396, 397, 407, 398, 2101, 2101, 2101, 2101, + + 667, 2101, 395, 1859, 399, 664, 400, 2101, 401, 2101, + 395, 1948, 665, 2101, 2101, 2101, 406, 2101, 668, 410, + 666, 402, 1947, 407, 2101, 2101, 2101, 395, 408, 667, + 669, 675, 2101, 2101, 2101, 2064, 464, 649, 1849, 676, + 2101, 465, 1847, 2101, 2101, 2101, 395, 668, 410, 1936, + 2101, 2101, 2101, 1811, 651, 677, 678, 408, 393, 669, + 675, 2101, 1920, 2101, 395, 393, 649, 393, 676, 393, + 411, 393, 2101, 679, 680, 2101, 2101, 2101, 393, 681, + 2101, 682, 654, 651, 677, 678, 2101, 393, 2101, 395, + 468, 650, 712, 713, 393, 469, 393, 2101, 393, 411, + + 393, 1916, 679, 680, 684, 685, 686, 393, 681, 714, + 682, 654, 652, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 650, 712, 713, 715, 2101, 1910, 2101, 395, 2101, 716, + 2101, 395, 918, 918, 2101, 2101, 2101, 2101, 714, 1795, + 412, 652, 717, 1902, 413, 395, 2101, 719, 2101, 395, + 1901, 653, 715, 414, 2101, 2101, 2101, 415, 716, 2101, + 655, 2101, 2101, 2101, 1783, 2101, 659, 2101, 395, 412, + 393, 717, 2101, 413, 2101, 395, 719, 1773, 656, 393, + 653, 856, 414, 393, 1741, 739, 415, 739, 722, 655, + 740, 740, 2101, 2101, 2101, 659, 1870, 2101, 1736, 393, + + 727, 2101, 657, 2101, 728, 2101, 395, 656, 393, 473, + 473, 473, 393, 480, 480, 480, 692, 722, 1734, 2101, + 697, 486, 486, 486, 709, 702, 496, 496, 496, 727, + 729, 657, 732, 728, 658, 707, 707, 707, 2101, 511, + 511, 511, 710, 711, 733, 2101, 720, 720, 720, 518, + 518, 518, 1865, 709, 725, 725, 725, 743, 744, 729, + 745, 732, 741, 658, 531, 531, 531, 734, 734, 734, + 395, 710, 711, 733, 537, 537, 2101, 737, 737, 538, + 742, 539, 539, 749, 750, 736, 743, 744, 738, 745, + 1861, 741, 540, 477, 547, 547, 547, 484, 747, 747, + + 747, 752, 554, 554, 1720, 490, 1019, 557, 557, 742, + 497, 760, 749, 750, 736, 761, 762, 738, 558, 708, + 755, 540, 755, 512, 763, 756, 756, 764, 765, 757, + 721, 757, 766, 519, 758, 758, 767, 768, 726, 770, + 760, 771, 769, 772, 761, 762, 773, 558, 532, 1854, + 774, 735, 775, 763, 776, 777, 764, 765, 778, 779, + 780, 766, 781, 782, 783, 767, 768, 784, 770, 785, + 771, 769, 772, 786, 787, 773, 788, 789, 548, 774, + 790, 775, 748, 776, 777, 753, 791, 778, 779, 780, + 792, 781, 782, 783, 793, 794, 784, 795, 785, 797, + + 798, 799, 786, 787, 800, 788, 789, 801, 802, 790, + 803, 804, 805, 806, 807, 791, 808, 809, 810, 792, + 811, 812, 813, 793, 794, 814, 795, 815, 797, 798, + 799, 817, 820, 800, 821, 824, 801, 802, 825, 803, + 804, 805, 806, 807, 1705, 808, 809, 810, 828, 811, + 812, 813, 831, 822, 814, 823, 815, 818, 818, 818, + 817, 820, 826, 821, 824, 832, 829, 825, 830, 833, + 834, 835, 836, 827, 837, 838, 840, 828, 841, 839, + 842, 831, 822, 843, 823, 844, 845, 846, 1844, 395, + 395, 826, 395, 858, 832, 829, 395, 830, 833, 834, + + 835, 836, 827, 837, 838, 840, 395, 841, 839, 842, + 395, 395, 843, 847, 844, 845, 846, 849, 395, 848, + 859, 850, 858, 860, 852, 861, 854, 862, 864, 857, + 865, 867, 868, 869, 870, 871, 872, 873, 874, 855, + 875, 819, 847, 876, 876, 876, 849, 1701, 848, 859, + 850, 886, 860, 852, 861, 854, 862, 864, 857, 865, + 867, 868, 869, 870, 871, 872, 873, 874, 855, 875, + 878, 878, 878, 880, 880, 880, 883, 707, 707, 887, + 886, 888, 889, 890, 891, 892, 893, 895, 897, 720, + 720, 900, 902, 725, 725, 905, 906, 908, 1693, 909, + + 911, 734, 734, 740, 740, 919, 920, 1840, 887, 921, + 888, 889, 890, 891, 892, 893, 895, 914, 922, 914, + 900, 923, 915, 915, 905, 906, 908, 877, 909, 737, + 737, 916, 928, 916, 919, 920, 917, 917, 921, 929, + 738, 925, 747, 747, 2101, 2101, 931, 922, 2101, 2101, + 923, 752, 554, 554, 879, 756, 756, 881, 756, 756, + 884, 928, 758, 758, 758, 758, 759, 759, 929, 738, + 932, 933, 898, 934, 935, 931, 903, 936, 937, 939, + 940, 941, 938, 942, 912, 943, 944, 945, 946, 947, + 948, 949, 950, 951, 952, 953, 2101, 2101, 954, 932, + + 933, 956, 934, 935, 957, 958, 936, 937, 939, 940, + 941, 938, 942, 962, 943, 944, 945, 946, 947, 948, + 949, 950, 951, 952, 953, 926, 930, 954, 963, 964, + 956, 965, 966, 957, 958, 753, 959, 959, 959, 967, + 968, 969, 962, 970, 970, 970, 972, 973, 974, 975, + 976, 977, 978, 979, 980, 981, 982, 963, 964, 983, + 965, 966, 992, 992, 992, 994, 995, 996, 967, 968, + 969, 997, 998, 999, 1000, 972, 973, 974, 975, 976, + 977, 978, 979, 980, 981, 982, 1001, 1002, 983, 1003, + 1004, 1687, 1005, 1006, 994, 995, 996, 1007, 395, 395, + + 997, 998, 999, 1000, 1008, 1009, 1010, 1011, 1012, 1014, + 1838, 1021, 395, 1676, 395, 1001, 1002, 1817, 1003, 1004, + 960, 1005, 1006, 1018, 395, 1815, 1007, 971, 818, 818, + 818, 395, 395, 1008, 1009, 1010, 1011, 1012, 1014, 1016, + 1021, 1809, 1017, 395, 1022, 1026, 993, 1027, 1020, 1023, + 984, 1028, 1018, 985, 986, 1029, 1024, 1030, 1031, 1032, + 1033, 1034, 1035, 1036, 987, 988, 989, 1037, 1016, 1039, + 990, 1017, 1040, 1022, 1026, 1041, 1027, 1020, 1023, 395, + 1028, 1808, 985, 986, 1029, 1024, 1030, 1031, 1032, 1033, + 1034, 1035, 1036, 987, 988, 989, 1037, 1058, 1039, 990, + + 1059, 1040, 2101, 2101, 1041, 1061, 1043, 876, 876, 1048, + 878, 878, 819, 1044, 1062, 1165, 1049, 1053, 880, 880, + 1067, 1054, 883, 707, 707, 1069, 1058, 1070, 1072, 1059, + 1806, 1063, 1063, 1063, 1061, 1074, 1064, 2101, 2101, 897, + 720, 720, 1076, 1062, 1065, 1644, 2101, 2101, 1801, 1067, + 2101, 2101, 2101, 2101, 1069, 1078, 1070, 1072, 902, 725, + 725, 2101, 2101, 1079, 1074, 911, 734, 734, 2101, 2101, + 1080, 1076, 915, 915, 915, 915, 917, 917, 917, 917, + 1057, 918, 918, 1081, 1078, 1082, 2101, 2101, 2101, 2101, + 1045, 1084, 1079, 1050, 925, 747, 747, 2101, 2101, 1080, + + 1087, 1055, 1085, 1085, 1085, 1088, 884, 1089, 1090, 1091, + 1092, 1093, 1081, 1638, 1082, 1066, 1094, 1797, 1095, 1096, + 1084, 1097, 1098, 898, 1071, 1099, 1100, 1101, 1102, 1087, + 1073, 1105, 1106, 1107, 1088, 1108, 1089, 1090, 1091, 1092, + 1093, 1112, 903, 1113, 1117, 1094, 1077, 1095, 1096, 912, + 1097, 1098, 1627, 1118, 1099, 1100, 1101, 1102, 1119, 1120, + 1105, 1106, 1107, 1123, 1108, 1622, 1083, 959, 959, 959, + 1112, 1124, 1113, 1117, 1125, 752, 1786, 1126, 926, 1114, + 1114, 1114, 1118, 970, 970, 970, 1086, 1119, 1120, 1127, + 1128, 1129, 1123, 1130, 1115, 1131, 1109, 1132, 1134, 1135, + + 1124, 1136, 1137, 1125, 1138, 1110, 1126, 1139, 1149, 1150, + 1151, 1152, 1133, 1153, 1154, 1155, 1156, 1157, 1127, 1128, + 1129, 1121, 1130, 1158, 1131, 1109, 1159, 1134, 1135, 1161, + 1136, 1137, 1162, 1138, 1110, 1172, 1139, 1149, 1150, 1151, + 1152, 1133, 1153, 1154, 1155, 1156, 1157, 395, 395, 395, + 1121, 960, 1158, 395, 395, 1159, 395, 1173, 1161, 1174, + 1167, 1162, 395, 1116, 1172, 1617, 395, 971, 992, 992, + 992, 1164, 1166, 1782, 1170, 1163, 1176, 1779, 1178, 1169, + 1345, 1346, 1179, 1181, 1778, 1168, 1173, 1183, 1174, 1167, + 1184, 1185, 1190, 1140, 1141, 1142, 1191, 1143, 395, 1776, + + 1164, 1166, 1171, 1170, 1163, 1176, 1144, 1178, 1169, 1145, + 1146, 1179, 1181, 1147, 2101, 2101, 1183, 1054, 1601, 1184, + 1185, 1190, 1140, 1141, 1142, 1191, 1143, 2101, 2101, 1043, + 876, 876, 1345, 1044, 1387, 1144, 1044, 1450, 1145, 1146, + 2101, 2101, 1147, 2101, 2101, 1771, 1044, 2101, 2101, 1044, + 2101, 2101, 993, 1049, 1048, 878, 878, 2101, 2101, 2101, + 2101, 1049, 1192, 1049, 1193, 1049, 1053, 880, 880, 1586, + 1054, 2101, 2101, 1204, 1054, 2101, 2101, 1205, 1054, 1063, + 1063, 1063, 1761, 1206, 1064, 1197, 1063, 1063, 2101, 2101, + 1198, 1192, 1065, 1193, 1207, 2101, 2101, 1209, 1199, 1740, + + 1210, 1211, 1204, 2101, 2101, 1195, 1205, 1212, 2101, 2101, + 1213, 1200, 1206, 1045, 1214, 1215, 1085, 1085, 1085, 1218, + 1219, 1187, 1220, 1207, 1437, 1438, 1209, 1221, 883, 1210, + 1211, 1222, 1223, 1224, 1195, 1228, 1212, 1188, 1050, 1213, + 1200, 1229, 1230, 1214, 1215, 1231, 1232, 1233, 1218, 1219, + 1055, 1220, 1738, 1189, 1216, 1234, 1221, 1225, 1225, 1225, + 1222, 1223, 1224, 1066, 1228, 1238, 897, 1239, 1240, 1201, + 1229, 1230, 1226, 902, 1231, 1232, 1233, 1235, 1235, 1235, + 1241, 911, 1244, 1216, 1234, 1245, 925, 1114, 1114, 1114, + 1246, 1247, 1236, 1248, 1238, 1249, 1239, 1240, 1250, 1251, + + 1086, 1252, 1115, 1253, 1260, 1261, 1254, 1254, 1254, 1241, + 1262, 1244, 1570, 1257, 1245, 1263, 1265, 1258, 1259, 1246, + 1247, 1255, 1248, 1267, 1249, 1268, 1269, 1250, 1251, 1270, + 1252, 1271, 1253, 1260, 1261, 1272, 1273, 1274, 1275, 1262, + 1276, 1227, 1257, 1277, 1263, 1265, 1258, 1259, 1278, 1278, + 1278, 1281, 1267, 1282, 1268, 1269, 1283, 1284, 1270, 1285, + 1271, 1237, 395, 1279, 1272, 1273, 1274, 1275, 395, 1276, + 395, 1116, 1277, 1294, 395, 1286, 395, 395, 1295, 1287, + 1281, 1296, 1282, 395, 1297, 1283, 1284, 1288, 1285, 1289, + 1256, 1290, 1292, 1298, 1293, 1300, 1291, 1301, 1299, 1302, + + 1303, 1304, 1294, 1560, 1286, 1305, 1306, 1295, 1287, 1727, + 1296, 2101, 2101, 1297, 1307, 1308, 1288, 1044, 1289, 1309, + 1290, 1292, 1298, 1293, 1300, 1291, 1301, 1299, 1302, 1303, + 1304, 1310, 1280, 1311, 1305, 1306, 2101, 2101, 2101, 2101, + 1312, 1054, 1049, 1307, 1308, 1197, 1063, 1063, 1309, 1723, + 1198, 1197, 1063, 1063, 1318, 1548, 1198, 395, 1199, 1319, + 1310, 1717, 1311, 1541, 1199, 1197, 1063, 1063, 1320, 1312, + 1198, 1200, 1316, 1316, 1316, 1321, 1713, 1200, 1199, 1197, + 1063, 1063, 1534, 1318, 1198, 1197, 1063, 1063, 1319, 1043, + 1198, 1200, 1313, 1492, 1322, 1323, 1324, 1320, 1199, 1710, + + 1200, 1704, 1325, 1326, 1321, 1200, 1200, 1327, 1328, 1329, + 1330, 1200, 1333, 1334, 1048, 1691, 1053, 1197, 1063, 1063, + 1200, 1335, 1198, 1322, 1323, 1324, 1336, 395, 1314, 1201, + 1199, 1325, 1326, 1337, 1200, 1201, 1327, 1328, 1329, 1330, + 1200, 1333, 1334, 1200, 1225, 1225, 1225, 1340, 1341, 1201, + 1335, 1235, 1235, 1235, 1342, 1336, 1317, 1314, 1343, 1226, + 1347, 1348, 1337, 1201, 1352, 1353, 1236, 395, 1359, 1201, + 1679, 1360, 1200, 1349, 1349, 1349, 1340, 1341, 1361, 1362, + 1363, 1655, 1364, 1342, 1254, 1254, 1254, 1343, 1350, 1347, + 1348, 1365, 1366, 1352, 1353, 1367, 1315, 1359, 1368, 1255, + + 1360, 1201, 1369, 1370, 1371, 1372, 1354, 1361, 1362, 1363, + 1355, 1364, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1647, + 1365, 1366, 1356, 1382, 1367, 1383, 1384, 1368, 1227, 1385, + 395, 1369, 1370, 1371, 1372, 1237, 395, 1394, 1395, 1355, + 1451, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1278, 1278, + 1278, 1356, 1382, 395, 1383, 1384, 1386, 1351, 1385, 395, + 395, 395, 395, 1279, 1397, 1388, 1394, 1395, 1256, 1389, + 1391, 1390, 1392, 1398, 1393, 1399, 1400, 1401, 1402, 1403, + 1404, 1405, 1406, 1407, 1408, 1386, 1409, 1410, 1411, 1412, + 1424, 1444, 1445, 1397, 1388, 1425, 1345, 2101, 1389, 1391, + + 1390, 1392, 1398, 1393, 1399, 1400, 1401, 1402, 1403, 1404, + 1405, 1406, 1407, 1408, 1635, 1409, 1410, 1411, 1412, 1424, + 1413, 1414, 1414, 1630, 1425, 1198, 1197, 1063, 1063, 1426, + 1427, 1198, 1280, 1313, 1428, 1197, 1063, 1063, 1429, 1199, + 1198, 1430, 1418, 1414, 1414, 1431, 1200, 1419, 1199, 1416, + 1432, 1433, 1200, 1434, 1435, 1420, 1439, 1440, 1426, 1427, + 1441, 1200, 1442, 1428, 1446, 1447, 1448, 1429, 1421, 1449, + 1430, 1451, 1451, 1451, 1431, 1200, 1454, 1455, 1416, 1432, + 1433, 1200, 1434, 1435, 1458, 1439, 1440, 1459, 1460, 1441, + 1200, 1442, 1461, 1446, 1447, 1448, 1466, 1421, 1449, 1463, + + 1464, 1452, 1468, 1471, 1415, 1454, 1455, 1349, 1349, 1349, + 1201, 1469, 1472, 1458, 1197, 1473, 1459, 1460, 1474, 1201, + 1475, 1461, 1350, 1476, 1477, 1466, 1422, 1478, 1479, 1470, + 1452, 1468, 1471, 1480, 1482, 1483, 1484, 1485, 1487, 1488, + 1469, 1472, 1489, 1490, 1473, 1491, 1496, 1474, 395, 1475, + 395, 1497, 1476, 1477, 1481, 1453, 1478, 1479, 1470, 1498, + 1500, 1501, 1480, 1482, 1483, 1484, 1485, 1493, 1502, 1494, + 1503, 1489, 1490, 1504, 1491, 1496, 1505, 1506, 1437, 2101, + 1497, 1522, 1523, 1481, 1444, 2101, 1625, 1524, 1498, 1500, + 1501, 1351, 1525, 1554, 1555, 1437, 1493, 1502, 1494, 1503, + + 1533, 1604, 1504, 1526, 1349, 1505, 1506, 1508, 1414, 1414, + 1522, 1523, 1509, 1418, 1414, 1414, 1524, 1551, 1419, 1316, + 1510, 1525, 1197, 1063, 1063, 1527, 1420, 1198, 1499, 1197, + 1063, 1063, 1526, 1511, 1198, 1199, 1528, 1529, 1530, 1421, + 1531, 1495, 1199, 395, 1515, 1063, 1063, 1444, 1200, 1419, + 395, 1532, 1540, 1463, 1527, 1200, 395, 1420, 1559, 1418, + 1414, 1414, 1511, 1514, 1419, 1528, 1529, 1530, 1421, 1531, + 1421, 395, 1420, 1537, 1517, 1063, 1063, 1200, 395, 1509, + 1532, 1515, 1063, 1063, 1200, 1421, 1419, 1510, 1538, 1463, + 2101, 1512, 1514, 1539, 1519, 1544, 1278, 1422, 1467, 1421, + + 1511, 1513, 1537, 1534, 1534, 1534, 1201, 1421, 1545, 1541, + 1541, 1541, 1546, 1201, 1421, 1465, 1547, 1538, 1515, 1063, + 1063, 1254, 1539, 1419, 1544, 1515, 1063, 1063, 1516, 1511, + 1419, 1420, 1457, 1535, 1456, 1550, 1421, 1545, 1420, 1542, + 1552, 1546, 1556, 1422, 1421, 1547, 1557, 1558, 1548, 1548, + 1548, 1421, 1451, 1451, 1451, 1560, 1560, 1560, 1518, 1563, + 1564, 1520, 1535, 1115, 1550, 1516, 1565, 1566, 1542, 1552, + 1567, 1556, 1568, 1421, 1569, 1557, 1558, 1570, 1570, 1570, + 1421, 1572, 1452, 1573, 1574, 1561, 1575, 1536, 1563, 1564, + 1520, 1576, 1577, 1543, 1578, 1565, 1566, 1579, 1580, 1567, + + 1581, 1568, 1516, 1569, 1521, 1582, 1583, 1235, 1584, 1516, + 1572, 1452, 1573, 1574, 1561, 1575, 1487, 2101, 1589, 1487, + 1576, 1577, 1590, 1578, 1585, 395, 1579, 1580, 1594, 1581, + 1595, 395, 1549, 395, 1582, 1583, 1453, 1584, 1596, 1562, + 1586, 1586, 1586, 1592, 1597, 1598, 1593, 1589, 1599, 1600, + 1605, 1590, 1591, 1606, 1601, 1601, 1601, 1594, 1613, 1595, + 1225, 1571, 1554, 2101, 1614, 1615, 1554, 1596, 1602, 1616, + 1587, 1637, 1592, 1597, 1598, 1593, 1423, 1599, 1600, 1605, + 1620, 1591, 1606, 1621, 1517, 1063, 1063, 1613, 1624, 1509, + 1508, 1414, 1414, 1614, 1615, 1509, 1396, 1510, 1616, 1587, + + 1517, 1063, 1063, 1510, 1381, 1509, 1517, 1063, 1063, 1620, + 1511, 1509, 1621, 1510, 1626, 1629, 1511, 1624, 1380, 1607, + 1517, 1063, 1063, 1358, 1588, 1509, 1511, 1617, 1617, 1617, + 1357, 1114, 1511, 1510, 1570, 1570, 1570, 1339, 1603, 1511, + 1631, 1618, 1632, 1626, 1629, 1511, 1511, 1517, 1063, 1063, + 1338, 1332, 1509, 1331, 1085, 1511, 1063, 1197, 1063, 1063, + 1510, 1511, 1198, 1608, 1622, 1622, 1622, 395, 1518, 1631, + 1199, 1632, 1633, 1511, 1512, 1511, 1627, 1627, 1627, 1226, + 1197, 1063, 1063, 1200, 1518, 1198, 395, 1534, 1534, 1534, + 1518, 1236, 1608, 1199, 1515, 1063, 1063, 992, 1266, 1419, + + 1634, 1633, 1511, 1636, 1518, 1264, 1200, 1420, 1515, 1063, + 1063, 1619, 1200, 1419, 1517, 1063, 1063, 1535, 1571, 1509, + 1421, 1420, 1641, 1548, 1548, 1548, 1609, 1510, 970, 1634, + 1243, 1518, 1636, 1242, 1421, 1200, 1413, 959, 1115, 1217, + 1511, 1201, 1517, 1063, 1063, 1208, 1535, 1509, 1623, 1421, + 1642, 1641, 1203, 1202, 1194, 1510, 880, 1418, 1414, 1414, + 1628, 878, 1419, 1421, 1201, 1541, 1541, 1541, 1511, 1511, + 1519, 1536, 1643, 876, 1638, 1638, 1638, 1186, 1516, 1642, + 1515, 1063, 1063, 1421, 1646, 1419, 1182, 1610, 1180, 1515, + 1063, 1063, 1516, 1420, 1419, 1542, 1648, 1511, 1518, 1649, + + 1650, 1643, 1420, 1612, 1639, 1651, 1421, 1549, 1644, 1644, + 1644, 1652, 1421, 1646, 1653, 1421, 1560, 1560, 1560, 1654, + 1177, 1611, 1656, 1255, 1542, 1648, 1518, 1657, 1649, 1650, + 1658, 1659, 1612, 1639, 1651, 1421, 1660, 1661, 1662, 1175, + 1652, 1422, 1663, 1653, 1421, 1672, 1561, 1673, 1654, 1543, + 1674, 1656, 1675, 1160, 1148, 818, 1657, 1122, 1640, 1658, + 1659, 1678, 1680, 1111, 1516, 1660, 1661, 1662, 1418, 1681, + 1683, 1663, 1104, 1516, 1672, 1561, 1673, 395, 1103, 1674, + 1684, 1675, 1676, 1676, 1676, 1586, 1586, 1586, 747, 734, + 1678, 1680, 1645, 1685, 1686, 1075, 725, 1279, 1681, 1683, + + 1562, 1664, 1734, 1734, 1734, 1682, 1692, 1665, 1698, 1684, + 1699, 1666, 1667, 1700, 1708, 1587, 1668, 1669, 1670, 1711, + 1671, 720, 1685, 1686, 1601, 1601, 1601, 1068, 1060, 1709, + 1664, 1687, 1687, 1687, 1682, 1692, 1665, 1698, 1602, 1699, + 1666, 1667, 1700, 1708, 1587, 1668, 1669, 1670, 1711, 1671, + 1693, 1693, 1693, 707, 1688, 1508, 1414, 1414, 1709, 1056, + 1509, 1051, 1517, 1063, 1063, 1046, 1677, 1509, 1607, 1588, + 1694, 1038, 1025, 1689, 1013, 1510, 991, 1517, 1063, 1063, + 1712, 1511, 1509, 1688, 1714, 1696, 1735, 1715, 1511, 961, + 1510, 955, 1515, 1063, 1063, 554, 927, 1419, 1716, 1517, + + 1063, 1063, 1689, 1511, 1509, 1420, 1718, 913, 1603, 1712, + 1511, 1719, 1510, 1714, 1696, 1690, 1715, 1511, 1421, 1617, + 1617, 1617, 1701, 1701, 1701, 1511, 1722, 1716, 1705, 1705, + 1705, 907, 1511, 1618, 1695, 1718, 1515, 1063, 1063, 1512, + 1719, 1419, 1622, 1622, 1622, 1702, 1518, 1421, 1706, 1420, + 1627, 1627, 1627, 1724, 1511, 1722, 1508, 1226, 1720, 1720, + 1720, 1518, 1421, 1725, 1726, 1236, 1638, 1638, 1638, 1728, + 1697, 1515, 1729, 1350, 1702, 1730, 1516, 904, 1517, 1644, + 1644, 1644, 1724, 1518, 1731, 1732, 1733, 1736, 1736, 1736, + 1739, 1421, 1725, 1726, 1255, 1743, 1639, 899, 1728, 1697, + + 1744, 1729, 1745, 1619, 1730, 894, 1703, 1741, 1741, 1741, + 1746, 1749, 1707, 1731, 1732, 1733, 1750, 1751, 1752, 1739, + 1516, 885, 1757, 1758, 1743, 1639, 1623, 1747, 1753, 1744, + 1748, 1745, 1759, 1754, 1628, 1760, 1762, 1763, 1764, 1746, + 1749, 1755, 1721, 1766, 1767, 1750, 1751, 1752, 1756, 395, + 1640, 1757, 1758, 1676, 1676, 1676, 1747, 1753, 1765, 1748, + 1768, 1759, 1754, 1645, 1760, 1762, 1763, 1764, 1279, 1769, + 1755, 1737, 1766, 1767, 1687, 1687, 1687, 1756, 1770, 1772, + 1693, 1693, 1693, 1773, 1773, 1773, 486, 1765, 1780, 1768, + 704, 1742, 1781, 1705, 1705, 1705, 1787, 1688, 1769, 1774, + + 1694, 1515, 1063, 1063, 1788, 1789, 1419, 1770, 1772, 1790, + 1517, 1063, 1063, 1706, 1420, 1509, 1689, 1780, 1701, 1701, + 1701, 1781, 1791, 1510, 1792, 1787, 1688, 1421, 1793, 704, + 1783, 1783, 1783, 1788, 1789, 480, 1511, 1677, 1790, 1794, + 1798, 1702, 1799, 1800, 1777, 1689, 1784, 1795, 1795, 1795, + 1802, 1791, 1803, 1792, 1804, 1805, 1421, 1793, 1690, 1720, + 1720, 1720, 1807, 1810, 1695, 1511, 699, 1775, 1794, 1798, + 1702, 1799, 1800, 1777, 1350, 1813, 699, 1707, 473, 1802, + 1816, 1803, 1818, 1804, 1805, 1516, 1734, 1734, 1734, 1819, + 1820, 1807, 1810, 1821, 1518, 1736, 1736, 1736, 1811, 1811, + + 1811, 1822, 1703, 1823, 1813, 1741, 1741, 1741, 1824, 1816, + 1825, 1818, 1826, 1827, 1785, 1828, 1829, 1830, 1819, 1820, + 1831, 1832, 1821, 1833, 1834, 694, 395, 1836, 694, 866, + 1822, 1796, 1823, 1837, 1839, 863, 816, 1824, 1841, 1825, + 1842, 1826, 1827, 1721, 1828, 1829, 1830, 1814, 796, 1831, + 1832, 754, 1833, 1834, 1835, 1843, 1836, 1693, 1693, 1693, + 547, 553, 1837, 1839, 1773, 1773, 1773, 1841, 1845, 1842, + 1735, 1846, 746, 1517, 1063, 1063, 1814, 1694, 1509, 1737, + 1774, 531, 1812, 1835, 1843, 1851, 1510, 1852, 1853, 1742, + 1705, 1705, 1705, 1783, 1783, 1783, 1855, 1845, 1856, 1511, + + 1846, 1847, 1847, 1847, 1849, 1849, 1849, 1857, 1858, 1784, + 1706, 1795, 1795, 1795, 1851, 1866, 1852, 1853, 1859, 1859, + 1859, 1862, 1862, 1862, 1869, 1855, 1871, 1856, 1511, 1867, + 1867, 1867, 1811, 1811, 1811, 1863, 1857, 1858, 1872, 1875, + 1873, 1695, 1876, 1877, 1866, 545, 1878, 1881, 1775, 1874, + 1879, 1880, 1882, 1869, 1883, 1871, 518, 1518, 1884, 526, + 1885, 1886, 1887, 1888, 1888, 1888, 511, 1872, 1875, 1873, + 1890, 1876, 1877, 516, 1707, 1878, 1881, 1785, 395, 1879, + 1880, 1882, 1892, 1883, 718, 1848, 1893, 1884, 1850, 1885, + 1886, 1887, 1894, 1895, 1896, 1796, 1897, 1898, 1899, 1890, + + 1900, 1903, 1860, 1904, 496, 1864, 508, 1891, 1847, 1847, + 1847, 1892, 1907, 1868, 1908, 1893, 1812, 1849, 1849, 1849, + 1909, 1894, 1895, 1896, 1911, 1897, 1898, 1899, 1917, 1900, + 1903, 1918, 1904, 1905, 1905, 1905, 1891, 1859, 1859, 1859, + 1921, 1907, 1922, 1908, 1862, 1862, 1862, 1889, 1923, 1909, + 1924, 1925, 1926, 1911, 1912, 1912, 1912, 1917, 1863, 1927, + 1918, 1867, 1867, 1867, 1928, 1929, 1930, 706, 1931, 1921, + 1932, 1922, 705, 395, 1933, 1934, 1935, 1923, 1937, 1924, + 1925, 1926, 1888, 1888, 1888, 1939, 1940, 1941, 1927, 1942, + 1943, 1913, 1848, 1928, 1929, 1930, 1914, 1931, 1944, 1932, + + 1938, 1850, 1919, 1933, 1934, 1935, 1945, 1937, 1946, 1905, + 1905, 1905, 1951, 1952, 1939, 1940, 1941, 1906, 1942, 1943, + 1913, 1860, 1953, 1954, 1956, 1914, 1957, 1944, 1864, 1938, + 1958, 1919, 1912, 1912, 1912, 1945, 1959, 1946, 1915, 1960, + 704, 1951, 1952, 1961, 1962, 1868, 1963, 1964, 1965, 1966, + 1967, 1953, 1954, 1956, 1968, 1957, 1969, 1970, 1971, 1958, + 1972, 1973, 1974, 1975, 701, 1959, 1889, 1976, 1960, 1913, + 1977, 1978, 1961, 1962, 1914, 1963, 1964, 1965, 1966, 1967, + 1979, 1983, 1984, 1968, 1987, 1969, 1970, 1971, 1988, 1972, + 1973, 1974, 1975, 1906, 1989, 1990, 1976, 1993, 1913, 1977, + + 1978, 1994, 1995, 1914, 1991, 1991, 1991, 1996, 1997, 1979, + 1983, 1984, 1998, 1987, 1999, 2000, 1915, 1988, 2001, 2002, + 2003, 2004, 2005, 1989, 1990, 2007, 1993, 2008, 2009, 2010, + 1994, 1995, 2011, 2012, 2015, 700, 1996, 1997, 2013, 2013, + 2013, 1998, 2017, 1999, 2000, 2018, 2019, 2001, 2002, 2003, + 2004, 2005, 2020, 2021, 2007, 2023, 2008, 2009, 2010, 699, + 2024, 2011, 2012, 2015, 1991, 1991, 1991, 2025, 2026, 2027, + 2028, 2017, 2029, 2032, 2018, 2019, 1991, 1991, 1991, 2033, + 2034, 2020, 2021, 2035, 2023, 2013, 2013, 2013, 1992, 2024, + 2036, 2037, 2038, 696, 2039, 2040, 2025, 2026, 2027, 2028, + + 2041, 2029, 2032, 2042, 2043, 2044, 2045, 2046, 2033, 2034, + 2047, 2048, 2035, 2049, 2050, 2084, 2084, 2084, 2055, 2036, + 2037, 2038, 2014, 2039, 2040, 2056, 2057, 2058, 2059, 2041, + 2060, 695, 2042, 2043, 2044, 2045, 2046, 2061, 2067, 2047, + 2048, 694, 2049, 2050, 2051, 2051, 2051, 2055, 1992, 2070, + 2071, 2072, 2052, 691, 2056, 2057, 2058, 2059, 2073, 2060, + 1992, 1693, 1693, 1693, 2075, 2076, 2061, 2067, 2077, 2014, + 1705, 1705, 1705, 2078, 2051, 2051, 2051, 2079, 2070, 2071, + 2072, 1694, 2052, 2080, 2081, 2082, 2083, 2073, 2086, 2087, + 1706, 2088, 2089, 2075, 2076, 2091, 2092, 2077, 2093, 2085, + + 690, 2094, 2078, 2084, 2084, 2084, 2079, 2084, 2084, 2084, + 2095, 2096, 2080, 2081, 2082, 2083, 689, 2086, 2087, 2097, + 2088, 2089, 2098, 2099, 2091, 2092, 2100, 2093, 2053, 2054, + 2094, 1991, 1991, 1991, 2084, 2084, 2084, 688, 687, 2095, + 2096, 683, 446, 674, 673, 1695, 672, 671, 2097, 670, + 427, 2098, 2099, 421, 1707, 2100, 417, 392, 2053, 2054, + 416, 303, 375, 310, 305, 376, 553, 550, 546, 545, + 541, 527, 526, 522, 517, 516, 514, 509, 508, 501, + 500, 495, 493, 471, 470, 467, 466, 2085, 463, 462, + 461, 2085, 460, 448, 447, 444, 443, 440, 439, 427, + + 421, 417, 416, 389, 377, 376, 375, 314, 310, 305, + 2101, 218, 218, 216, 216, 1992, 2101, 2101, 2085, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 163, + 163, 163, 163, 163, 163, 163, 163, 163, 163, 97, + 97, 97, 97, 97, 97, 97, 97, 97, 97, 190, + 190, 190, 190, 190, 190, 190, 190, 190, 190, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 206, + 206, 206, 206, 206, 206, 206, 206, 206, 206, 209, + 209, 209, 209, 209, 209, 209, 209, 209, 209, 212, + + 212, 212, 212, 212, 212, 212, 212, 212, 212, 215, + 215, 215, 215, 215, 215, 215, 215, 215, 215, 217, + 217, 217, 217, 217, 217, 217, 217, 217, 217, 238, + 238, 238, 238, 238, 238, 238, 238, 238, 238, 240, + 240, 240, 240, 240, 240, 240, 240, 240, 240, 268, + 268, 268, 268, 268, 268, 268, 268, 268, 268, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 294, + 294, 294, 294, 294, 294, 294, 294, 294, 294, 306, + 306, 307, 307, 2101, 307, 307, 307, 307, 307, 307, + 307, 309, 309, 374, 2101, 2101, 2101, 2101, 374, 393, + + 393, 2101, 393, 2101, 393, 393, 418, 418, 420, 420, + 426, 2101, 2101, 2101, 2101, 426, 428, 2101, 428, 472, + 472, 472, 472, 472, 2101, 472, 472, 2101, 472, 478, + 478, 478, 478, 478, 478, 478, 478, 478, 478, 479, + 479, 479, 479, 479, 2101, 479, 479, 2101, 479, 485, + 485, 485, 485, 485, 485, 485, 485, 485, 485, 491, + 491, 491, 491, 491, 491, 491, 491, 491, 491, 492, + 492, 492, 492, 492, 492, 492, 492, 492, 494, 494, + 494, 494, 494, 494, 494, 494, 494, 498, 498, 2101, + 498, 498, 498, 498, 498, 498, 498, 513, 513, 2101, + + 513, 513, 513, 513, 513, 513, 513, 520, 520, 2101, + 520, 520, 520, 520, 520, 520, 520, 533, 533, 2101, + 533, 533, 533, 533, 533, 533, 533, 549, 549, 2101, + 549, 549, 549, 549, 549, 549, 549, 306, 306, 307, + 307, 2101, 307, 307, 307, 307, 307, 307, 307, 309, + 309, 374, 2101, 2101, 2101, 2101, 374, 393, 393, 2101, + 393, 2101, 393, 393, 418, 418, 420, 420, 426, 2101, + 2101, 2101, 2101, 426, 428, 2101, 428, 472, 472, 472, + 472, 472, 2101, 472, 472, 2101, 472, 693, 693, 2101, + 693, 693, 693, 693, 693, 693, 693, 479, 479, 479, + + 479, 479, 2101, 479, 479, 2101, 479, 698, 698, 2101, + 698, 698, 698, 698, 698, 698, 698, 703, 703, 2101, + 703, 703, 703, 703, 703, 703, 703, 492, 492, 492, + 492, 492, 492, 492, 492, 492, 494, 494, 494, 494, + 494, 494, 494, 494, 494, 498, 498, 2101, 498, 498, + 498, 498, 498, 498, 498, 513, 513, 2101, 513, 513, + 513, 513, 513, 513, 513, 520, 520, 2101, 520, 520, + 520, 520, 520, 520, 520, 533, 533, 2101, 533, 533, + 533, 533, 533, 533, 533, 549, 549, 2101, 549, 549, + 549, 549, 549, 549, 549, 751, 751, 751, 751, 751, + + 751, 751, 751, 751, 751, 393, 2101, 2101, 393, 428, + 2101, 428, 882, 882, 882, 882, 882, 882, 882, 882, + 882, 882, 896, 896, 896, 896, 896, 896, 896, 896, + 896, 896, 901, 901, 901, 901, 901, 901, 901, 901, + 901, 901, 910, 910, 910, 910, 910, 910, 910, 910, + 910, 910, 924, 924, 924, 924, 924, 924, 924, 924, + 924, 924, 1042, 1042, 1042, 1042, 1042, 1042, 1042, 1042, + 1042, 1042, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, + 1047, 1047, 1052, 1052, 1052, 1052, 1052, 1052, 1052, 1052, + 1052, 1052, 1196, 1196, 1196, 1196, 1196, 1196, 1196, 1196, + + 1196, 1196, 1344, 1344, 1344, 1344, 1344, 1344, 1344, 1344, + 1344, 1344, 1417, 1417, 1417, 1417, 1417, 1417, 1417, 1417, + 1417, 1417, 1436, 1436, 1436, 1436, 1436, 1436, 1436, 1436, + 1436, 1436, 1443, 1443, 1443, 1443, 1443, 1443, 1443, 1443, + 1443, 1443, 1462, 1462, 1462, 1462, 1462, 1462, 1462, 1462, + 1462, 1462, 1486, 1486, 1486, 1486, 1486, 1486, 1486, 1486, + 1486, 1486, 1507, 1507, 1507, 1507, 1507, 1507, 1507, 1507, + 1507, 1507, 1553, 1553, 1553, 1553, 1553, 1553, 1553, 1553, + 1553, 1553, 53, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101 } ; -static yyconst flex_int16_t yy_chk[6218] = +static yyconst flex_int16_t yy_chk[6271] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1860,15 +1877,15 @@ static yyconst flex_int16_t yy_chk[6218] = 14, 14, 14, 13, 28, 29, 13, 14, 13, 30, 14, 31, 14, 15, 15, 15, 32, 16, 16, 16, 15, 39, 40, 15, 16, 15, 76, 16, 47, 16, - 48, 100, 2153, 79, 2151, 68, 100, 68, 68, 71, + 48, 100, 2182, 79, 2180, 68, 100, 68, 68, 71, 81, 27, 82, 71, 39, 40, 28, 29, 68, 85, - 73, 30, 2149, 72, 73, 76, 31, 70, 47, 70, + 73, 30, 2178, 72, 73, 76, 31, 70, 47, 70, 48, 32, 79, 73, 70, 70, 70, 72, 71, 81, 72, 82, 71, 39, 40, 101, 101, 68, 85, 73, 13, 13, 72, 73, 14, 14, 70, 47, 70, 48, 102, 102, 73, 70, 70, 70, 72, 15, 15, 72, - 2148, 16, 16, 19, 19, 19, 19, 19, 19, 19, + 2177, 16, 16, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, @@ -1879,18 +1896,18 @@ static yyconst flex_int16_t yy_chk[6218] = 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 21, 21, 21, 22, 22, 22, 280, 280, 21, 90, - 112, 22, 23, 23, 23, 112, 2145, 23, 2143, 23, - 92, 23, 23, 24, 24, 24, 2142, 103, 24, 104, + 112, 22, 23, 23, 23, 112, 2174, 23, 2172, 23, + 92, 23, 23, 24, 24, 24, 2171, 103, 24, 104, 24, 74, 24, 24, 25, 25, 25, 74, 90, 25, 105, 25, 25, 25, 25, 26, 26, 26, 75, 92, - 26, 75, 26, 26, 26, 26, 103, 77, 104, 2139, + 26, 75, 26, 26, 26, 26, 103, 77, 104, 2168, - 74, 77, 80, 117, 106, 77, 74, 2130, 117, 105, + 74, 77, 80, 117, 106, 77, 74, 2159, 117, 105, 80, 78, 86, 107, 78, 84, 80, 75, 283, 283, - 75, 78, 86, 84, 21, 21, 77, 22, 22, 2126, + 75, 78, 86, 84, 21, 21, 77, 22, 22, 2155, 77, 80, 84, 106, 77, 84, 23, 23, 108, 80, - 78, 86, 107, 78, 84, 80, 2123, 24, 24, 133, - 78, 86, 84, 2120, 133, 142, 2116, 159, 25, 25, + 78, 86, 107, 78, 84, 80, 2152, 24, 24, 133, + 78, 86, 84, 2149, 133, 142, 2145, 159, 25, 25, 142, 84, 159, 167, 84, 311, 311, 108, 167, 26, 26, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, @@ -1902,17 +1919,17 @@ static yyconst flex_int16_t yy_chk[6218] = 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 41, 41, - 41, 2114, 41, 41, 2113, 41, 41, 41, 42, 42, + 41, 2143, 41, 41, 134, 41, 41, 41, 42, 42, 42, 41, 42, 42, 83, 42, 42, 42, 83, 87, - 2110, 42, 87, 88, 134, 135, 83, 89, 87, 89, - 83, 148, 89, 88, 87, 149, 150, 91, 151, 41, - - 173, 173, 2101, 83, 91, 91, 180, 83, 87, 42, - 181, 87, 88, 134, 135, 83, 89, 87, 89, 83, - 148, 89, 88, 87, 149, 150, 91, 151, 41, 173, - 173, 176, 196, 91, 91, 180, 176, 196, 42, 181, - 203, 378, 41, 41, 282, 203, 378, 282, 282, 376, - 376, 376, 42, 42, 43, 43, 43, 43, 43, 43, + 2142, 42, 87, 135, 88, 148, 83, 89, 87, 89, + 83, 88, 89, 134, 87, 149, 150, 91, 151, 41, + + 180, 88, 2139, 83, 91, 91, 181, 83, 87, 42, + 2130, 87, 135, 88, 148, 83, 89, 87, 89, 83, + 88, 89, 230, 87, 149, 150, 91, 151, 41, 180, + 88, 176, 196, 91, 91, 181, 176, 196, 42, 203, + 2126, 379, 41, 41, 203, 282, 379, 2123, 282, 282, + 2120, 230, 42, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, @@ -1922,25 +1939,25 @@ static yyconst flex_int16_t yy_chk[6218] = 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, - 43, 45, 45, 45, 45, 45, 45, 45, 230, 45, - 45, 45, 45, 45, 45, 45, 45, 45, 380, 380, - 45, 45, 45, 45, 45, 381, 381, 232, 183, 2097, - 45, 55, 55, 55, 56, 56, 56, 230, 233, 182, - 2094, 116, 116, 116, 182, 183, 2091, 119, 119, 119, - 234, 45, 116, 45, 116, 116, 232, 183, 119, 45, - - 119, 119, 388, 388, 121, 121, 121, 233, 182, 220, - 220, 220, 2061, 182, 183, 121, 239, 121, 121, 234, + 43, 45, 45, 45, 45, 45, 45, 45, 232, 45, + 45, 45, 45, 45, 45, 45, 45, 45, 381, 381, + 45, 45, 45, 45, 45, 382, 382, 173, 173, 2090, + 45, 55, 55, 55, 56, 56, 56, 232, 233, 182, + 2085, 116, 116, 116, 182, 234, 2074, 119, 119, 119, + 239, 45, 116, 45, 116, 116, 173, 173, 119, 45, + + 119, 119, 389, 389, 121, 121, 121, 233, 182, 220, + 220, 220, 653, 182, 234, 121, 653, 121, 121, 239, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 121, 46, 46, 46, 46, - 46, 46, 46, 46, 46, 239, 249, 46, 46, 46, - 46, 46, 262, 263, 264, 55, 152, 46, 56, 2056, - 152, 270, 272, 231, 121, 116, 231, 288, 289, 152, - 290, 119, 300, 152, 301, 249, 319, 321, 46, 389, - 46, 262, 263, 264, 389, 152, 46, 2033, 121, 152, - 270, 272, 231, 220, 2032, 231, 288, 289, 152, 290, - - 651, 300, 152, 301, 651, 319, 321, 46, 46, 46, + 46, 46, 46, 46, 46, 183, 249, 46, 46, 46, + 46, 46, 262, 263, 264, 55, 152, 46, 56, 2066, + 152, 270, 183, 231, 121, 116, 231, 272, 288, 152, + 289, 119, 290, 152, 183, 249, 300, 301, 46, 390, + 46, 262, 263, 264, 390, 152, 46, 2065, 121, 152, + 270, 183, 231, 220, 2064, 231, 272, 288, 152, 289, + + 2057, 290, 152, 471, 471, 300, 301, 46, 46, 46, 46, 46, 46, 46, 46, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, @@ -1951,547 +1968,552 @@ static yyconst flex_int16_t yy_chk[6218] = 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 51, 51, 51, 313, 313, 51, 2030, 52, - 52, 52, 322, 323, 52, 51, 313, 324, 118, 118, - 118, 51, 52, 2012, 325, 221, 221, 221, 52, 118, - 51, 118, 118, 241, 241, 241, 327, 52, 122, 122, - 122, 322, 323, 402, 118, 313, 324, 51, 402, 122, - 421, 122, 122, 325, 52, 421, 124, 124, 124, 51, - 1998, 284, 653, 284, 284, 327, 52, 124, 328, 124, - 124, 1996, 122, 118, 284, 329, 51, 328, 123, 123, - 123, 843, 124, 52, 328, 843, 51, 470, 470, 123, - 330, 123, 123, 52, 114, 114, 114, 328, 653, 315, - - 315, 122, 118, 284, 329, 114, 328, 114, 114, 221, - 315, 124, 656, 328, 125, 125, 125, 241, 123, 330, - 114, 114, 122, 114, 331, 125, 1994, 125, 125, 242, - 242, 242, 114, 428, 114, 333, 114, 125, 428, 315, - 124, 253, 253, 253, 126, 126, 126, 123, 656, 114, - 114, 1989, 114, 331, 317, 126, 317, 126, 126, 317, - 317, 114, 123, 114, 333, 114, 125, 254, 254, 254, - 207, 207, 207, 336, 1984, 207, 1975, 207, 114, 115, - 115, 115, 126, 207, 210, 210, 210, 516, 516, 210, - 115, 210, 115, 115, 460, 460, 460, 210, 125, 213, - - 213, 213, 336, 213, 213, 115, 115, 316, 115, 316, - 316, 126, 213, 242, 274, 274, 274, 115, 337, 115, - 316, 115, 275, 275, 275, 253, 1966, 440, 126, 295, - 295, 295, 440, 338, 115, 115, 1965, 115, 296, 296, - 296, 303, 303, 303, 318, 339, 115, 337, 115, 316, - 115, 254, 318, 340, 207, 312, 312, 312, 1939, 326, - 326, 332, 338, 115, 127, 127, 127, 326, 210, 1938, - 341, 1937, 332, 318, 339, 127, 334, 127, 127, 334, - 335, 318, 340, 213, 1935, 341, 341, 127, 326, 326, - 332, 127, 342, 1932, 335, 343, 326, 1923, 274, 341, - - 127, 332, 345, 346, 127, 334, 275, 347, 334, 335, - 1921, 348, 349, 295, 341, 341, 127, 350, 351, 1905, - 127, 342, 296, 335, 343, 303, 344, 352, 344, 127, - 353, 345, 346, 127, 354, 355, 347, 1901, 356, 312, - 348, 349, 356, 358, 360, 357, 350, 351, 127, 361, - 357, 362, 358, 359, 358, 344, 352, 344, 357, 353, - 357, 359, 363, 354, 355, 364, 357, 356, 365, 368, - 369, 356, 358, 360, 357, 368, 370, 371, 361, 357, - 362, 358, 359, 358, 367, 364, 382, 357, 367, 357, - 359, 363, 383, 366, 364, 357, 366, 365, 368, 369, - - 384, 385, 386, 387, 368, 370, 371, 423, 1900, 424, - 526, 526, 1895, 367, 364, 382, 430, 367, 444, 431, - 432, 383, 366, 444, 433, 366, 392, 392, 392, 384, - 385, 386, 387, 391, 391, 391, 423, 392, 424, 392, - 392, 393, 393, 393, 391, 430, 391, 391, 431, 432, - 534, 534, 393, 433, 393, 393, 395, 395, 395, 391, - 391, 1894, 391, 396, 396, 396, 434, 395, 1893, 395, - 395, 391, 435, 391, 396, 391, 396, 396, 1892, 397, - 397, 397, 436, 437, 449, 450, 453, 395, 391, 391, - 397, 391, 397, 397, 451, 434, 1888, 396, 454, 451, - - 391, 435, 391, 455, 391, 456, 1887, 398, 398, 398, - 392, 436, 437, 449, 450, 453, 395, 391, 398, 397, - 398, 398, 399, 399, 399, 393, 396, 454, 558, 558, - 398, 1875, 455, 399, 456, 399, 399, 457, 1894, 458, - 395, 404, 404, 404, 463, 467, 1856, 396, 397, 463, - 467, 501, 404, 502, 404, 404, 405, 405, 405, 398, - 399, 503, 1854, 397, 504, 404, 457, 405, 458, 405, - 405, 406, 472, 472, 472, 406, 479, 479, 479, 472, - 501, 505, 502, 479, 406, 407, 407, 407, 406, 399, - 503, 398, 1850, 504, 404, 506, 407, 405, 407, 407, - - 406, 408, 408, 408, 406, 1846, 399, 400, 400, 400, - 505, 1338, 408, 406, 408, 408, 1338, 406, 400, 409, - 400, 400, 407, 409, 506, 404, 405, 1841, 509, 514, - 400, 1837, 409, 522, 400, 1835, 409, 559, 559, 408, - 405, 523, 524, 400, 495, 495, 495, 400, 409, 737, - 737, 407, 409, 410, 410, 410, 472, 509, 514, 400, - 479, 409, 522, 400, 410, 409, 410, 410, 408, 407, - 523, 524, 400, 411, 411, 411, 400, 1831, 410, 412, - 412, 412, 738, 738, 411, 408, 411, 411, 749, 749, - 412, 400, 412, 412, 499, 499, 499, 411, 413, 413, - - 413, 414, 485, 485, 485, 414, 485, 410, 1827, 413, - 528, 413, 413, 500, 414, 510, 510, 510, 414, 529, - 412, 513, 513, 513, 753, 753, 411, 1802, 495, 541, - 414, 500, 500, 542, 414, 754, 754, 410, 540, 528, - 413, 1799, 500, 414, 517, 517, 517, 414, 529, 412, - 521, 521, 521, 530, 530, 530, 540, 411, 541, 848, - 500, 500, 542, 412, 535, 535, 535, 540, 543, 413, - 536, 536, 537, 537, 538, 550, 538, 538, 499, 1429, - 551, 536, 413, 537, 1429, 540, 485, 538, 539, 561, - 539, 755, 755, 539, 539, 848, 562, 543, 563, 510, - - 546, 546, 546, 564, 550, 513, 549, 549, 549, 551, - 536, 565, 537, 553, 553, 553, 538, 555, 561, 555, - 556, 556, 555, 555, 557, 562, 557, 563, 517, 557, - 557, 556, 564, 566, 521, 567, 568, 530, 569, 570, - 565, 571, 572, 569, 574, 573, 575, 576, 535, 573, - 577, 578, 579, 580, 582, 1796, 583, 584, 585, 586, - 556, 1795, 566, 587, 567, 568, 589, 569, 570, 590, - 571, 572, 569, 574, 573, 575, 576, 591, 573, 577, - 578, 579, 580, 582, 546, 583, 584, 585, 586, 592, - 549, 593, 587, 594, 595, 589, 596, 553, 590, 598, - - 599, 600, 601, 602, 603, 604, 591, 605, 606, 607, - 608, 609, 610, 611, 612, 613, 614, 615, 592, 616, - 593, 618, 594, 595, 1792, 596, 620, 621, 598, 599, - 600, 601, 602, 603, 604, 623, 605, 606, 607, 608, - 609, 610, 611, 612, 613, 614, 615, 622, 616, 622, - 618, 619, 619, 619, 624, 620, 621, 625, 626, 627, - 628, 629, 630, 631, 623, 624, 632, 633, 634, 635, - 639, 640, 634, 641, 642, 643, 622, 644, 622, 650, - 649, 647, 652, 624, 648, 655, 625, 626, 627, 628, - 629, 630, 631, 648, 624, 632, 633, 634, 635, 639, - - 640, 634, 641, 642, 643, 647, 644, 649, 650, 654, - 652, 658, 657, 655, 659, 660, 661, 662, 664, 665, - 667, 673, 648, 657, 674, 675, 676, 677, 678, 654, - 679, 680, 756, 756, 647, 619, 649, 650, 1790, 652, - 658, 1785, 655, 659, 660, 661, 662, 664, 665, 667, - 673, 707, 657, 674, 675, 676, 677, 678, 654, 679, - 680, 693, 693, 693, 698, 698, 698, 703, 703, 703, - 705, 705, 705, 708, 709, 710, 711, 712, 713, 714, - 707, 717, 718, 718, 718, 720, 723, 723, 723, 725, - 726, 730, 731, 732, 732, 732, 739, 740, 735, 735, - - 741, 742, 708, 709, 710, 711, 712, 713, 714, 735, - 717, 734, 743, 734, 720, 747, 734, 734, 725, 726, - 730, 731, 736, 748, 736, 739, 740, 736, 736, 741, - 742, 745, 745, 745, 750, 750, 750, 1784, 735, 751, - 751, 743, 757, 757, 747, 693, 758, 759, 698, 760, - 761, 703, 748, 762, 705, 763, 765, 766, 767, 768, - 765, 769, 770, 771, 772, 773, 718, 776, 777, 778, - 723, 779, 780, 781, 782, 758, 759, 732, 760, 761, - 783, 786, 762, 787, 763, 765, 766, 767, 768, 765, - 769, 770, 771, 772, 773, 788, 776, 777, 778, 793, - - 779, 780, 781, 782, 790, 790, 790, 794, 795, 783, - 786, 796, 787, 797, 798, 745, 800, 751, 750, 801, - 802, 802, 802, 803, 788, 804, 805, 806, 793, 808, - 809, 810, 811, 812, 813, 814, 794, 795, 815, 819, - 796, 820, 797, 798, 822, 800, 823, 824, 801, 818, - 818, 818, 803, 825, 804, 805, 806, 826, 808, 809, - 810, 811, 812, 813, 814, 827, 828, 815, 819, 830, - 820, 831, 832, 822, 833, 823, 824, 851, 850, 835, - 837, 838, 825, 839, 841, 855, 826, 845, 790, 844, - 1774, 850, 1773, 846, 827, 828, 856, 851, 830, 1770, - - 831, 832, 849, 833, 802, 816, 816, 816, 835, 837, - 838, 852, 839, 841, 855, 845, 844, 846, 1764, 849, - 850, 853, 857, 858, 859, 856, 851, 816, 852, 860, - 816, 816, 861, 818, 853, 862, 863, 864, 866, 867, - 869, 816, 816, 816, 845, 844, 846, 816, 849, 870, - 871, 857, 858, 859, 878, 878, 1763, 852, 860, 816, - 816, 861, 1759, 853, 862, 863, 864, 866, 867, 869, - 816, 816, 816, 872, 872, 872, 816, 882, 870, 871, - 872, 874, 874, 874, 879, 879, 879, 883, 874, 816, - 876, 876, 876, 885, 876, 880, 880, 886, 887, 887, - - 887, 888, 890, 887, 891, 896, 882, 892, 892, 894, - 894, 887, 893, 893, 893, 901, 883, 897, 897, 898, - 898, 898, 885, 899, 899, 903, 886, 906, 906, 915, - 888, 890, 916, 891, 896, 907, 907, 907, 908, 908, - 910, 910, 911, 911, 901, 912, 912, 913, 913, 914, - 914, 917, 918, 1758, 903, 919, 925, 872, 915, 920, - 920, 916, 921, 921, 921, 874, 922, 922, 879, 926, - 926, 928, 1749, 880, 876, 927, 927, 927, 929, 932, - 917, 918, 887, 933, 919, 925, 934, 894, 935, 936, - 1732, 938, 939, 940, 941, 942, 893, 943, 944, 945, - - 928, 899, 946, 898, 950, 951, 952, 929, 932, 954, - 1235, 1235, 933, 957, 959, 934, 908, 935, 936, 907, - 938, 939, 940, 941, 942, 961, 943, 944, 945, 962, - 1730, 946, 963, 950, 951, 952, 965, 968, 954, 955, - 955, 955, 957, 959, 922, 972, 921, 926, 960, 960, - 960, 966, 966, 966, 961, 973, 974, 975, 962, 927, - 980, 963, 976, 960, 977, 965, 968, 978, 955, 979, - 981, 982, 983, 984, 972, 980, 985, 955, 986, 990, - 992, 994, 995, 997, 973, 974, 975, 998, 1000, 966, - 1001, 976, 1002, 977, 1010, 1004, 978, 955, 979, 981, - - 982, 983, 984, 1011, 980, 985, 955, 986, 990, 992, - 994, 995, 997, 1007, 1008, 1014, 998, 1000, 966, 1001, - 1011, 1002, 1010, 955, 1004, 1013, 1019, 1014, 1012, 1017, - 1015, 1016, 960, 1018, 1725, 966, 988, 988, 988, 1021, - 1436, 1017, 1007, 1008, 1013, 1436, 1022, 1024, 1723, 1011, - 1026, 1010, 1720, 1027, 1016, 1019, 1014, 1029, 1031, 1032, - 1033, 988, 988, 988, 1012, 988, 1015, 1715, 1021, 1018, - 1017, 1051, 1051, 1013, 988, 1022, 1024, 988, 988, 1026, - 1454, 988, 1027, 1016, 1052, 1454, 1029, 1031, 1032, 1033, - 988, 988, 988, 1712, 988, 1036, 1036, 1037, 1037, 1037, - - 1478, 1036, 1710, 988, 1037, 1478, 988, 988, 1038, 1038, - 988, 1039, 1039, 1052, 1038, 1041, 1041, 1039, 1065, 1065, - 988, 1041, 1042, 1042, 1042, 1043, 1043, 1044, 1044, 1042, - 1053, 1043, 1054, 1044, 1046, 1046, 1055, 1046, 1047, 1047, - 1047, 1061, 1047, 1048, 1048, 1062, 1048, 1049, 1049, 1051, - 1049, 1063, 1702, 1057, 1057, 1057, 1066, 1069, 1057, 1053, - 1699, 1054, 1058, 1058, 1058, 1055, 1057, 1058, 1070, 1072, - 1061, 1067, 1067, 1073, 1062, 1058, 1071, 1071, 1074, 1057, - 1063, 1037, 1075, 1076, 1081, 1066, 1069, 1082, 1058, 1039, - 1077, 1077, 1079, 1079, 1079, 1084, 1065, 1070, 1072, 1086, - - 1087, 1696, 1073, 1089, 1090, 1044, 1042, 1074, 1057, 1092, - 1094, 1075, 1076, 1081, 1095, 1096, 1082, 1058, 1097, 1091, - 1091, 1091, 1047, 1098, 1084, 1049, 1100, 1693, 1086, 1087, - 1079, 1103, 1089, 1090, 1091, 1104, 1106, 1057, 1092, 1094, - 1107, 1111, 1692, 1095, 1096, 1688, 1058, 1097, 1112, 1067, - 1114, 1115, 1098, 1117, 1071, 1100, 1102, 1102, 1102, 1079, - 1103, 1118, 1120, 1122, 1104, 1106, 1123, 1124, 1077, 1107, - 1111, 1102, 1108, 1108, 1108, 1127, 1079, 1112, 1128, 1114, - 1115, 1129, 1117, 1125, 1125, 1125, 1130, 1108, 1132, 1126, - 1118, 1120, 1122, 1126, 1126, 1123, 1124, 1134, 1125, 1135, - - 1136, 1137, 1138, 1091, 1127, 1139, 1140, 1128, 1141, 1143, - 1129, 1545, 1145, 1146, 1687, 1130, 1545, 1132, 1126, 1148, - 1150, 1151, 1126, 1126, 1153, 1154, 1134, 1684, 1135, 1136, - 1137, 1138, 1680, 1156, 1139, 1140, 1157, 1141, 1143, 1159, - 1102, 1145, 1146, 1147, 1147, 1147, 1156, 1157, 1148, 1150, - 1151, 1158, 1159, 1153, 1154, 1161, 1108, 1162, 1147, 1165, - 1166, 1164, 1167, 1168, 1170, 1171, 1162, 1125, 1158, 1170, - 1173, 1175, 1161, 1176, 1177, 1156, 1157, 1161, 1164, 1178, - 1179, 1159, 1324, 1324, 1180, 1180, 1183, 1184, 1165, 1166, - 1180, 1167, 1168, 1170, 1171, 1162, 1185, 1158, 1170, 1173, - - 1175, 1161, 1176, 1177, 1181, 1181, 1161, 1164, 1178, 1179, - 1181, 1182, 1182, 1186, 1182, 1183, 1184, 1187, 1188, 1189, - 1189, 1189, 1331, 1331, 1189, 1185, 1934, 1147, 1190, 1190, - 1190, 1198, 1189, 1190, 1679, 1191, 1191, 1191, 1668, 1199, - 1191, 1190, 1186, 1201, 1202, 1189, 1187, 1188, 1191, 1666, - 1192, 1192, 1192, 1647, 1190, 1192, 1195, 1195, 1195, 1203, - 1198, 1191, 1180, 1192, 1204, 1193, 1193, 1193, 1199, 1645, - 1193, 1205, 1201, 1202, 1189, 1206, 1192, 1208, 1193, 1337, - 1337, 1209, 1181, 1190, 1212, 1214, 1215, 1644, 1203, 1182, - 1191, 1193, 1221, 1204, 1224, 1225, 1194, 1194, 1194, 1226, - - 1205, 1194, 1934, 1189, 1206, 1192, 1208, 1227, 1193, 1194, - 1209, 1636, 1190, 1212, 1214, 1215, 1218, 1218, 1218, 1191, - 1193, 1221, 1194, 1224, 1225, 1231, 1232, 1634, 1226, 1233, - 1234, 1218, 1239, 1240, 1192, 1244, 1227, 1193, 1246, 1630, - 1195, 1228, 1228, 1228, 1250, 1243, 1243, 1243, 1251, 1193, - 1252, 1194, 1253, 1254, 1231, 1232, 1228, 1255, 1233, 1234, - 1243, 1239, 1240, 1256, 1244, 1625, 1257, 1246, 1247, 1247, - 1247, 1623, 1258, 1250, 1259, 1194, 1260, 1251, 1261, 1252, - 1194, 1253, 1254, 1247, 1262, 1263, 1255, 1264, 1265, 1266, - 1247, 1267, 1256, 1268, 1247, 1257, 1280, 1269, 1270, 1275, - - 1218, 1258, 1276, 1259, 1277, 1260, 1278, 1261, 1279, 1271, - 1271, 1271, 1282, 1262, 1263, 1287, 1264, 1265, 1266, 1620, - 1267, 1618, 1268, 1247, 1271, 1228, 1269, 1270, 1275, 1243, - 1281, 1276, 1280, 1277, 1279, 1278, 1283, 1284, 1285, 1286, - 1288, 1290, 1281, 1291, 1287, 1282, 1284, 1292, 1283, 1293, - 1285, 1286, 1247, 1294, 1295, 1296, 1297, 1298, 1299, 1300, - 1301, 1302, 1303, 1279, 1615, 1304, 1305, 1349, 1349, 1288, - 1290, 1281, 1291, 1613, 1282, 1284, 1292, 1283, 1293, 1285, - 1286, 1311, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, - 1302, 1303, 1312, 1271, 1304, 1305, 1306, 1306, 1306, 1609, - - 1313, 1306, 1307, 1307, 1307, 1314, 1315, 1307, 1593, 1306, - 1311, 1308, 1308, 1308, 1316, 1307, 1308, 1317, 1309, 1309, - 1309, 1312, 1306, 1309, 1308, 1307, 1318, 1320, 1307, 1313, - 1321, 1309, 1322, 1323, 1314, 1315, 1327, 1308, 1328, 1329, - 1330, 1333, 1334, 1316, 1309, 1335, 1317, 1336, 1339, 1339, - 1339, 1306, 1340, 1341, 1307, 1318, 1320, 1307, 1346, 1321, - 1347, 1322, 1323, 1348, 1352, 1327, 1308, 1328, 1329, 1330, - 1333, 1334, 1354, 1309, 1335, 1356, 1336, 1357, 1339, 1355, - 1306, 1340, 1341, 1342, 1342, 1342, 1307, 1346, 1358, 1347, - 1308, 1359, 1348, 1352, 1360, 1308, 1361, 1355, 1342, 1362, - - 1363, 1354, 1309, 1383, 1356, 1364, 1357, 1339, 1355, 1365, - 1366, 1367, 1368, 1370, 1372, 1372, 1374, 1358, 1376, 1377, - 1359, 1378, 1388, 1360, 1383, 1361, 1355, 1379, 1362, 1363, - 1365, 1391, 1339, 1395, 1364, 1398, 1399, 1400, 1365, 1366, - 1367, 1368, 1370, 1401, 1379, 1374, 1402, 1376, 1377, 1403, - 1404, 1388, 1416, 1383, 1428, 1428, 1583, 1378, 1417, 1365, - 1391, 1582, 1395, 1418, 1398, 1399, 1400, 1342, 1419, 1405, - 1405, 1405, 1401, 1379, 1405, 1402, 1435, 1435, 1403, 1404, - 1578, 1416, 1405, 1406, 1406, 1406, 1561, 1417, 1406, 1407, - 1407, 1407, 1418, 1552, 1407, 1405, 1406, 1419, 1420, 1408, - - 1408, 1408, 1407, 1421, 1408, 1409, 1409, 1409, 1542, 1406, - 1409, 1540, 1408, 1422, 1423, 1407, 1424, 1426, 1409, 1534, - 1427, 1410, 1410, 1410, 1405, 1408, 1410, 1420, 1527, 1432, - 1433, 1409, 1421, 1408, 1410, 1411, 1411, 1411, 1406, 1494, - 1411, 1449, 1422, 1423, 1407, 1424, 1426, 1410, 1411, 1427, - 1434, 1438, 1439, 1405, 1408, 1430, 1430, 1430, 1432, 1433, - 1409, 1411, 1408, 1440, 1437, 1437, 1437, 1406, 1407, 1412, - 1412, 1412, 1445, 1407, 1412, 1441, 1410, 1448, 1448, 1434, - 1438, 1439, 1412, 1408, 1444, 1430, 1413, 1413, 1413, 1409, - 1411, 1413, 1440, 1415, 1437, 1412, 1414, 1414, 1414, 1413, - - 1397, 1414, 1447, 1451, 1441, 1410, 1442, 1442, 1442, 1414, - 1387, 1452, 1413, 1444, 1430, 1443, 1443, 1443, 1456, 1411, - 1457, 1442, 1414, 1437, 1412, 1453, 1453, 1458, 1459, 1413, - 1460, 1447, 1451, 1455, 1455, 1455, 1461, 1462, 1464, 1430, - 1452, 1413, 1463, 1463, 1463, 1443, 1385, 1456, 1437, 1457, - 1465, 1414, 1466, 1412, 1467, 1468, 1458, 1459, 1413, 1460, - 1469, 1384, 1470, 1455, 1471, 1461, 1462, 1464, 1472, 1473, - 1413, 1474, 1475, 1476, 1443, 1414, 1477, 1477, 1481, 1465, - 1414, 1466, 1482, 1467, 1468, 1479, 1479, 1479, 1382, 1469, - 1442, 1470, 1455, 1471, 1483, 1486, 1484, 1472, 1473, 1443, - - 1474, 1475, 1476, 1485, 1487, 1488, 1489, 1481, 1484, 1490, - 1491, 1482, 1492, 1495, 1496, 1479, 1485, 1455, 1493, 1493, - 1493, 1483, 1544, 1544, 1486, 1381, 1463, 1513, 1380, 1515, - 1373, 1516, 1493, 1487, 1488, 1489, 1353, 1484, 1490, 1491, - 1351, 1492, 1495, 1496, 1479, 1485, 1498, 1498, 1498, 1350, - 1483, 1498, 1499, 1499, 1499, 1344, 1513, 1499, 1515, 1498, - 1516, 1500, 1500, 1500, 1518, 1499, 1500, 1343, 1332, 1479, - 1520, 1325, 1498, 1310, 1500, 1521, 1289, 1526, 1499, 1501, - 1501, 1501, 1273, 1530, 1501, 1272, 1533, 1500, 1535, 1249, - 1248, 1236, 1501, 1518, 1230, 1502, 1502, 1502, 1229, 1520, - - 1502, 1498, 1493, 1220, 1521, 1501, 1526, 1499, 1502, 1503, - 1503, 1503, 1530, 1536, 1503, 1533, 1500, 1535, 1504, 1504, - 1504, 1502, 1503, 1504, 1219, 1537, 1210, 1505, 1505, 1505, - 1498, 1504, 1505, 1538, 1501, 1503, 1499, 1541, 1502, 1196, - 1505, 1163, 1536, 1160, 1504, 1500, 1506, 1506, 1506, 1547, - 1502, 1506, 1142, 1505, 1537, 1507, 1507, 1507, 1133, 1506, - 1507, 1131, 1538, 1501, 1503, 1548, 1541, 1502, 1507, 1519, - 1519, 1519, 1506, 1504, 1524, 1524, 1524, 1116, 1547, 1502, - 1110, 1507, 1505, 1519, 1109, 1525, 1525, 1525, 1503, 1524, - 1508, 1508, 1508, 1503, 1548, 1508, 1551, 1504, 1531, 1531, - - 1531, 1506, 1504, 1508, 1539, 1539, 1539, 1509, 1509, 1509, - 1507, 1505, 1509, 1531, 1105, 1525, 1508, 1080, 1553, 1539, - 1509, 1068, 1510, 1510, 1510, 1551, 1060, 1510, 1554, 1059, - 1506, 1056, 1050, 1509, 1507, 1510, 1532, 1532, 1532, 1507, - 1045, 1511, 1511, 1511, 1525, 1508, 1511, 1553, 1510, 1040, - 1512, 1512, 1512, 1519, 1511, 1512, 1034, 1554, 1524, 1546, - 1546, 1546, 1509, 1512, 1511, 1555, 1532, 1511, 1556, 1525, - 1549, 1549, 1549, 1557, 1508, 1558, 1512, 1510, 1550, 1550, - 1550, 1559, 1531, 1562, 1563, 1549, 1509, 1564, 1539, 1546, - 1565, 1509, 1566, 1511, 1555, 1532, 1511, 1556, 1560, 1560, - - 1560, 1567, 1557, 1568, 1558, 1512, 1510, 1569, 1550, 1571, - 1559, 1572, 1562, 1563, 1581, 1573, 1564, 1574, 1546, 1565, - 1532, 1566, 1576, 1576, 1576, 1511, 1575, 1575, 1575, 1512, - 1567, 1577, 1568, 1030, 1512, 1579, 1569, 1550, 1571, 1580, - 1572, 1575, 1581, 1546, 1573, 1584, 1574, 1587, 1588, 1590, - 1594, 1603, 1576, 1028, 1549, 1591, 1591, 1591, 1605, 1025, - 1577, 1023, 1550, 1006, 1579, 1592, 1592, 1592, 1580, 1591, - 989, 1581, 1606, 987, 1584, 967, 1587, 1588, 1590, 1594, - 1603, 1576, 1560, 1570, 1595, 1595, 1595, 1605, 1592, 1570, - 1597, 1597, 1597, 1570, 1570, 1597, 956, 949, 1570, 1570, - - 1570, 1606, 1570, 1597, 1595, 947, 1576, 1592, 1614, 923, - 1575, 909, 1570, 902, 1616, 900, 1597, 1592, 1570, 1619, - 895, 889, 1570, 1570, 1610, 1610, 1610, 1570, 1570, 1570, - 884, 1570, 881, 1598, 1598, 1598, 1592, 1614, 1598, 1591, - 1607, 1607, 1607, 1616, 1610, 1597, 1598, 877, 1619, 1592, - 1599, 1599, 1599, 875, 1607, 1599, 1598, 873, 1621, 1598, - 1600, 1600, 1600, 1599, 868, 1600, 1611, 854, 1595, 840, - 1601, 1601, 1601, 1600, 1597, 1601, 1599, 1602, 1602, 1602, - 1622, 1611, 1602, 1601, 1624, 1598, 1600, 1621, 1598, 817, - 1602, 1608, 1608, 1608, 1626, 1611, 1601, 1612, 1612, 1612, - - 1628, 1628, 1628, 1602, 791, 1599, 1629, 1631, 1610, 1622, - 1611, 1602, 1612, 1624, 1608, 1600, 1632, 1598, 1617, 1617, - 1617, 785, 752, 1626, 1607, 1601, 1627, 1627, 1627, 1599, - 1628, 1635, 1602, 1617, 1599, 1629, 1631, 1637, 1638, 1600, - 1602, 1627, 1639, 1608, 1600, 1632, 746, 1640, 1641, 1601, - 1633, 1633, 1633, 1646, 1601, 1642, 1642, 1642, 1649, 1628, - 1635, 1602, 1643, 1643, 1643, 1633, 1637, 1638, 1648, 1648, - 1648, 1639, 1650, 733, 1651, 1608, 1640, 1641, 1652, 1654, - 1653, 1612, 1646, 1653, 1628, 1655, 1656, 1649, 1657, 1658, - 1659, 1660, 1661, 1663, 727, 724, 719, 1658, 1657, 1667, - - 1669, 1650, 1617, 1651, 1658, 715, 1670, 1652, 1654, 1653, - 1627, 706, 1653, 1672, 1655, 1656, 1674, 1657, 1658, 1659, - 1660, 1661, 1663, 1665, 1665, 1665, 1658, 1657, 1667, 1669, - 1675, 1671, 1677, 1658, 1633, 1670, 1678, 1681, 1665, 1642, - 1671, 704, 1672, 702, 701, 1674, 1643, 1676, 1676, 1676, - 699, 1689, 1648, 1682, 1682, 1682, 1683, 1683, 1683, 1675, - 697, 1677, 1685, 1685, 1685, 1678, 1681, 1685, 1691, 1671, - 1676, 696, 1683, 1682, 694, 1685, 1697, 1686, 1686, 1686, - 1689, 1698, 1686, 1690, 1690, 1690, 1700, 692, 1685, 1676, - 1686, 1694, 1694, 1694, 1701, 1703, 1685, 1691, 1704, 1676, - - 1695, 1695, 1695, 1686, 1705, 1697, 1690, 1665, 691, 1706, - 1698, 1694, 1708, 1708, 1708, 1700, 1695, 1685, 1676, 1709, - 1709, 1709, 1711, 1701, 1703, 1685, 1713, 1704, 1714, 1716, - 1717, 1676, 1686, 1705, 1709, 1690, 1718, 1682, 1706, 1719, - 1683, 1721, 1722, 1722, 1722, 1726, 1685, 1724, 1724, 1724, - 1728, 1711, 1727, 1727, 1727, 1713, 1731, 1714, 1716, 1717, - 1733, 1686, 1729, 1729, 1729, 1718, 1734, 1690, 1719, 1735, - 1721, 1736, 1737, 1738, 1726, 1694, 1739, 1740, 1741, 1728, - 1742, 1744, 1745, 1746, 1695, 1731, 1747, 1750, 1751, 1733, - 1752, 666, 1753, 1754, 663, 1734, 1708, 617, 1735, 1757, - - 1736, 1737, 1738, 1709, 1729, 1739, 1740, 1741, 1760, 1742, - 1744, 1745, 1746, 1766, 597, 1747, 1750, 1751, 1767, 1752, - 1753, 554, 1754, 1756, 1756, 1756, 1722, 1769, 1757, 552, - 547, 1724, 1775, 1729, 545, 544, 1727, 1760, 1761, 1761, - 1761, 1776, 1766, 1756, 531, 1780, 1729, 1767, 525, 1753, - 1765, 1765, 1765, 1781, 1761, 1765, 1769, 1768, 1768, 1768, - 1782, 1775, 1786, 1765, 1771, 1771, 1771, 1777, 1777, 1777, - 1776, 1779, 1779, 1779, 1780, 1787, 1765, 1768, 518, 1788, - 1771, 1793, 1781, 1783, 1783, 1783, 1789, 1789, 1789, 1782, - 1797, 1786, 1791, 1791, 1791, 1794, 1794, 1794, 1798, 1798, - - 1798, 1800, 1801, 1803, 1787, 1765, 1791, 1756, 1788, 1804, - 1793, 1805, 1803, 1806, 1807, 1808, 1808, 1809, 1810, 1797, - 1812, 1813, 1761, 1814, 1815, 1816, 1819, 1819, 1819, 1821, - 1800, 1801, 1803, 1822, 1765, 1823, 1824, 1825, 1804, 1826, - 1805, 1768, 1806, 1807, 1808, 1808, 1809, 1810, 1771, 1812, - 1813, 1777, 1814, 1815, 1816, 1779, 1828, 1829, 1821, 1830, - 1832, 1833, 1822, 515, 1823, 1824, 1825, 1783, 1826, 1838, - 1789, 1834, 1834, 1834, 1839, 1843, 1791, 1844, 1847, 1794, - 1851, 511, 1798, 508, 507, 1828, 1829, 496, 1830, 1832, - 1833, 1822, 1836, 1836, 1836, 1842, 1842, 1842, 1838, 1845, - - 1845, 1845, 1852, 1839, 1843, 1855, 1844, 1847, 1857, 1851, - 1819, 1848, 1848, 1848, 1849, 1849, 1849, 1853, 1853, 1853, - 1858, 1860, 489, 1861, 1862, 1848, 488, 487, 1863, 483, - 1864, 1852, 1865, 1866, 1855, 1868, 1869, 1857, 1870, 1871, - 1873, 1874, 1874, 1874, 1876, 482, 1878, 1877, 1879, 1858, - 1860, 1849, 1861, 1862, 480, 1834, 1849, 1863, 1853, 1864, - 1880, 1865, 1866, 1884, 1868, 1869, 476, 1870, 1871, 1873, - 1885, 1886, 475, 1876, 1877, 1878, 1836, 1879, 1889, 1842, - 1849, 1890, 1896, 1845, 473, 1849, 1898, 1853, 1899, 1880, - 1902, 1903, 1884, 1904, 1906, 1848, 469, 1907, 1849, 1885, - - 1886, 1853, 1908, 1877, 1891, 1891, 1891, 1889, 1909, 1911, - 1890, 1896, 1897, 1897, 1897, 1898, 1912, 1899, 1913, 1902, - 1903, 1914, 1904, 1906, 1915, 1874, 1907, 1916, 1917, 1919, - 1922, 1908, 1925, 1926, 1927, 1928, 1929, 1909, 1911, 1930, - 1931, 1933, 1936, 1940, 1941, 1912, 1942, 1913, 1943, 1897, - 1914, 466, 465, 1915, 1897, 1945, 1916, 1917, 1919, 1922, - 1946, 1925, 1926, 1927, 1928, 1929, 1947, 462, 1930, 1931, - 1933, 1936, 1940, 1941, 1949, 1942, 1951, 1943, 1897, 1944, - 1944, 1944, 1953, 1897, 1945, 1954, 1956, 1957, 1891, 1946, - 1958, 1960, 1961, 1963, 1967, 1947, 1897, 1968, 1969, 1970, - - 1971, 1973, 461, 1949, 459, 1951, 1972, 1972, 1972, 1977, - 1978, 1953, 1980, 1981, 1954, 1956, 1957, 1983, 1985, 1958, - 1960, 1961, 1963, 1967, 1986, 1988, 1968, 1969, 1970, 1971, - 1973, 1974, 1974, 1974, 447, 1990, 1991, 1992, 1977, 1978, - 1997, 1980, 1981, 1993, 1993, 1993, 1983, 1985, 1995, 1995, - 1995, 1999, 2001, 1986, 1988, 2003, 2004, 2005, 2007, 2008, - 2009, 446, 2010, 1944, 1990, 1991, 1992, 2011, 2013, 1997, - 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, - 1999, 2001, 2024, 2025, 2003, 2004, 2005, 2007, 2008, 2009, - 1972, 2010, 2028, 2027, 2027, 2027, 2011, 2013, 443, 2014, - - 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2031, - 2034, 2024, 2025, 2027, 2035, 1974, 2029, 2029, 2029, 2037, - 2038, 2028, 2039, 2040, 2041, 2042, 2043, 1993, 2045, 2046, - 2047, 2048, 1995, 2050, 2050, 2050, 2029, 2051, 2031, 2034, - 2052, 2053, 2054, 2035, 2055, 2055, 2055, 2057, 2037, 2038, - 2059, 2039, 2040, 2041, 2042, 2043, 2060, 2045, 2046, 2047, - 2048, 2058, 2058, 2058, 2063, 2065, 2051, 2066, 2067, 2052, - 2053, 2054, 2064, 2064, 2064, 2068, 2057, 2027, 2069, 2059, - 2070, 2071, 2071, 2071, 442, 2060, 439, 438, 425, 419, - 417, 415, 401, 2063, 2065, 375, 2066, 2067, 373, 309, - - 2029, 306, 304, 302, 2068, 298, 293, 2069, 292, 2070, - 287, 266, 265, 261, 251, 250, 248, 2050, 237, 236, - 229, 227, 218, 216, 205, 204, 198, 197, 2055, 189, - 188, 185, 184, 169, 168, 162, 161, 154, 153, 136, - 132, 130, 128, 109, 95, 2058, 94, 93, 67, 61, - 58, 53, 36, 35, 34, 33, 2064, 0, 0, 0, - 0, 0, 0, 0, 0, 2071, 2073, 2073, 2073, 2073, - 2073, 2073, 2073, 2073, 2073, 2073, 2074, 2074, 2074, 2074, - 2074, 2074, 2074, 2074, 2074, 2074, 2075, 2075, 2075, 2075, - 2075, 2075, 2075, 2075, 2075, 2075, 2076, 2076, 2076, 2076, - - 2076, 2076, 2076, 2076, 2076, 2076, 2077, 2077, 2077, 2077, - 2077, 2077, 2077, 2077, 2077, 2077, 2078, 2078, 2078, 2078, - 2078, 2078, 2078, 2078, 2078, 2078, 2079, 2079, 2079, 2079, - 2079, 2079, 2079, 2079, 2079, 2079, 2080, 2080, 2080, 2080, - 2080, 2080, 2080, 2080, 2080, 2080, 2081, 2081, 2081, 2081, - 2081, 2081, 2081, 2081, 2081, 2081, 2082, 2082, 2082, 2082, - 2082, 2082, 2082, 2082, 2082, 2082, 2083, 2083, 2083, 2083, - 2083, 2083, 2083, 2083, 2083, 2083, 2084, 2084, 2084, 2084, - 2084, 2084, 2084, 2084, 2084, 2084, 2085, 2085, 2085, 2085, - 2085, 2085, 2085, 2085, 2085, 2085, 2086, 2086, 2086, 2086, - - 2086, 2086, 2086, 2086, 2086, 2086, 2087, 2087, 2087, 2087, - 2087, 2087, 2087, 2087, 2087, 2087, 2088, 2088, 2088, 2088, - 2088, 2088, 2088, 2088, 2088, 2088, 2089, 2089, 2090, 2090, - 0, 2090, 2090, 2090, 2090, 2090, 2090, 2090, 2092, 2092, - 2093, 0, 0, 0, 0, 2093, 2095, 2095, 0, 2095, - 0, 2095, 2095, 2096, 2096, 2098, 2098, 2099, 0, 0, - 0, 0, 2099, 2100, 0, 2100, 2102, 2102, 2102, 2102, - 2102, 0, 2102, 2102, 0, 2102, 2103, 2103, 2103, 2103, - 2103, 2103, 2103, 2103, 2103, 2103, 2104, 2104, 2104, 2104, - 2104, 0, 2104, 2104, 0, 2104, 2105, 2105, 2105, 2105, - - 2105, 2105, 2105, 2105, 2105, 2105, 2106, 2106, 2106, 2106, - 2106, 2106, 2106, 2106, 2106, 2106, 2107, 2107, 2107, 2107, - 2107, 2107, 2107, 2107, 2107, 2108, 2108, 2108, 2108, 2108, - 2108, 2108, 2108, 2108, 2109, 2109, 0, 2109, 2109, 2109, - 2109, 2109, 2109, 2109, 2111, 2111, 0, 2111, 2111, 2111, - 2111, 2111, 2111, 2111, 2112, 2112, 0, 2112, 2112, 2112, - 2112, 2112, 2112, 2112, 2115, 2115, 0, 2115, 2115, 2115, - 2115, 2115, 2115, 2115, 2117, 2117, 0, 2117, 2117, 2117, - 2117, 2117, 2117, 2117, 2118, 2118, 2119, 2119, 0, 2119, - 2119, 2119, 2119, 2119, 2119, 2119, 2121, 2121, 2122, 0, - - 0, 0, 0, 2122, 2124, 2124, 0, 2124, 0, 2124, - 2124, 2125, 2125, 2127, 2127, 2128, 0, 0, 0, 0, - 2128, 2129, 0, 2129, 2131, 2131, 2131, 2131, 2131, 0, - 2131, 2131, 0, 2131, 2132, 2132, 0, 2132, 2132, 2132, - 2132, 2132, 2132, 2132, 2133, 2133, 2133, 2133, 2133, 0, - 2133, 2133, 0, 2133, 2134, 2134, 0, 2134, 2134, 2134, - 2134, 2134, 2134, 2134, 2135, 2135, 0, 2135, 2135, 2135, - 2135, 2135, 2135, 2135, 2136, 2136, 2136, 2136, 2136, 2136, - 2136, 2136, 2136, 2137, 2137, 2137, 2137, 2137, 2137, 2137, - 2137, 2137, 2138, 2138, 0, 2138, 2138, 2138, 2138, 2138, - - 2138, 2138, 2140, 2140, 0, 2140, 2140, 2140, 2140, 2140, - 2140, 2140, 2141, 2141, 0, 2141, 2141, 2141, 2141, 2141, - 2141, 2141, 2144, 2144, 0, 2144, 2144, 2144, 2144, 2144, - 2144, 2144, 2146, 2146, 0, 2146, 2146, 2146, 2146, 2146, - 2146, 2146, 2147, 2147, 2147, 2147, 2147, 2147, 2147, 2147, - 2147, 2147, 2150, 0, 0, 2150, 2152, 0, 2152, 2154, - 2154, 2154, 2154, 2154, 2154, 2154, 2154, 2154, 2154, 2155, - 2155, 2155, 2155, 2155, 2155, 2155, 2155, 2155, 2155, 2156, - 2156, 2156, 2156, 2156, 2156, 2156, 2156, 2156, 2156, 2157, - 2157, 2157, 2157, 2157, 2157, 2157, 2157, 2157, 2157, 2158, - - 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2158, 2159, - 2159, 2159, 2159, 2159, 2159, 2159, 2159, 2159, 2159, 2160, - 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2161, - 2161, 2161, 2161, 2161, 2161, 2161, 2161, 2161, 2161, 2162, - 2162, 2162, 2162, 2162, 2162, 2162, 2162, 2162, 2162, 2163, - 2163, 2163, 2163, 2163, 2163, 2163, 2163, 2163, 2163, 2164, - 2164, 2164, 2164, 2164, 2164, 2164, 2164, 2164, 2164, 2165, - 2165, 2165, 2165, 2165, 2165, 2165, 2165, 2165, 2165, 2166, - 2166, 2166, 2166, 2166, 2166, 2166, 2166, 2166, 2166, 2167, - 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2168, - - 2168, 2168, 2168, 2168, 2168, 2168, 2168, 2168, 2168, 2169, - 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2169, 2170, - 2170, 2170, 2170, 2170, 2170, 2170, 2170, 2170, 2170, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - - 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, 2072, - 2072, 2072, 2072, 2072, 2072, 2072, 2072 + 49, 49, 51, 51, 51, 313, 313, 51, 2056, 52, + 52, 52, 319, 321, 52, 51, 313, 322, 118, 118, + 118, 51, 52, 2054, 323, 221, 221, 221, 52, 118, + 51, 118, 118, 241, 241, 241, 324, 52, 122, 122, + 122, 319, 321, 2053, 118, 313, 322, 51, 325, 122, + 1949, 122, 122, 323, 52, 327, 124, 124, 124, 51, + 2050, 284, 655, 284, 284, 324, 52, 124, 329, 124, + 124, 1908, 122, 118, 284, 330, 51, 325, 123, 123, + 123, 847, 124, 52, 327, 847, 51, 517, 517, 123, + 331, 123, 123, 52, 114, 114, 114, 329, 655, 315, + + 315, 122, 118, 284, 330, 114, 2031, 114, 114, 221, + 315, 124, 527, 527, 125, 125, 125, 241, 123, 331, + 114, 114, 122, 114, 332, 125, 1949, 125, 125, 242, + 242, 242, 114, 403, 114, 332, 114, 125, 403, 315, + 124, 253, 253, 253, 126, 126, 126, 123, 1908, 114, + 114, 2016, 114, 332, 317, 126, 317, 126, 126, 317, + 317, 114, 123, 114, 332, 114, 125, 254, 254, 254, + 207, 207, 207, 333, 2014, 207, 2012, 207, 114, 115, + 115, 115, 126, 207, 210, 210, 210, 535, 535, 210, + 115, 210, 115, 115, 377, 377, 377, 210, 125, 213, + + 213, 213, 333, 213, 213, 115, 115, 316, 115, 316, + 316, 126, 213, 242, 274, 274, 274, 115, 335, 115, + 316, 115, 275, 275, 275, 253, 559, 559, 126, 295, + 295, 295, 335, 336, 115, 115, 2006, 115, 296, 296, + 296, 303, 303, 303, 318, 337, 115, 335, 115, 316, + 115, 254, 318, 422, 207, 312, 312, 312, 422, 326, + 326, 335, 336, 115, 127, 127, 127, 326, 210, 2001, + 328, 1992, 338, 318, 337, 127, 339, 127, 127, 328, + 340, 318, 334, 213, 1982, 334, 328, 127, 326, 326, + 342, 127, 343, 1981, 345, 344, 326, 344, 274, 328, + + 127, 338, 341, 346, 127, 339, 275, 347, 328, 340, + 1955, 334, 348, 295, 334, 328, 127, 341, 341, 342, + 127, 343, 296, 345, 344, 303, 344, 349, 350, 127, + 351, 341, 346, 127, 352, 353, 347, 354, 355, 312, + 356, 348, 358, 360, 356, 361, 341, 341, 127, 362, + 359, 358, 363, 358, 364, 357, 349, 350, 359, 351, + 357, 366, 1954, 352, 353, 365, 354, 355, 357, 356, + 357, 358, 360, 356, 361, 370, 357, 371, 362, 359, + 358, 363, 358, 364, 357, 365, 372, 359, 367, 357, + 366, 367, 368, 383, 365, 369, 368, 357, 384, 357, + + 385, 369, 386, 387, 370, 357, 371, 388, 429, 560, + 560, 424, 1953, 429, 365, 372, 425, 367, 441, 1950, + 367, 368, 383, 441, 369, 368, 1947, 384, 1938, 385, + 369, 386, 387, 393, 393, 393, 388, 392, 392, 392, + 424, 394, 394, 394, 393, 425, 393, 393, 392, 431, + 392, 392, 394, 432, 394, 394, 396, 396, 396, 2052, + 397, 397, 397, 392, 392, 1936, 392, 396, 1920, 396, + 396, 397, 1916, 397, 397, 392, 433, 392, 431, 392, + 445, 452, 432, 434, 1915, 445, 452, 396, 398, 398, + 398, 435, 392, 392, 397, 392, 399, 399, 399, 398, + + 436, 398, 398, 1910, 392, 433, 392, 399, 392, 399, + 399, 1907, 434, 405, 405, 405, 396, 393, 437, 399, + 435, 392, 1906, 397, 405, 394, 405, 405, 398, 436, + 438, 450, 400, 400, 400, 2052, 464, 405, 1902, 451, + 396, 464, 1901, 400, 397, 400, 400, 437, 399, 1889, + 406, 406, 406, 1870, 407, 454, 455, 398, 407, 438, + 450, 406, 1868, 406, 406, 410, 405, 407, 451, 410, + 400, 407, 398, 456, 457, 408, 408, 408, 410, 458, + 399, 459, 410, 407, 454, 455, 408, 407, 408, 408, + 468, 406, 502, 503, 410, 468, 407, 405, 410, 400, + + 407, 1864, 456, 457, 461, 461, 461, 410, 458, 504, + 459, 410, 408, 409, 409, 409, 400, 401, 401, 401, + 406, 502, 503, 505, 409, 1860, 409, 409, 401, 506, + 401, 401, 739, 739, 406, 411, 411, 411, 504, 1854, + 401, 408, 507, 1850, 401, 658, 411, 510, 411, 411, + 1848, 409, 505, 401, 412, 412, 412, 401, 506, 408, + 411, 413, 413, 413, 1844, 412, 415, 412, 412, 401, + 415, 507, 413, 401, 413, 413, 510, 1840, 412, 415, + 409, 658, 401, 415, 1815, 540, 401, 540, 515, 411, + 540, 540, 414, 414, 414, 415, 1812, 409, 1809, 415, + + 523, 401, 413, 414, 524, 414, 414, 412, 415, 473, + 473, 473, 415, 480, 480, 480, 473, 515, 1808, 411, + 480, 486, 486, 486, 501, 486, 496, 496, 496, 523, + 525, 413, 529, 524, 414, 500, 500, 500, 412, 511, + 511, 511, 501, 501, 530, 413, 514, 514, 514, 518, + 518, 518, 1805, 501, 522, 522, 522, 542, 543, 525, + 544, 529, 541, 414, 531, 531, 531, 536, 536, 536, + 852, 501, 501, 530, 537, 537, 414, 538, 538, 539, + 541, 539, 539, 551, 552, 537, 542, 543, 538, 544, + 1803, 541, 539, 473, 547, 547, 547, 480, 550, 550, + + 550, 554, 554, 554, 1797, 486, 852, 557, 557, 541, + 496, 562, 551, 552, 537, 563, 564, 538, 557, 500, + 556, 539, 556, 511, 565, 556, 556, 566, 567, 558, + 514, 558, 568, 518, 558, 558, 569, 570, 522, 571, + 562, 572, 570, 573, 563, 564, 574, 557, 531, 1796, + 574, 536, 575, 565, 576, 577, 566, 567, 578, 579, + 580, 568, 581, 583, 584, 569, 570, 585, 571, 586, + 572, 570, 573, 587, 588, 574, 590, 591, 547, 574, + 592, 575, 550, 576, 577, 554, 593, 578, 579, 580, + 594, 581, 583, 584, 595, 596, 585, 597, 586, 599, + + 600, 601, 587, 588, 602, 590, 591, 603, 604, 592, + 605, 606, 607, 608, 609, 593, 610, 611, 612, 594, + 613, 614, 615, 595, 596, 616, 597, 617, 599, 600, + 601, 619, 621, 602, 622, 624, 603, 604, 625, 605, + 606, 607, 608, 609, 1786, 610, 611, 612, 627, 613, + 614, 615, 629, 623, 616, 623, 617, 620, 620, 620, + 619, 621, 626, 622, 624, 630, 628, 625, 628, 631, + 632, 633, 634, 626, 635, 636, 637, 627, 641, 636, + 642, 629, 623, 643, 623, 644, 645, 646, 1785, 649, + 651, 626, 652, 660, 630, 628, 654, 628, 631, 632, + + 633, 634, 626, 635, 636, 637, 656, 641, 636, 642, + 650, 657, 643, 649, 644, 645, 646, 651, 659, 650, + 661, 652, 660, 662, 654, 663, 656, 664, 666, 659, + 667, 669, 675, 676, 677, 678, 679, 680, 681, 657, + 682, 620, 649, 695, 695, 695, 651, 1782, 650, 661, + 652, 709, 662, 654, 663, 656, 664, 666, 659, 667, + 669, 675, 676, 677, 678, 679, 680, 681, 657, 682, + 700, 700, 700, 705, 705, 705, 707, 707, 707, 710, + 709, 711, 712, 713, 714, 715, 716, 719, 720, 720, + 720, 722, 725, 725, 725, 727, 728, 732, 1776, 733, + + 734, 734, 734, 740, 740, 741, 742, 1775, 710, 743, + 711, 712, 713, 714, 715, 716, 719, 736, 744, 736, + 722, 745, 736, 736, 727, 728, 732, 695, 733, 737, + 737, 738, 749, 738, 741, 742, 738, 738, 743, 750, + 737, 747, 747, 747, 751, 751, 760, 744, 753, 753, + 745, 752, 752, 752, 700, 755, 755, 705, 756, 756, + 707, 749, 757, 757, 758, 758, 759, 759, 750, 737, + 761, 762, 720, 763, 764, 760, 725, 765, 767, 768, + 769, 770, 767, 771, 734, 772, 773, 774, 775, 778, + 779, 780, 781, 782, 783, 784, 882, 882, 785, 761, + + 762, 788, 763, 764, 789, 790, 765, 767, 768, 769, + 770, 767, 771, 795, 772, 773, 774, 775, 778, 779, + 780, 781, 782, 783, 784, 747, 753, 785, 796, 797, + 788, 798, 799, 789, 790, 752, 792, 792, 792, 800, + 802, 803, 795, 804, 804, 804, 805, 806, 807, 808, + 810, 811, 812, 813, 814, 815, 816, 796, 797, 817, + 798, 799, 820, 820, 820, 821, 822, 824, 800, 802, + 803, 825, 826, 827, 828, 805, 806, 807, 808, 810, + 811, 812, 813, 814, 815, 816, 829, 830, 817, 831, + 832, 1771, 834, 835, 821, 822, 824, 836, 854, 850, + + 825, 826, 827, 828, 837, 839, 841, 842, 843, 845, + 1770, 854, 848, 1761, 849, 829, 830, 1744, 831, 832, + 792, 834, 835, 850, 855, 1742, 836, 804, 818, 818, + 818, 853, 856, 837, 839, 841, 842, 843, 845, 848, + 854, 1737, 849, 857, 855, 859, 820, 860, 853, 856, + 818, 861, 850, 818, 818, 862, 857, 863, 864, 865, + 866, 867, 868, 870, 818, 818, 818, 871, 848, 873, + 818, 849, 874, 855, 859, 875, 860, 853, 856, 1018, + 861, 1735, 818, 818, 862, 857, 863, 864, 865, 866, + 867, 868, 870, 818, 818, 818, 871, 886, 873, 818, + + 887, 874, 884, 884, 875, 889, 876, 876, 876, 878, + 878, 878, 818, 876, 890, 1018, 878, 880, 880, 880, + 892, 880, 883, 883, 883, 894, 886, 895, 900, 887, + 1732, 891, 891, 891, 889, 905, 891, 896, 896, 897, + 897, 897, 907, 890, 891, 1727, 898, 898, 1726, 892, + 901, 901, 903, 903, 894, 919, 895, 900, 902, 902, + 902, 910, 910, 920, 905, 911, 911, 911, 912, 912, + 921, 907, 914, 914, 915, 915, 916, 916, 917, 917, + 884, 918, 918, 922, 919, 923, 924, 924, 926, 926, + 876, 929, 920, 878, 925, 925, 925, 930, 930, 921, + + 932, 880, 931, 931, 931, 933, 883, 936, 937, 938, + 939, 940, 922, 1723, 923, 891, 942, 1721, 943, 944, + 929, 945, 946, 897, 898, 947, 948, 949, 950, 932, + 903, 954, 955, 956, 933, 958, 936, 937, 938, 939, + 940, 961, 902, 963, 965, 942, 912, 943, 944, 911, + 945, 946, 1713, 966, 947, 948, 949, 950, 967, 969, + 954, 955, 956, 972, 958, 1710, 926, 959, 959, 959, + 961, 976, 963, 965, 977, 930, 1707, 978, 925, 964, + 964, 964, 966, 970, 970, 970, 931, 967, 969, 979, + 980, 981, 972, 982, 964, 983, 959, 984, 985, 986, + + 976, 987, 988, 977, 989, 959, 978, 990, 994, 997, + 999, 1000, 984, 1002, 1003, 1004, 1006, 1007, 979, 980, + 981, 970, 982, 1008, 983, 959, 1010, 985, 986, 1013, + 987, 988, 1014, 989, 959, 1025, 990, 994, 997, 999, + 1000, 984, 1002, 1003, 1004, 1006, 1007, 1016, 1020, 1021, + 970, 959, 1008, 1019, 1017, 1010, 1022, 1027, 1013, 1028, + 1020, 1014, 1023, 964, 1025, 1704, 1024, 970, 992, 992, + 992, 1017, 1019, 1703, 1023, 1016, 1030, 1699, 1032, 1022, + 1242, 1242, 1033, 1035, 1698, 1021, 1027, 1037, 1028, 1020, + 1038, 1039, 1058, 992, 992, 992, 1059, 992, 1287, 1695, + + 1017, 1019, 1024, 1023, 1016, 1030, 992, 1032, 1022, 992, + 992, 1033, 1035, 992, 1052, 1052, 1037, 1052, 1691, 1038, + 1039, 1058, 992, 992, 992, 1059, 992, 1042, 1042, 1043, + 1043, 1043, 1345, 1042, 1287, 992, 1043, 1345, 992, 992, + 1044, 1044, 992, 1045, 1045, 1690, 1044, 1047, 1047, 1045, + 1057, 1057, 992, 1047, 1048, 1048, 1048, 1049, 1049, 1050, + 1050, 1048, 1060, 1049, 1061, 1050, 1053, 1053, 1053, 1679, + 1053, 1054, 1054, 1067, 1054, 1055, 1055, 1068, 1055, 1063, + 1063, 1063, 1677, 1069, 1063, 1064, 1064, 1064, 1071, 1071, + 1064, 1060, 1063, 1061, 1072, 1073, 1073, 1075, 1064, 1658, + + 1076, 1078, 1067, 1077, 1077, 1063, 1068, 1079, 1083, 1083, + 1080, 1064, 1069, 1043, 1081, 1082, 1085, 1085, 1085, 1087, + 1088, 1045, 1090, 1072, 1331, 1331, 1075, 1092, 1057, 1076, + 1078, 1093, 1095, 1096, 1063, 1098, 1079, 1050, 1048, 1080, + 1064, 1100, 1101, 1081, 1082, 1102, 1103, 1104, 1087, 1088, + 1053, 1090, 1656, 1055, 1085, 1106, 1092, 1097, 1097, 1097, + 1093, 1095, 1096, 1063, 1098, 1109, 1071, 1110, 1112, 1064, + 1100, 1101, 1097, 1073, 1102, 1103, 1104, 1108, 1108, 1108, + 1113, 1077, 1117, 1085, 1106, 1118, 1083, 1114, 1114, 1114, + 1120, 1121, 1108, 1123, 1109, 1124, 1110, 1112, 1126, 1128, + + 1085, 1129, 1114, 1130, 1133, 1134, 1131, 1131, 1131, 1113, + 1135, 1117, 1655, 1132, 1118, 1136, 1138, 1132, 1132, 1120, + 1121, 1131, 1123, 1140, 1124, 1141, 1142, 1126, 1128, 1143, + 1129, 1144, 1130, 1133, 1134, 1145, 1146, 1147, 1149, 1135, + 1151, 1097, 1132, 1152, 1136, 1138, 1132, 1132, 1154, 1154, + 1154, 1155, 1140, 1157, 1141, 1142, 1158, 1160, 1143, 1161, + 1144, 1108, 1163, 1154, 1145, 1146, 1147, 1149, 1164, 1151, + 1165, 1114, 1152, 1172, 1168, 1163, 1166, 1171, 1173, 1164, + 1155, 1174, 1157, 1169, 1175, 1158, 1160, 1165, 1161, 1166, + 1131, 1168, 1169, 1177, 1171, 1178, 1168, 1180, 1177, 1182, + + 1183, 1184, 1172, 1647, 1163, 1185, 1186, 1173, 1164, 1645, + 1174, 1187, 1187, 1175, 1190, 1191, 1165, 1187, 1166, 1192, + 1168, 1169, 1177, 1171, 1178, 1168, 1180, 1177, 1182, 1183, + 1184, 1193, 1154, 1194, 1185, 1186, 1188, 1188, 1189, 1189, + 1195, 1189, 1188, 1190, 1191, 1196, 1196, 1196, 1192, 1640, + 1196, 1197, 1197, 1197, 1205, 1635, 1197, 1386, 1196, 1206, + 1193, 1633, 1194, 1630, 1197, 1198, 1198, 1198, 1208, 1195, + 1198, 1196, 1202, 1202, 1202, 1209, 1628, 1197, 1198, 1199, + 1199, 1199, 1625, 1205, 1199, 1200, 1200, 1200, 1206, 1187, + 1200, 1198, 1199, 1386, 1210, 1211, 1212, 1208, 1200, 1623, + + 1196, 1619, 1213, 1215, 1209, 1199, 1197, 1216, 1219, 1221, + 1222, 1200, 1228, 1231, 1188, 1603, 1189, 1201, 1201, 1201, + 1198, 1232, 1201, 1210, 1211, 1212, 1233, 1593, 1200, 1196, + 1201, 1213, 1215, 1234, 1199, 1197, 1216, 1219, 1221, 1222, + 1200, 1228, 1231, 1201, 1225, 1225, 1225, 1238, 1239, 1198, + 1232, 1235, 1235, 1235, 1240, 1233, 1202, 1200, 1241, 1225, + 1246, 1247, 1234, 1199, 1251, 1253, 1235, 1592, 1257, 1200, + 1588, 1258, 1201, 1250, 1250, 1250, 1238, 1239, 1259, 1260, + 1261, 1571, 1262, 1240, 1254, 1254, 1254, 1241, 1250, 1246, + 1247, 1263, 1264, 1251, 1253, 1265, 1201, 1257, 1266, 1254, + + 1258, 1201, 1267, 1268, 1269, 1270, 1254, 1259, 1260, 1261, + 1254, 1262, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1562, + 1263, 1264, 1254, 1282, 1265, 1283, 1284, 1266, 1225, 1285, + 1286, 1267, 1268, 1269, 1270, 1235, 1289, 1294, 1295, 1254, + 1551, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1278, + 1278, 1254, 1282, 1288, 1283, 1284, 1286, 1250, 1285, 1290, + 1292, 1291, 1293, 1278, 1297, 1288, 1294, 1295, 1254, 1289, + 1291, 1290, 1292, 1298, 1293, 1299, 1300, 1301, 1302, 1303, + 1304, 1305, 1306, 1307, 1308, 1286, 1309, 1310, 1311, 1312, + 1318, 1338, 1338, 1297, 1288, 1319, 1344, 1344, 1289, 1291, + + 1290, 1292, 1298, 1293, 1299, 1300, 1301, 1302, 1303, 1304, + 1305, 1306, 1307, 1308, 1549, 1309, 1310, 1311, 1312, 1318, + 1313, 1313, 1313, 1543, 1319, 1313, 1314, 1314, 1314, 1320, + 1321, 1314, 1278, 1313, 1322, 1315, 1315, 1315, 1323, 1314, + 1315, 1324, 1316, 1316, 1316, 1325, 1313, 1316, 1315, 1314, + 1327, 1328, 1314, 1329, 1330, 1316, 1334, 1335, 1320, 1321, + 1336, 1315, 1337, 1322, 1340, 1341, 1342, 1323, 1316, 1343, + 1324, 1346, 1346, 1346, 1325, 1313, 1347, 1348, 1314, 1327, + 1328, 1314, 1329, 1330, 1353, 1334, 1335, 1354, 1355, 1336, + 1315, 1337, 1356, 1340, 1341, 1342, 1360, 1316, 1343, 1357, + + 1357, 1346, 1362, 1364, 1313, 1347, 1348, 1349, 1349, 1349, + 1314, 1363, 1365, 1353, 1315, 1366, 1354, 1355, 1367, 1315, + 1368, 1356, 1349, 1369, 1370, 1360, 1316, 1371, 1372, 1363, + 1346, 1362, 1364, 1373, 1374, 1375, 1376, 1378, 1380, 1380, + 1363, 1365, 1382, 1384, 1366, 1385, 1396, 1367, 1391, 1368, + 1387, 1399, 1369, 1370, 1373, 1346, 1371, 1372, 1363, 1403, + 1406, 1407, 1373, 1374, 1375, 1376, 1378, 1387, 1408, 1391, + 1409, 1382, 1384, 1410, 1385, 1396, 1411, 1412, 1436, 1436, + 1399, 1424, 1425, 1373, 1443, 1443, 1536, 1426, 1403, 1406, + 1407, 1349, 1427, 1456, 1456, 1437, 1387, 1408, 1391, 1409, + + 1437, 1503, 1410, 1428, 1457, 1411, 1412, 1413, 1413, 1413, + 1424, 1425, 1413, 1414, 1414, 1414, 1426, 1453, 1414, 1423, + 1413, 1427, 1415, 1415, 1415, 1429, 1414, 1415, 1405, 1416, + 1416, 1416, 1428, 1413, 1416, 1415, 1430, 1431, 1432, 1414, + 1434, 1395, 1416, 1393, 1417, 1417, 1417, 1444, 1415, 1417, + 1392, 1435, 1444, 1463, 1429, 1416, 1390, 1417, 1463, 1418, + 1418, 1418, 1413, 1416, 1418, 1430, 1431, 1432, 1414, 1434, + 1417, 1389, 1418, 1440, 1419, 1419, 1419, 1415, 1388, 1419, + 1435, 1420, 1420, 1420, 1416, 1418, 1420, 1419, 1441, 1462, + 1462, 1413, 1416, 1442, 1420, 1446, 1381, 1414, 1361, 1417, + + 1419, 1415, 1440, 1438, 1438, 1438, 1415, 1420, 1447, 1445, + 1445, 1445, 1448, 1416, 1418, 1359, 1449, 1441, 1421, 1421, + 1421, 1358, 1442, 1421, 1446, 1422, 1422, 1422, 1417, 1419, + 1422, 1421, 1351, 1438, 1350, 1452, 1420, 1447, 1422, 1445, + 1455, 1448, 1459, 1418, 1421, 1449, 1460, 1461, 1450, 1450, + 1450, 1422, 1451, 1451, 1451, 1464, 1464, 1464, 1419, 1465, + 1466, 1421, 1438, 1450, 1452, 1420, 1467, 1468, 1445, 1455, + 1469, 1459, 1470, 1421, 1471, 1460, 1461, 1472, 1472, 1472, + 1422, 1473, 1451, 1474, 1475, 1464, 1476, 1438, 1465, 1466, + 1421, 1477, 1478, 1445, 1479, 1467, 1468, 1480, 1481, 1469, + + 1482, 1470, 1421, 1471, 1422, 1483, 1484, 1339, 1485, 1422, + 1473, 1451, 1474, 1475, 1464, 1476, 1486, 1486, 1490, 1487, + 1477, 1478, 1491, 1479, 1487, 1492, 1480, 1481, 1495, 1482, + 1496, 1493, 1450, 1494, 1483, 1484, 1451, 1485, 1497, 1464, + 1488, 1488, 1488, 1493, 1498, 1499, 1494, 1490, 1500, 1501, + 1504, 1491, 1492, 1505, 1502, 1502, 1502, 1495, 1522, 1496, + 1332, 1472, 1553, 1553, 1524, 1525, 1554, 1497, 1502, 1527, + 1488, 1554, 1493, 1498, 1499, 1494, 1317, 1500, 1501, 1504, + 1529, 1492, 1505, 1530, 1507, 1507, 1507, 1522, 1535, 1507, + 1508, 1508, 1508, 1524, 1525, 1508, 1296, 1507, 1527, 1488, + + 1509, 1509, 1509, 1508, 1280, 1509, 1510, 1510, 1510, 1529, + 1507, 1510, 1530, 1509, 1539, 1542, 1508, 1535, 1279, 1510, + 1511, 1511, 1511, 1256, 1488, 1511, 1509, 1528, 1528, 1528, + 1255, 1243, 1510, 1511, 1570, 1570, 1570, 1237, 1502, 1507, + 1544, 1528, 1545, 1539, 1542, 1508, 1511, 1512, 1512, 1512, + 1236, 1227, 1512, 1226, 1217, 1509, 1203, 1513, 1513, 1513, + 1512, 1510, 1513, 1511, 1533, 1533, 1533, 1170, 1507, 1544, + 1513, 1545, 1546, 1512, 1508, 1511, 1540, 1540, 1540, 1533, + 1514, 1514, 1514, 1513, 1509, 1514, 1167, 1534, 1534, 1534, + 1510, 1540, 1511, 1514, 1515, 1515, 1515, 1148, 1139, 1515, + + 1547, 1546, 1512, 1550, 1511, 1137, 1514, 1515, 1516, 1516, + 1516, 1528, 1513, 1516, 1517, 1517, 1517, 1534, 1570, 1517, + 1515, 1516, 1556, 1548, 1548, 1548, 1512, 1517, 1122, 1547, + 1116, 1512, 1550, 1115, 1516, 1514, 1513, 1111, 1548, 1086, + 1517, 1513, 1518, 1518, 1518, 1074, 1534, 1518, 1533, 1515, + 1557, 1556, 1066, 1065, 1062, 1518, 1056, 1519, 1519, 1519, + 1540, 1051, 1519, 1516, 1514, 1541, 1541, 1541, 1518, 1517, + 1519, 1534, 1558, 1046, 1555, 1555, 1555, 1040, 1515, 1557, + 1520, 1520, 1520, 1519, 1561, 1520, 1036, 1516, 1034, 1521, + 1521, 1521, 1516, 1520, 1521, 1541, 1563, 1518, 1517, 1564, + + 1565, 1558, 1521, 1520, 1555, 1566, 1520, 1548, 1559, 1559, + 1559, 1567, 1519, 1561, 1568, 1521, 1560, 1560, 1560, 1569, + 1031, 1518, 1572, 1559, 1541, 1563, 1518, 1573, 1564, 1565, + 1574, 1575, 1520, 1555, 1566, 1520, 1576, 1577, 1578, 1029, + 1567, 1519, 1579, 1568, 1521, 1581, 1560, 1582, 1569, 1541, + 1583, 1572, 1584, 1012, 993, 991, 1573, 971, 1555, 1574, + 1575, 1587, 1589, 960, 1520, 1576, 1577, 1578, 1521, 1590, + 1594, 1579, 953, 1521, 1581, 1560, 1582, 1591, 951, 1583, + 1597, 1584, 1585, 1585, 1585, 1586, 1586, 1586, 927, 913, + 1587, 1589, 1559, 1598, 1600, 906, 904, 1585, 1590, 1594, + + 1560, 1580, 1653, 1653, 1653, 1591, 1604, 1580, 1613, 1597, + 1615, 1580, 1580, 1616, 1621, 1586, 1580, 1580, 1580, 1624, + 1580, 899, 1598, 1600, 1601, 1601, 1601, 893, 888, 1621, + 1580, 1602, 1602, 1602, 1591, 1604, 1580, 1613, 1601, 1615, + 1580, 1580, 1616, 1621, 1586, 1580, 1580, 1580, 1624, 1580, + 1605, 1605, 1605, 885, 1602, 1607, 1607, 1607, 1621, 881, + 1607, 879, 1608, 1608, 1608, 877, 1585, 1608, 1607, 1586, + 1605, 872, 858, 1602, 844, 1608, 819, 1609, 1609, 1609, + 1626, 1607, 1609, 1602, 1629, 1608, 1653, 1631, 1608, 793, + 1609, 787, 1610, 1610, 1610, 754, 748, 1610, 1632, 1611, + + 1611, 1611, 1602, 1609, 1611, 1610, 1634, 735, 1601, 1626, + 1607, 1636, 1611, 1629, 1608, 1602, 1631, 1608, 1610, 1617, + 1617, 1617, 1618, 1618, 1618, 1611, 1639, 1632, 1620, 1620, + 1620, 729, 1609, 1617, 1605, 1634, 1612, 1612, 1612, 1607, + 1636, 1612, 1622, 1622, 1622, 1618, 1608, 1610, 1620, 1612, + 1627, 1627, 1627, 1641, 1611, 1639, 1609, 1622, 1637, 1637, + 1637, 1609, 1612, 1642, 1643, 1627, 1638, 1638, 1638, 1646, + 1612, 1610, 1648, 1637, 1618, 1649, 1610, 726, 1611, 1644, + 1644, 1644, 1641, 1611, 1650, 1651, 1652, 1654, 1654, 1654, + 1657, 1612, 1642, 1643, 1644, 1660, 1638, 721, 1646, 1612, + + 1661, 1648, 1662, 1617, 1649, 717, 1618, 1659, 1659, 1659, + 1663, 1665, 1620, 1650, 1651, 1652, 1666, 1667, 1668, 1657, + 1612, 708, 1670, 1671, 1660, 1638, 1622, 1664, 1668, 1661, + 1664, 1662, 1672, 1669, 1627, 1674, 1678, 1680, 1681, 1663, + 1665, 1669, 1637, 1683, 1685, 1666, 1667, 1668, 1669, 1682, + 1638, 1670, 1671, 1676, 1676, 1676, 1664, 1668, 1682, 1664, + 1686, 1672, 1669, 1644, 1674, 1678, 1680, 1681, 1676, 1688, + 1669, 1654, 1683, 1685, 1687, 1687, 1687, 1669, 1689, 1692, + 1693, 1693, 1693, 1694, 1694, 1694, 706, 1682, 1700, 1686, + 704, 1659, 1702, 1705, 1705, 1705, 1708, 1687, 1688, 1694, + + 1693, 1697, 1697, 1697, 1709, 1711, 1697, 1689, 1692, 1712, + 1696, 1696, 1696, 1705, 1697, 1696, 1687, 1700, 1701, 1701, + 1701, 1702, 1714, 1696, 1715, 1708, 1687, 1697, 1716, 703, + 1706, 1706, 1706, 1709, 1711, 701, 1696, 1676, 1712, 1717, + 1722, 1701, 1724, 1725, 1696, 1687, 1706, 1719, 1719, 1719, + 1728, 1714, 1729, 1715, 1730, 1731, 1697, 1716, 1687, 1720, + 1720, 1720, 1733, 1738, 1693, 1696, 699, 1694, 1717, 1722, + 1701, 1724, 1725, 1696, 1720, 1740, 698, 1705, 696, 1728, + 1743, 1729, 1745, 1730, 1731, 1697, 1734, 1734, 1734, 1746, + 1747, 1733, 1738, 1748, 1696, 1736, 1736, 1736, 1739, 1739, + + 1739, 1749, 1701, 1750, 1740, 1741, 1741, 1741, 1751, 1743, + 1752, 1745, 1753, 1754, 1706, 1756, 1757, 1758, 1746, 1747, + 1759, 1762, 1748, 1763, 1764, 694, 1765, 1766, 693, 668, + 1749, 1719, 1750, 1769, 1772, 665, 618, 1751, 1778, 1752, + 1779, 1753, 1754, 1720, 1756, 1757, 1758, 1741, 598, 1759, + 1762, 555, 1763, 1764, 1765, 1781, 1766, 1768, 1768, 1768, + 553, 548, 1769, 1772, 1773, 1773, 1773, 1778, 1787, 1779, + 1734, 1788, 546, 1777, 1777, 1777, 1741, 1768, 1777, 1736, + 1773, 545, 1739, 1765, 1781, 1792, 1777, 1793, 1794, 1741, + 1780, 1780, 1780, 1783, 1783, 1783, 1798, 1787, 1799, 1777, + + 1788, 1789, 1789, 1789, 1791, 1791, 1791, 1800, 1801, 1783, + 1780, 1795, 1795, 1795, 1792, 1806, 1793, 1794, 1802, 1802, + 1802, 1804, 1804, 1804, 1810, 1798, 1813, 1799, 1777, 1807, + 1807, 1807, 1811, 1811, 1811, 1804, 1800, 1801, 1814, 1817, + 1816, 1768, 1818, 1819, 1806, 532, 1820, 1822, 1773, 1816, + 1821, 1821, 1823, 1810, 1825, 1813, 526, 1777, 1826, 519, + 1827, 1828, 1829, 1832, 1832, 1832, 516, 1814, 1817, 1816, + 1834, 1818, 1819, 512, 1780, 1820, 1822, 1783, 1835, 1821, + 1821, 1823, 1836, 1825, 509, 1789, 1837, 1826, 1791, 1827, + 1828, 1829, 1838, 1839, 1841, 1795, 1842, 1843, 1845, 1834, + + 1846, 1851, 1802, 1852, 508, 1804, 497, 1835, 1847, 1847, + 1847, 1836, 1856, 1807, 1857, 1837, 1811, 1849, 1849, 1849, + 1858, 1838, 1839, 1841, 1861, 1842, 1843, 1845, 1865, 1846, + 1851, 1866, 1852, 1855, 1855, 1855, 1835, 1859, 1859, 1859, + 1869, 1856, 1871, 1857, 1862, 1862, 1862, 1832, 1872, 1858, + 1874, 1875, 1876, 1861, 1863, 1863, 1863, 1865, 1862, 1877, + 1866, 1867, 1867, 1867, 1878, 1879, 1880, 490, 1882, 1869, + 1883, 1871, 489, 1891, 1884, 1885, 1887, 1872, 1890, 1874, + 1875, 1876, 1888, 1888, 1888, 1892, 1893, 1894, 1877, 1898, + 1899, 1863, 1847, 1878, 1879, 1880, 1863, 1882, 1900, 1883, + + 1891, 1849, 1867, 1884, 1885, 1887, 1903, 1890, 1904, 1905, + 1905, 1905, 1909, 1911, 1892, 1893, 1894, 1855, 1898, 1899, + 1863, 1859, 1913, 1914, 1917, 1863, 1918, 1900, 1862, 1891, + 1919, 1867, 1912, 1912, 1912, 1903, 1921, 1904, 1863, 1922, + 488, 1909, 1911, 1923, 1924, 1867, 1926, 1927, 1928, 1929, + 1930, 1913, 1914, 1917, 1931, 1918, 1932, 1934, 1937, 1919, + 1940, 1941, 1942, 1943, 484, 1921, 1888, 1944, 1922, 1912, + 1945, 1946, 1923, 1924, 1912, 1926, 1927, 1928, 1929, 1930, + 1948, 1951, 1952, 1931, 1956, 1932, 1934, 1937, 1957, 1940, + 1941, 1942, 1943, 1905, 1958, 1959, 1944, 1961, 1912, 1945, + + 1946, 1962, 1963, 1912, 1960, 1960, 1960, 1965, 1967, 1948, + 1951, 1952, 1969, 1956, 1970, 1972, 1912, 1957, 1973, 1974, + 1976, 1977, 1979, 1958, 1959, 1983, 1961, 1984, 1985, 1986, + 1962, 1963, 1987, 1988, 1990, 483, 1965, 1967, 1989, 1989, + 1989, 1969, 1994, 1970, 1972, 1995, 1997, 1973, 1974, 1976, + 1977, 1979, 1998, 2000, 1983, 2002, 1984, 1985, 1986, 481, + 2003, 1987, 1988, 1990, 1991, 1991, 1991, 2005, 2007, 2008, + 2009, 1994, 2010, 2015, 1995, 1997, 2011, 2011, 2011, 2017, + 2019, 1998, 2000, 2021, 2002, 2013, 2013, 2013, 1960, 2003, + 2022, 2023, 2025, 477, 2026, 2027, 2005, 2007, 2008, 2009, + + 2028, 2010, 2015, 2029, 2030, 2032, 2033, 2034, 2017, 2019, + 2035, 2036, 2021, 2037, 2038, 2079, 2079, 2079, 2040, 2022, + 2023, 2025, 1989, 2026, 2027, 2041, 2042, 2043, 2044, 2028, + 2045, 476, 2029, 2030, 2032, 2033, 2034, 2048, 2055, 2035, + 2036, 474, 2037, 2038, 2039, 2039, 2039, 2040, 1991, 2058, + 2059, 2061, 2039, 470, 2041, 2042, 2043, 2044, 2062, 2045, + 2011, 2047, 2047, 2047, 2067, 2068, 2048, 2055, 2069, 2013, + 2049, 2049, 2049, 2070, 2051, 2051, 2051, 2071, 2058, 2059, + 2061, 2047, 2051, 2073, 2075, 2076, 2077, 2062, 2080, 2081, + 2049, 2082, 2083, 2067, 2068, 2086, 2088, 2069, 2089, 2079, + + 467, 2092, 2070, 2084, 2084, 2084, 2071, 2087, 2087, 2087, + 2094, 2095, 2073, 2075, 2076, 2077, 466, 2080, 2081, 2096, + 2082, 2083, 2097, 2098, 2086, 2088, 2099, 2089, 2039, 2039, + 2092, 2093, 2093, 2093, 2100, 2100, 2100, 463, 462, 2094, + 2095, 460, 448, 447, 444, 2047, 443, 440, 2096, 439, + 426, 2097, 2098, 420, 2049, 2099, 418, 416, 2051, 2051, + 402, 376, 374, 309, 306, 304, 302, 298, 293, 292, + 287, 266, 265, 261, 251, 250, 248, 237, 236, 229, + 227, 218, 216, 205, 204, 198, 197, 2084, 189, 188, + 185, 2087, 184, 169, 168, 162, 161, 154, 153, 136, + + 132, 130, 128, 109, 95, 94, 93, 67, 61, 58, + 53, 36, 35, 34, 33, 2093, 0, 0, 2100, 2102, + 2102, 2102, 2102, 2102, 2102, 2102, 2102, 2102, 2102, 2103, + 2103, 2103, 2103, 2103, 2103, 2103, 2103, 2103, 2103, 2104, + 2104, 2104, 2104, 2104, 2104, 2104, 2104, 2104, 2104, 2105, + 2105, 2105, 2105, 2105, 2105, 2105, 2105, 2105, 2105, 2106, + 2106, 2106, 2106, 2106, 2106, 2106, 2106, 2106, 2106, 2107, + 2107, 2107, 2107, 2107, 2107, 2107, 2107, 2107, 2107, 2108, + 2108, 2108, 2108, 2108, 2108, 2108, 2108, 2108, 2108, 2109, + 2109, 2109, 2109, 2109, 2109, 2109, 2109, 2109, 2109, 2110, + + 2110, 2110, 2110, 2110, 2110, 2110, 2110, 2110, 2110, 2111, + 2111, 2111, 2111, 2111, 2111, 2111, 2111, 2111, 2111, 2112, + 2112, 2112, 2112, 2112, 2112, 2112, 2112, 2112, 2112, 2113, + 2113, 2113, 2113, 2113, 2113, 2113, 2113, 2113, 2113, 2114, + 2114, 2114, 2114, 2114, 2114, 2114, 2114, 2114, 2114, 2115, + 2115, 2115, 2115, 2115, 2115, 2115, 2115, 2115, 2115, 2116, + 2116, 2116, 2116, 2116, 2116, 2116, 2116, 2116, 2116, 2117, + 2117, 2117, 2117, 2117, 2117, 2117, 2117, 2117, 2117, 2118, + 2118, 2119, 2119, 0, 2119, 2119, 2119, 2119, 2119, 2119, + 2119, 2121, 2121, 2122, 0, 0, 0, 0, 2122, 2124, + + 2124, 0, 2124, 0, 2124, 2124, 2125, 2125, 2127, 2127, + 2128, 0, 0, 0, 0, 2128, 2129, 0, 2129, 2131, + 2131, 2131, 2131, 2131, 0, 2131, 2131, 0, 2131, 2132, + 2132, 2132, 2132, 2132, 2132, 2132, 2132, 2132, 2132, 2133, + 2133, 2133, 2133, 2133, 0, 2133, 2133, 0, 2133, 2134, + 2134, 2134, 2134, 2134, 2134, 2134, 2134, 2134, 2134, 2135, + 2135, 2135, 2135, 2135, 2135, 2135, 2135, 2135, 2135, 2136, + 2136, 2136, 2136, 2136, 2136, 2136, 2136, 2136, 2137, 2137, + 2137, 2137, 2137, 2137, 2137, 2137, 2137, 2138, 2138, 0, + 2138, 2138, 2138, 2138, 2138, 2138, 2138, 2140, 2140, 0, + + 2140, 2140, 2140, 2140, 2140, 2140, 2140, 2141, 2141, 0, + 2141, 2141, 2141, 2141, 2141, 2141, 2141, 2144, 2144, 0, + 2144, 2144, 2144, 2144, 2144, 2144, 2144, 2146, 2146, 0, + 2146, 2146, 2146, 2146, 2146, 2146, 2146, 2147, 2147, 2148, + 2148, 0, 2148, 2148, 2148, 2148, 2148, 2148, 2148, 2150, + 2150, 2151, 0, 0, 0, 0, 2151, 2153, 2153, 0, + 2153, 0, 2153, 2153, 2154, 2154, 2156, 2156, 2157, 0, + 0, 0, 0, 2157, 2158, 0, 2158, 2160, 2160, 2160, + 2160, 2160, 0, 2160, 2160, 0, 2160, 2161, 2161, 0, + 2161, 2161, 2161, 2161, 2161, 2161, 2161, 2162, 2162, 2162, + + 2162, 2162, 0, 2162, 2162, 0, 2162, 2163, 2163, 0, + 2163, 2163, 2163, 2163, 2163, 2163, 2163, 2164, 2164, 0, + 2164, 2164, 2164, 2164, 2164, 2164, 2164, 2165, 2165, 2165, + 2165, 2165, 2165, 2165, 2165, 2165, 2166, 2166, 2166, 2166, + 2166, 2166, 2166, 2166, 2166, 2167, 2167, 0, 2167, 2167, + 2167, 2167, 2167, 2167, 2167, 2169, 2169, 0, 2169, 2169, + 2169, 2169, 2169, 2169, 2169, 2170, 2170, 0, 2170, 2170, + 2170, 2170, 2170, 2170, 2170, 2173, 2173, 0, 2173, 2173, + 2173, 2173, 2173, 2173, 2173, 2175, 2175, 0, 2175, 2175, + 2175, 2175, 2175, 2175, 2175, 2176, 2176, 2176, 2176, 2176, + + 2176, 2176, 2176, 2176, 2176, 2179, 0, 0, 2179, 2181, + 0, 2181, 2183, 2183, 2183, 2183, 2183, 2183, 2183, 2183, + 2183, 2183, 2184, 2184, 2184, 2184, 2184, 2184, 2184, 2184, + 2184, 2184, 2185, 2185, 2185, 2185, 2185, 2185, 2185, 2185, + 2185, 2185, 2186, 2186, 2186, 2186, 2186, 2186, 2186, 2186, + 2186, 2186, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, + 2187, 2187, 2188, 2188, 2188, 2188, 2188, 2188, 2188, 2188, + 2188, 2188, 2189, 2189, 2189, 2189, 2189, 2189, 2189, 2189, + 2189, 2189, 2190, 2190, 2190, 2190, 2190, 2190, 2190, 2190, + 2190, 2190, 2191, 2191, 2191, 2191, 2191, 2191, 2191, 2191, + + 2191, 2191, 2192, 2192, 2192, 2192, 2192, 2192, 2192, 2192, + 2192, 2192, 2193, 2193, 2193, 2193, 2193, 2193, 2193, 2193, + 2193, 2193, 2194, 2194, 2194, 2194, 2194, 2194, 2194, 2194, + 2194, 2194, 2195, 2195, 2195, 2195, 2195, 2195, 2195, 2195, + 2195, 2195, 2196, 2196, 2196, 2196, 2196, 2196, 2196, 2196, + 2196, 2196, 2197, 2197, 2197, 2197, 2197, 2197, 2197, 2197, + 2197, 2197, 2198, 2198, 2198, 2198, 2198, 2198, 2198, 2198, + 2198, 2198, 2199, 2199, 2199, 2199, 2199, 2199, 2199, 2199, + 2199, 2199, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, + 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101 } ; /* The intent behind this definition is that it'll catch @@ -2769,7 +2791,7 @@ do {\ /*following character status will be rewrite by gen_parse.sh according to connection character*/ -#line 2774 "ob_proxy_parser_utf8_lex.c" +#line 2796 "ob_proxy_parser_utf8_lex.c" #define INITIAL 0 #define hint 1 @@ -3038,11 +3060,11 @@ YY_DECL register int yy_act; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; -#line 365 "ob_proxy_parser_utf8.l" +#line 366 "ob_proxy_parser_utf8.l" /* basic dml stmt: */ -#line 3048 "ob_proxy_parser_utf8_lex.c" +#line 3070 "ob_proxy_parser_utf8_lex.c" yylval = yylval_param; @@ -3099,13 +3121,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 2073 ) + if ( yy_current_state >= 2102 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 6130 ); + while ( yy_base[yy_current_state] != 6183 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -3131,83 +3153,83 @@ do_action: /* This label is used only to access EOF actions. */ case 1: YY_RULE_SETUP -#line 368 "ob_proxy_parser_utf8.l" +#line 369 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_SELECT); PUSH_STATE(in_expr); return SELECT; } YY_BREAK case 2: YY_RULE_SETUP -#line 369 "ob_proxy_parser_utf8.l" +#line 370 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_DELETE); PUSH_STATE_IF_NOT_ICMD(in_expr); return DELETE; } YY_BREAK case 3: YY_RULE_SETUP -#line 370 "ob_proxy_parser_utf8.l" +#line 371 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_INSERT); return INSERT; } YY_BREAK case 4: YY_RULE_SETUP -#line 371 "ob_proxy_parser_utf8.l" +#line 372 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_UPDATE); return UPDATE; } YY_BREAK case 5: YY_RULE_SETUP -#line 372 "ob_proxy_parser_utf8.l" +#line 373 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_REPLACE); return REPLACE; } YY_BREAK case 6: YY_RULE_SETUP -#line 373 "ob_proxy_parser_utf8.l" +#line 374 "ob_proxy_parser_utf8.l" { SET_DML_STMT(OBPROXY_T_MERGE); return MERGE; } YY_BREAK case 7: YY_RULE_SETUP -#line 374 "ob_proxy_parser_utf8.l" +#line 375 "ob_proxy_parser_utf8.l" { SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW; } YY_BREAK case 8: YY_RULE_SETUP -#line 375 "ob_proxy_parser_utf8.l" +#line 376 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(XA); } YY_BREAK /* if begin is for starting transaction, will set has_anonymous_block = false in yacc */ case 9: YY_RULE_SETUP -#line 377 "ob_proxy_parser_utf8.l" +#line 378 "ob_proxy_parser_utf8.l" { SET_HAS_ANONYMOUS_BLOCK(); RETURN_NON_RESERVED_KEYWORD(BEGI); } YY_BREAK case 10: YY_RULE_SETUP -#line 378 "ob_proxy_parser_utf8.l" +#line 379 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(START); } YY_BREAK case 11: YY_RULE_SETUP -#line 379 "ob_proxy_parser_utf8.l" +#line 380 "ob_proxy_parser_utf8.l" { SET_BASIC_STMT(OBPROXY_T_COMMIT); RETURN_IGNORED_WORD(); } YY_BREAK case 12: YY_RULE_SETUP -#line 380 "ob_proxy_parser_utf8.l" +#line 381 "ob_proxy_parser_utf8.l" { SET_BASIC_STMT(OBPROXY_T_ROLLBACK); RETURN_IGNORED_WORD(); } YY_BREAK case 13: YY_RULE_SETUP -#line 381 "ob_proxy_parser_utf8.l" +#line 382 "ob_proxy_parser_utf8.l" { SET_BASIC_STMT(OBPROXY_T_SET); PUSH_STATE_IF_NOT_ICMD(set_expr); return SET; } YY_BREAK case 14: YY_RULE_SETUP -#line 382 "ob_proxy_parser_utf8.l" +#line 383 "ob_proxy_parser_utf8.l" { SET_BASIC_STMT(OBPROXY_T_CALL); return CALL; } YY_BREAK case 15: YY_RULE_SETUP -#line 383 "ob_proxy_parser_utf8.l" +#line 384 "ob_proxy_parser_utf8.l" { SET_HAS_ANONYMOUS_BLOCK(); PUSH_STATE(in_anonymous_block); } YY_BREAK case 16: YY_RULE_SETUP -#line 384 "ob_proxy_parser_utf8.l" +#line 385 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = ob_proxy_parser_utf8_yyget_extra(yyscanner); if (OB_NOTNULL(p)) { if (OBPROXY_T_INSERT == p->cur_stmt_type_) @@ -3218,703 +3240,724 @@ YY_RULE_SETUP /* basic ddl stmt */ case 17: YY_RULE_SETUP -#line 392 "ob_proxy_parser_utf8.l" +#line 393 "ob_proxy_parser_utf8.l" { return CREATE; } YY_BREAK case 18: YY_RULE_SETUP -#line 393 "ob_proxy_parser_utf8.l" +#line 394 "ob_proxy_parser_utf8.l" { return DROP; } YY_BREAK case 19: YY_RULE_SETUP -#line 394 "ob_proxy_parser_utf8.l" +#line 395 "ob_proxy_parser_utf8.l" { return ALTER; } YY_BREAK case 20: YY_RULE_SETUP -#line 395 "ob_proxy_parser_utf8.l" +#line 396 "ob_proxy_parser_utf8.l" { return TRUNCATE; } YY_BREAK case 21: YY_RULE_SETUP -#line 396 "ob_proxy_parser_utf8.l" +#line 397 "ob_proxy_parser_utf8.l" { return RENAME; } YY_BREAK case 22: YY_RULE_SETUP -#line 397 "ob_proxy_parser_utf8.l" +#line 398 "ob_proxy_parser_utf8.l" { return INDEX; } YY_BREAK -/* ps stmt */ case 23: YY_RULE_SETUP -#line 400 "ob_proxy_parser_utf8.l" -{ return USING; } +#line 399 "ob_proxy_parser_utf8.l" +{ return TABLE; } YY_BREAK case 24: YY_RULE_SETUP -#line 401 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(prepare); return PREPARE; } +#line 400 "ob_proxy_parser_utf8.l" +{ return UNIQUE; } YY_BREAK +/* ps stmt */ case 25: YY_RULE_SETUP -#line 402 "ob_proxy_parser_utf8.l" -{ return EXECUTE; } +#line 403 "ob_proxy_parser_utf8.l" +{ return USING; } YY_BREAK -/* oracle ddl stmt */ case 26: YY_RULE_SETUP -#line 405 "ob_proxy_parser_utf8.l" -{ return GRANT; } +#line 404 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(prepare); return PREPARE; } YY_BREAK case 27: YY_RULE_SETUP -#line 406 "ob_proxy_parser_utf8.l" -{ return REVOKE; } +#line 405 "ob_proxy_parser_utf8.l" +{ return EXECUTE; } YY_BREAK +/* oracle ddl stmt */ case 28: YY_RULE_SETUP -#line 407 "ob_proxy_parser_utf8.l" -{ return ANALYZE; } +#line 408 "ob_proxy_parser_utf8.l" +{ return GRANT; } YY_BREAK case 29: YY_RULE_SETUP -#line 408 "ob_proxy_parser_utf8.l" -{ return PURGE; } +#line 409 "ob_proxy_parser_utf8.l" +{ return REVOKE; } YY_BREAK case 30: YY_RULE_SETUP -#line 409 "ob_proxy_parser_utf8.l" -{ return COMMENT; } +#line 410 "ob_proxy_parser_utf8.l" +{ return ANALYZE; } YY_BREAK case 31: YY_RULE_SETUP -#line 410 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(FLASHBACK); } +#line 411 "ob_proxy_parser_utf8.l" +{ return PURGE; } YY_BREAK case 32: YY_RULE_SETUP -#line 411 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(AUDIT); } +#line 412 "ob_proxy_parser_utf8.l" +{ return COMMENT; } YY_BREAK case 33: YY_RULE_SETUP -#line 412 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(NOAUDIT); } +#line 413 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(FLASHBACK); } YY_BREAK case 34: YY_RULE_SETUP #line 414 "ob_proxy_parser_utf8.l" -{ return GROUP;} +{ RETURN_NON_RESERVED_KEYWORD(AUDIT); } YY_BREAK case 35: YY_RULE_SETUP #line 415 "ob_proxy_parser_utf8.l" -{ return HAVING;} +{ RETURN_NON_RESERVED_KEYWORD(NOAUDIT); } YY_BREAK case 36: YY_RULE_SETUP -#line 416 "ob_proxy_parser_utf8.l" -{ return ORDER;} +#line 417 "ob_proxy_parser_utf8.l" +{ return GROUP;} YY_BREAK case 37: YY_RULE_SETUP -#line 417 "ob_proxy_parser_utf8.l" -{ return FOR;} +#line 418 "ob_proxy_parser_utf8.l" +{ return HAVING;} YY_BREAK case 38: YY_RULE_SETUP -#line 418 "ob_proxy_parser_utf8.l" -{ return UNION;} +#line 419 "ob_proxy_parser_utf8.l" +{ return ORDER;} YY_BREAK case 39: YY_RULE_SETUP -#line 419 "ob_proxy_parser_utf8.l" -{ return AS; } +#line 420 "ob_proxy_parser_utf8.l" +{ return FOR;} YY_BREAK case 40: YY_RULE_SETUP -#line 420 "ob_proxy_parser_utf8.l" -{ return WHERE; } +#line 421 "ob_proxy_parser_utf8.l" +{ return UNION;} YY_BREAK case 41: YY_RULE_SETUP -#line 421 "ob_proxy_parser_utf8.l" -{ return VALUES; } +#line 422 "ob_proxy_parser_utf8.l" +{ return AS; } YY_BREAK case 42: YY_RULE_SETUP -#line 422 "ob_proxy_parser_utf8.l" -{ SET_HAS_EXPLAIN(); return EXPLAIN; } +#line 423 "ob_proxy_parser_utf8.l" +{ return WHERE; } YY_BREAK case 43: YY_RULE_SETUP -#line 423 "ob_proxy_parser_utf8.l" -{ SET_HAS_EXPLAIN(); return DESC; } +#line 424 "ob_proxy_parser_utf8.l" +{ return VALUES; } YY_BREAK case 44: YY_RULE_SETUP -#line 424 "ob_proxy_parser_utf8.l" -{ SET_HAS_EXPLAIN(); return DESCRIBE; } +#line 425 "ob_proxy_parser_utf8.l" +{ return VALUES; } YY_BREAK -/*change from non_reserved to reserved according https://dev.mysql.com/doc/refman/5.6/en/keywords.html*/ case 45: YY_RULE_SETUP #line 426 "ob_proxy_parser_utf8.l" -{ return READ; } +{ SET_HAS_EXPLAIN(); return EXPLAIN; } YY_BREAK case 46: YY_RULE_SETUP #line 427 "ob_proxy_parser_utf8.l" -{ return WITH; } +{ SET_HAS_EXPLAIN(); return DESC; } YY_BREAK case 47: YY_RULE_SETUP #line 428 "ob_proxy_parser_utf8.l" -{ return USE; } +{ SET_HAS_EXPLAIN(); return DESCRIBE; } YY_BREAK +/*change from non_reserved to reserved according https://dev.mysql.com/doc/refman/5.6/en/keywords.html*/ case 48: YY_RULE_SETUP -#line 429 "ob_proxy_parser_utf8.l" -{ return LIMIT; } +#line 430 "ob_proxy_parser_utf8.l" +{ return READ; } YY_BREAK case 49: YY_RULE_SETUP -#line 430 "ob_proxy_parser_utf8.l" -{ return ALL; } +#line 431 "ob_proxy_parser_utf8.l" +{ return WITH; } YY_BREAK case 50: YY_RULE_SETUP -#line 431 "ob_proxy_parser_utf8.l" -{ return LIKE; } +#line 432 "ob_proxy_parser_utf8.l" +{ return USE; } YY_BREAK case 51: YY_RULE_SETUP -#line 432 "ob_proxy_parser_utf8.l" -{ return PARTITION; } +#line 433 "ob_proxy_parser_utf8.l" +{ return LIMIT; } YY_BREAK case 52: YY_RULE_SETUP -#line 433 "ob_proxy_parser_utf8.l" -{ return BINARY; } +#line 434 "ob_proxy_parser_utf8.l" +{ return ALL; } YY_BREAK case 53: YY_RULE_SETUP -#line 434 "ob_proxy_parser_utf8.l" -{ return GROUP_NAME; } +#line 435 "ob_proxy_parser_utf8.l" +{ return LIKE; } YY_BREAK -/* to make bison easy, do not return these reserved keyword in non strict mode */ case 54: YY_RULE_SETUP -#line 437 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } +#line 436 "ob_proxy_parser_utf8.l" +{ return PARTITION; } YY_BREAK case 55: YY_RULE_SETUP -#line 438 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } +#line 437 "ob_proxy_parser_utf8.l" +{ return BINARY; } YY_BREAK case 56: YY_RULE_SETUP -#line 439 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } +#line 438 "ob_proxy_parser_utf8.l" +{ return GROUP_NAME; } YY_BREAK +/* to make bison easy, do not return these reserved keyword in non strict mode */ case 57: YY_RULE_SETUP -#line 440 "ob_proxy_parser_utf8.l" +#line 441 "ob_proxy_parser_utf8.l" { RETURN_IGNORED_WORD(); } YY_BREAK case 58: YY_RULE_SETUP -#line 441 "ob_proxy_parser_utf8.l" +#line 442 "ob_proxy_parser_utf8.l" { RETURN_IGNORED_WORD(); } YY_BREAK -/* no reserved keyword , don't forget to add these keyword in .y */ -/*refer: https://dev.mysql.com/doc/refman/5.6/en/keywords.html*/ case 59: YY_RULE_SETUP -#line 445 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(QUICK); } +#line 443 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD(); } YY_BREAK case 60: YY_RULE_SETUP -#line 446 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(COUNT); } +#line 444 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD(); } YY_BREAK case 61: YY_RULE_SETUP -#line 447 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(WARNINGS); } +#line 445 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD(); } YY_BREAK +/* no reserved keyword , don't forget to add these keyword in .y */ +/*refer: https://dev.mysql.com/doc/refman/5.6/en/keywords.html*/ case 62: YY_RULE_SETUP -#line 448 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(ERRORS); } +#line 449 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(QUICK); } YY_BREAK case 63: YY_RULE_SETUP -#line 449 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(TRACE); } +#line 450 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(COUNT); } YY_BREAK case 64: YY_RULE_SETUP -#line 450 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(TRANSACTION); } +#line 451 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(WARNINGS); } YY_BREAK case 65: YY_RULE_SETUP #line 452 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(ONLY); } +{ RETURN_NON_RESERVED_KEYWORD(ERRORS); } YY_BREAK case 66: YY_RULE_SETUP -#line 454 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(CONSISTENT); } +#line 453 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(TRACE); } YY_BREAK case 67: YY_RULE_SETUP -#line 455 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(SNAPSHOT); } +#line 454 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(TRANSACTION); } YY_BREAK case 68: YY_RULE_SETUP -#line 457 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(HELP); } +#line 456 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(ONLY); } YY_BREAK -/*set names*/ case 69: -/* rule 69 can match eol */ YY_RULE_SETUP -#line 461 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_NAMES); return SET_NAMES; } +#line 458 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(CONSISTENT); } YY_BREAK -/*set charset*/ case 70: -/* rule 70 can match eol */ YY_RULE_SETUP -#line 463 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_CHARSET); return SET_CHARSET; } +#line 459 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(SNAPSHOT); } YY_BREAK -/*set passwd*/ case 71: -/* rule 71 can match eol */ YY_RULE_SETUP -#line 465 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_PASSWORD); return SET_PASSWORD; } +#line 461 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(HELP); } YY_BREAK -/*set default*/ +/*set names*/ case 72: /* rule 72 can match eol */ YY_RULE_SETUP -#line 467 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_DEFAULT); return SET_DEFAULT; } +#line 465 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_NAMES); return SET_NAMES; } YY_BREAK +/*set charset*/ case 73: /* rule 73 can match eol */ YY_RULE_SETUP -#line 469 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_OB_READ_CONSISTENCY); return SET_OB_READ_CONSISTENCY; } +#line 467 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_CHARSET); return SET_CHARSET; } YY_BREAK +/*set passwd*/ case 74: /* rule 74 can match eol */ YY_RULE_SETUP -#line 470 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SET_TX_READ_ONLY); return SET_TX_READ_ONLY; } +#line 469 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_PASSWORD); return SET_PASSWORD; } YY_BREAK -/*internal cmd*/ -/*show net*/ +/*set default*/ case 75: /* rule 75 can match eol */ YY_RULE_SETUP -#line 474 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_NET); return SHOW_PROXYNET; } +#line 471 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_DEFAULT); return SET_DEFAULT; } YY_BREAK case 76: +/* rule 76 can match eol */ YY_RULE_SETUP -#line 475 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(THREAD); } +#line 473 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_OB_READ_CONSISTENCY); return SET_OB_READ_CONSISTENCY; } YY_BREAK case 77: +/* rule 77 can match eol */ YY_RULE_SETUP -#line 476 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(CONNECTION); } +#line 474 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SET_TX_READ_ONLY); return SET_TX_READ_ONLY; } YY_BREAK +/*internal cmd*/ +/*show net*/ case 78: +/* rule 78 can match eol */ YY_RULE_SETUP #line 478 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(OFFSET); } +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_NET); return SHOW_PROXYNET; } YY_BREAK -/*show session*/ case 79: -/* rule 79 can match eol */ YY_RULE_SETUP -#line 481 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_GLOBAL_SESSION); return SHOW_GLOBALSESSION; } +#line 479 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(THREAD); } YY_BREAK case 80: -/* rule 80 can match eol */ YY_RULE_SETUP -#line 482 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SESSION); return SHOW_PROXYSESSION; } +#line 480 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(CONNECTION); } YY_BREAK case 81: -/* rule 81 can match eol */ YY_RULE_SETUP -#line 483 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_PROCESSLIST); return SHOW_PROCESSLIST; } +#line 482 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(OFFSET); } YY_BREAK +/*show session*/ case 82: +/* rule 82 can match eol */ YY_RULE_SETUP -#line 484 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(ATTRIBUTE); } +#line 485 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_GLOBAL_SESSION); return SHOW_GLOBALSESSION; } YY_BREAK case 83: +/* rule 83 can match eol */ YY_RULE_SETUP -#line 485 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(VARIABLES); } +#line 486 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SESSION); return SHOW_PROXYSESSION; } YY_BREAK case 84: +/* rule 84 can match eol */ YY_RULE_SETUP #line 487 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(STAT); } +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_PROCESSLIST); return SHOW_PROCESSLIST; } YY_BREAK -/*show config*/ case 85: -/* rule 85 can match eol */ YY_RULE_SETUP -#line 490 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CONFIG); return SHOW_PROXYCONFIG; } +#line 488 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(ATTRIBUTE); } YY_BREAK case 86: YY_RULE_SETUP -#line 492 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(DIFF); } +#line 489 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(VARIABLES); } YY_BREAK case 87: YY_RULE_SETUP -#line 493 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(USER); } +#line 491 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(STAT); } YY_BREAK -/*show sm*/ +/*show config*/ case 88: /* rule 88 can match eol */ YY_RULE_SETUP -#line 496 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SM); return SHOW_PROXYSM; } +#line 494 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CONFIG); return SHOW_PROXYCONFIG; } YY_BREAK -/*show cluster*/ case 89: -/* rule 89 can match eol */ YY_RULE_SETUP -#line 499 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CLUSTER); return SHOW_PROXYCLUSTER; } +#line 496 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(DIFF); } YY_BREAK -/*show resource*/ case 90: -/* rule 90 can match eol */ YY_RULE_SETUP -#line 502 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_RESOURCE); return SHOW_PROXYRESOURCE; } +#line 497 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(USER); } YY_BREAK -/*show congestion*/ +/*show sm*/ case 91: /* rule 91 can match eol */ YY_RULE_SETUP -#line 505 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CONGESTION); return SHOW_PROXYCONGESTION; } +#line 500 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SM); return SHOW_PROXYSM; } YY_BREAK -/*show route*/ +/*show cluster*/ case 92: /* rule 92 can match eol */ YY_RULE_SETUP -#line 508 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_ROUTE); return SHOW_PROXYROUTE; } +#line 503 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CLUSTER); return SHOW_PROXYCLUSTER; } YY_BREAK +/*show resource*/ case 93: +/* rule 93 can match eol */ YY_RULE_SETUP -#line 510 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(ROUTINE); } +#line 506 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_RESOURCE); return SHOW_PROXYRESOURCE; } YY_BREAK -/*show vip*/ +/*show congestion*/ case 94: /* rule 94 can match eol */ YY_RULE_SETUP -#line 513 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_VIP); return SHOW_PROXYVIP; } +#line 509 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_CONGESTION); return SHOW_PROXYCONGESTION; } YY_BREAK -/*show memory*/ +/*show route*/ case 95: /* rule 95 can match eol */ YY_RULE_SETUP -#line 516 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_MEMORY); return SHOW_PROXYMEMORY; } +#line 512 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_ROUTE); return SHOW_PROXYROUTE; } YY_BREAK case 96: YY_RULE_SETUP -#line 517 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(OBJPOOL); } +#line 514 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(ROUTINE); } YY_BREAK -/*show sqlaudit*/ +/*show vip*/ case 97: /* rule 97 can match eol */ YY_RULE_SETUP -#line 520 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SQLAUDIT); return SHOW_SQLAUDIT; } +#line 517 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_VIP); return SHOW_PROXYVIP; } YY_BREAK -/*show warnlog*/ +/*show memory*/ case 98: /* rule 98 can match eol */ YY_RULE_SETUP -#line 523 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_WARNLOG); return SHOW_WARNLOG; } +#line 520 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_MEMORY); return SHOW_PROXYMEMORY; } YY_BREAK -/*show stat*/ case 99: -/* rule 99 can match eol */ YY_RULE_SETUP -#line 526 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_STAT); return SHOW_PROXYSTAT; } +#line 521 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(OBJPOOL); } YY_BREAK +/*show sqlaudit*/ case 100: +/* rule 100 can match eol */ YY_RULE_SETUP -#line 527 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(REFRESH); } +#line 524 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_SQLAUDIT); return SHOW_SQLAUDIT; } YY_BREAK -/*show trace*/ +/*show warnlog*/ case 101: /* rule 101 can match eol */ YY_RULE_SETUP -#line 530 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_TRACE); return SHOW_PROXYTRACE; } +#line 527 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_WARNLOG); return SHOW_WARNLOG; } YY_BREAK -/*show info*/ +/*show stat*/ case 102: /* rule 102 can match eol */ YY_RULE_SETUP -#line 533 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_INFO); return SHOW_PROXYINFO; } +#line 530 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_STAT); return SHOW_PROXYSTAT; } YY_BREAK case 103: YY_RULE_SETUP -#line 534 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(UPGRADE); } +#line 531 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(REFRESH); } YY_BREAK +/*show trace*/ case 104: +/* rule 104 can match eol */ YY_RULE_SETUP -#line 535 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(IDC); } +#line 534 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_TRACE); return SHOW_PROXYTRACE; } YY_BREAK +/*show info*/ case 105: /* rule 105 can match eol */ YY_RULE_SETUP #line 537 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SHOW); PUSH_STATE(show_topology); return SHOW_TOPOLOGY; } +{ SET_ICMD_STMT(OBPROXY_T_ICMD_SHOW_INFO); return SHOW_PROXYINFO; } YY_BREAK case 106: YY_RULE_SETUP #line 538 "ob_proxy_parser_utf8.l" -{ POP_STATE(); PUSH_STATE(INITIAL); return FROM; } +{ RETURN_NON_RESERVED_KEYWORD(UPGRADE); } YY_BREAK case 107: YY_RULE_SETUP #line 539 "ob_proxy_parser_utf8.l" -{ POP_STATE(); PUSH_STATE(INITIAL); return WHERE; } - YY_BREAK -case YY_STATE_EOF(show_topology): -#line 540 "ob_proxy_parser_utf8.l" -{ return END_P; } +{ RETURN_NON_RESERVED_KEYWORD(IDC); } YY_BREAK case 108: /* rule 108 can match eol */ YY_RULE_SETUP #line 541 "ob_proxy_parser_utf8.l" -{ } +{ SET_BASIC_STMT(OBPROXY_T_SHOW); PUSH_STATE(show_topology); return SHOW_TOPOLOGY; } YY_BREAK case 109: YY_RULE_SETUP #line 542 "ob_proxy_parser_utf8.l" -{ POP_STATE(); PUSH_STATE(INITIAL); return yytext[0]; } +{ POP_STATE(); PUSH_STATE(INITIAL); return FROM; } YY_BREAK case 110: YY_RULE_SETUP #line 543 "ob_proxy_parser_utf8.l" -{ return ERROR; } +{ POP_STATE(); PUSH_STATE(INITIAL); return WHERE; } + YY_BREAK +case YY_STATE_EOF(show_topology): +#line 544 "ob_proxy_parser_utf8.l" +{ return END_P; } YY_BREAK case 111: /* rule 111 can match eol */ YY_RULE_SETUP #line 545 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_DB_VERSION; } +{ } YY_BREAK case 112: -/* rule 112 can match eol */ YY_RULE_SETUP #line 546 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_DATABASES; } +{ POP_STATE(); PUSH_STATE(INITIAL); return yytext[0]; } YY_BREAK case 113: -/* rule 113 can match eol */ YY_RULE_SETUP #line 547 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_TABLES; } +{ return ERROR; } YY_BREAK case 114: /* rule 114 can match eol */ YY_RULE_SETUP -#line 548 "ob_proxy_parser_utf8.l" -{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_CREATE_TABLE; } +#line 549 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_DB_VERSION; } YY_BREAK case 115: /* rule 115 can match eol */ YY_RULE_SETUP -#line 549 "ob_proxy_parser_utf8.l" -{ SET_DML_STMT(OBPROXY_T_SELECT); return SELECT_DATABASE; } +#line 550 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_DATABASES; } YY_BREAK -/*alter config*/ case 116: /* rule 116 can match eol */ YY_RULE_SETUP -#line 552 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_ALTER_CONFIG); return ALTER_PROXYCONFIG; } +#line 551 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_TABLES; } YY_BREAK -/*alter resource*/ case 117: /* rule 117 can match eol */ YY_RULE_SETUP -#line 555 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_ALTER_RESOURCE); return ALTER_PROXYRESOURCE; } +#line 552 "ob_proxy_parser_utf8.l" +{ SET_BASIC_STMT(OBPROXY_T_SHOW); return SHOW_CREATE_TABLE; } YY_BREAK -/*ping proxy*/ case 118: /* rule 118 can match eol */ YY_RULE_SETUP -#line 558 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_PING_PROXY); return PING_PROXY; } +#line 553 "ob_proxy_parser_utf8.l" +{ SET_DML_STMT(OBPROXY_T_SELECT); return SELECT_DATABASE; } YY_BREAK -/*kill*/ case 119: /* rule 119 can match eol */ YY_RULE_SETUP -#line 561 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_SESSION); return KILL_PROXYSESSION; } +#line 554 "ob_proxy_parser_utf8.l" +{ SET_DML_STMT(OBPROXY_T_SELECT_PROXY_VERSION); return SELECT_PROXY_VERSION; } YY_BREAK +/*alter config*/ case 120: /* rule 120 can match eol */ YY_RULE_SETUP -#line 562 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_GLOBAL_SESSION); return KILL_GLOBALSESSION; } +#line 557 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_ALTER_CONFIG); return ALTER_PROXYCONFIG; } YY_BREAK +/*alter resource*/ case 121: +/* rule 121 can match eol */ YY_RULE_SETUP -#line 563 "ob_proxy_parser_utf8.l" -{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_MYSQL); return KILL; } +#line 560 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_ALTER_RESOURCE); return ALTER_PROXYRESOURCE; } YY_BREAK +/*ping proxy*/ case 122: +/* rule 122 can match eol */ YY_RULE_SETUP -#line 564 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(QUERY); } +#line 563 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_PING_PROXY); return PING_PROXY; } YY_BREAK -/* obproxy_route_addr */ +/*kill*/ case 123: /* rule 123 can match eol */ YY_RULE_SETUP -#line 567 "ob_proxy_parser_utf8.l" -{ return SELECT_OBPROXY_ROUTE_ADDR; } +#line 566 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_SESSION); return KILL_PROXYSESSION; } YY_BREAK case 124: /* rule 124 can match eol */ YY_RULE_SETUP +#line 567 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_GLOBAL_SESSION); return KILL_GLOBALSESSION; } + YY_BREAK +case 125: +YY_RULE_SETUP #line 568 "ob_proxy_parser_utf8.l" +{ SET_ICMD_STMT(OBPROXY_T_ICMD_KILL_MYSQL); return KILL; } + YY_BREAK +case 126: +YY_RULE_SETUP +#line 569 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(QUERY); } + YY_BREAK +/* obproxy_route_addr */ +case 127: +/* rule 127 can match eol */ +YY_RULE_SETUP +#line 572 "ob_proxy_parser_utf8.l" +{ return SELECT_OBPROXY_ROUTE_ADDR; } + YY_BREAK +case 128: +/* rule 128 can match eol */ +YY_RULE_SETUP +#line 573 "ob_proxy_parser_utf8.l" { return SET_OBPROXY_ROUTE_ADDR; } YY_BREAK /* identifer */ -case 125: +case 129: YY_RULE_SETUP -#line 571 "ob_proxy_parser_utf8.l" +#line 576 "ob_proxy_parser_utf8.l" { SET_FOUND_ROWS(); RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 126: +case 130: YY_RULE_SETUP -#line 572 "ob_proxy_parser_utf8.l" +#line 577 "ob_proxy_parser_utf8.l" { SET_ROW_COUNT(); RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 127: +case 131: YY_RULE_SETUP -#line 573 "ob_proxy_parser_utf8.l" +#line 578 "ob_proxy_parser_utf8.l" { SET_LAST_INSERT_ID(); RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 128: +case 132: YY_RULE_SETUP -#line 574 "ob_proxy_parser_utf8.l" +#line 579 "ob_proxy_parser_utf8.l" { return ','; } YY_BREAK -case 129: +case 133: YY_RULE_SETUP -#line 575 "ob_proxy_parser_utf8.l" +#line 580 "ob_proxy_parser_utf8.l" { RETURN_INT_NUM(); } YY_BREAK -case 130: +case 134: YY_RULE_SETUP -#line 576 "ob_proxy_parser_utf8.l" +#line 581 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 131: +case 135: YY_RULE_SETUP -#line 577 "ob_proxy_parser_utf8.l" +#line 582 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_SINGLE); } YY_BREAK -case 132: +case 136: YY_RULE_SETUP -#line 578 "ob_proxy_parser_utf8.l" +#line 583 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_DOUBLE); } YY_BREAK -case 133: +case 137: YY_RULE_SETUP -#line 579 "ob_proxy_parser_utf8.l" +#line 584 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_BACK); } YY_BREAK -case 134: +case 138: YY_RULE_SETUP -#line 580 "ob_proxy_parser_utf8.l" +#line 585 "ob_proxy_parser_utf8.l" { RETURN_NUMBER_VAL(); } YY_BREAK -case 135: +case 139: YY_RULE_SETUP -#line 581 "ob_proxy_parser_utf8.l" +#line 586 "ob_proxy_parser_utf8.l" { return PLACE_HOLDER; } YY_BREAK -case 136: +case 140: YY_RULE_SETUP -#line 582 "ob_proxy_parser_utf8.l" +#line 587 "ob_proxy_parser_utf8.l" { return yytext[0]; } YY_BREAK -case 137: +case 141: YY_RULE_SETUP -#line 583 "ob_proxy_parser_utf8.l" +#line 588 "ob_proxy_parser_utf8.l" { RETURN_WITH_CALL_CHECK(yytext[0]); } YY_BREAK -case 138: +case 142: YY_RULE_SETUP -#line 584 "ob_proxy_parser_utf8.l" +#line 589 "ob_proxy_parser_utf8.l" { RETURN_WITH_CALL_CHECK('('); } YY_BREAK -case 139: +case 143: YY_RULE_SETUP -#line 585 "ob_proxy_parser_utf8.l" +#line 590 "ob_proxy_parser_utf8.l" { RETURN_WITH_CALL_CHECK(')'); } YY_BREAK -case 140: -/* rule 140 can match eol */ +case 144: +/* rule 144 can match eol */ YY_RULE_SETUP -#line 586 "ob_proxy_parser_utf8.l" +#line 591 "ob_proxy_parser_utf8.l" { } YY_BREAK /* hint option */ -case 141: -/* rule 141 can match eol */ +case 145: +/* rule 145 can match eol */ YY_RULE_SETUP -#line 589 "ob_proxy_parser_utf8.l" +#line 594 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_expr); PUSH_STATE(hint); @@ -3922,20 +3965,20 @@ YY_RULE_SETUP return SELECT_HINT_BEGIN; } YY_BREAK -case 142: -/* rule 142 can match eol */ +case 146: +/* rule 146 can match eol */ YY_RULE_SETUP -#line 595 "ob_proxy_parser_utf8.l" +#line 600 "ob_proxy_parser_utf8.l" { PUSH_STATE(hint); SET_BASIC_STMT(OBPROXY_T_UPDATE); return UPDATE_HINT_BEGIN; } YY_BREAK -case 143: -/* rule 143 can match eol */ +case 147: +/* rule 147 can match eol */ YY_RULE_SETUP -#line 600 "ob_proxy_parser_utf8.l" +#line 605 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_expr); PUSH_STATE(hint); @@ -3943,95 +3986,95 @@ YY_RULE_SETUP return DELETE_HINT_BEGIN; } YY_BREAK -case 144: -/* rule 144 can match eol */ +case 148: +/* rule 148 can match eol */ YY_RULE_SETUP -#line 606 "ob_proxy_parser_utf8.l" +#line 611 "ob_proxy_parser_utf8.l" { PUSH_STATE(hint); SET_BASIC_STMT(OBPROXY_T_INSERT); return INSERT_HINT_BEGIN; } YY_BREAK -case 145: -/* rule 145 can match eol */ +case 149: +/* rule 149 can match eol */ YY_RULE_SETUP -#line 611 "ob_proxy_parser_utf8.l" +#line 616 "ob_proxy_parser_utf8.l" { PUSH_STATE(hint); SET_BASIC_STMT(OBPROXY_T_REPLACE); return REPLACE_HINT_BEGIN; } YY_BREAK -case 146: -/* rule 146 can match eol */ +case 150: +/* rule 150 can match eol */ YY_RULE_SETUP -#line 616 "ob_proxy_parser_utf8.l" +#line 621 "ob_proxy_parser_utf8.l" { PUSH_STATE(hint); SET_BASIC_STMT(OBPROXY_T_MERGE); return MERGE_HINT_BEGIN; } YY_BREAK -case 147: -/* rule 147 can match eol */ +case 151: +/* rule 151 can match eol */ YY_RULE_SETUP -#line 622 "ob_proxy_parser_utf8.l" +#line 627 "ob_proxy_parser_utf8.l" { return AUTOCOMMIT_0; } YY_BREAK -case 148: +case 152: YY_RULE_SETUP -#line 623 "ob_proxy_parser_utf8.l" +#line 628 "ob_proxy_parser_utf8.l" { return GLOBAL; } YY_BREAK -case 149: +case 153: YY_RULE_SETUP -#line 624 "ob_proxy_parser_utf8.l" +#line 629 "ob_proxy_parser_utf8.l" { return SESSION; } YY_BREAK -case 150: +case 154: YY_RULE_SETUP -#line 625 "ob_proxy_parser_utf8.l" +#line 630 "ob_proxy_parser_utf8.l" { RETURN_INT_NUM(); } YY_BREAK -case 151: +case 155: YY_RULE_SETUP -#line 626 "ob_proxy_parser_utf8.l" +#line 631 "ob_proxy_parser_utf8.l" { return ','; } YY_BREAK -case 152: +case 156: YY_RULE_SETUP -#line 627 "ob_proxy_parser_utf8.l" +#line 632 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 153: +case 157: YY_RULE_SETUP -#line 628 "ob_proxy_parser_utf8.l" +#line 633 "ob_proxy_parser_utf8.l" { RETURN_NUMBER_VAL(); } YY_BREAK -case 154: -/* rule 154 can match eol */ +case 158: +/* rule 158 can match eol */ YY_RULE_SETUP -#line 629 "ob_proxy_parser_utf8.l" +#line 634 "ob_proxy_parser_utf8.l" { } YY_BREAK -case 155: +case 159: YY_RULE_SETUP -#line 630 "ob_proxy_parser_utf8.l" +#line 635 "ob_proxy_parser_utf8.l" { return yytext[0]; } YY_BREAK case YY_STATE_EOF(set_expr): -#line 631 "ob_proxy_parser_utf8.l" +#line 636 "ob_proxy_parser_utf8.l" { return END_P; } YY_BREAK -case 156: +case 160: YY_RULE_SETUP -#line 632 "ob_proxy_parser_utf8.l" +#line 637 "ob_proxy_parser_utf8.l" { POP_STATE(); PUSH_STATE(INITIAL); return yytext[0]; } YY_BREAK -case 157: +case 161: YY_RULE_SETUP -#line 634 "ob_proxy_parser_utf8.l" +#line 639 "ob_proxy_parser_utf8.l" { PUSH_STATE(sq); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4042,9 +4085,9 @@ YY_RULE_SETUP } } YY_BREAK -case 158: +case 162: YY_RULE_SETUP -#line 644 "ob_proxy_parser_utf8.l" +#line 649 "ob_proxy_parser_utf8.l" { PUSH_STATE(dq); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4055,9 +4098,9 @@ YY_RULE_SETUP } } YY_BREAK -case 159: +case 163: YY_RULE_SETUP -#line 654 "ob_proxy_parser_utf8.l" +#line 659 "ob_proxy_parser_utf8.l" { PUSH_STATE(bt); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4068,90 +4111,90 @@ YY_RULE_SETUP } } YY_BREAK -case 160: +case 164: YY_RULE_SETUP -#line 663 "ob_proxy_parser_utf8.l" +#line 668 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK -case 161: +case 165: YY_RULE_SETUP -#line 665 "ob_proxy_parser_utf8.l" +#line 670 "ob_proxy_parser_utf8.l" { POP_STATE(); RETURN_IGNORED_WORD(); } YY_BREAK -case 162: +case 166: YY_RULE_SETUP -#line 666 "ob_proxy_parser_utf8.l" +#line 671 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 163: +case 167: YY_RULE_SETUP -#line 668 "ob_proxy_parser_utf8.l" +#line 673 "ob_proxy_parser_utf8.l" { return '('; } YY_BREAK -case 164: +case 168: YY_RULE_SETUP -#line 669 "ob_proxy_parser_utf8.l" +#line 674 "ob_proxy_parser_utf8.l" { return ')'; } YY_BREAK -case 165: +case 169: YY_RULE_SETUP -#line 670 "ob_proxy_parser_utf8.l" +#line 675 "ob_proxy_parser_utf8.l" { return QUERY_TIMEOUT; } YY_BREAK -case 166: +case 170: YY_RULE_SETUP -#line 671 "ob_proxy_parser_utf8.l" +#line 676 "ob_proxy_parser_utf8.l" { RETURN_INT_NUM(); } YY_BREAK -case 167: +case 171: YY_RULE_SETUP -#line 672 "ob_proxy_parser_utf8.l" +#line 677 "ob_proxy_parser_utf8.l" { return READ_CONSISTENCY; } YY_BREAK -case 168: +case 172: YY_RULE_SETUP -#line 673 "ob_proxy_parser_utf8.l" +#line 678 "ob_proxy_parser_utf8.l" { return WEAK; } YY_BREAK -case 169: +case 173: YY_RULE_SETUP -#line 674 "ob_proxy_parser_utf8.l" +#line 679 "ob_proxy_parser_utf8.l" { return STRONG; } YY_BREAK -case 170: +case 174: YY_RULE_SETUP -#line 675 "ob_proxy_parser_utf8.l" +#line 680 "ob_proxy_parser_utf8.l" { return FROZEN; } YY_BREAK -case 171: +case 175: YY_RULE_SETUP -#line 676 "ob_proxy_parser_utf8.l" +#line 681 "ob_proxy_parser_utf8.l" { return INDEX; } YY_BREAK -case 172: +case 176: YY_RULE_SETUP -#line 677 "ob_proxy_parser_utf8.l" +#line 682 "ob_proxy_parser_utf8.l" { return yytext[0]; } YY_BREAK -case 173: +case 177: YY_RULE_SETUP -#line 678 "ob_proxy_parser_utf8.l" +#line 683 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 174: +case 178: YY_RULE_SETUP -#line 679 "ob_proxy_parser_utf8.l" +#line 684 "ob_proxy_parser_utf8.l" { POP_STATE(); return HINT_END; } YY_BREAK -case 175: +case 179: YY_RULE_SETUP -#line 680 "ob_proxy_parser_utf8.l" +#line 685 "ob_proxy_parser_utf8.l" {} YY_BREAK /* comment */ -case 176: +case 180: YY_RULE_SETUP -#line 683 "ob_proxy_parser_utf8.l" +#line 688 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_c_comment); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4166,25 +4209,25 @@ YY_RULE_SETUP } } YY_BREAK -case 177: +case 181: YY_RULE_SETUP -#line 697 "ob_proxy_parser_utf8.l" +#line 702 "ob_proxy_parser_utf8.l" { POP_STATE(); } YY_BREAK -case 178: -/* rule 178 can match eol */ +case 182: +/* rule 182 can match eol */ YY_RULE_SETUP -#line 698 "ob_proxy_parser_utf8.l" +#line 703 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 179: +case 183: YY_RULE_SETUP -#line 699 "ob_proxy_parser_utf8.l" +#line 704 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 180: +case 184: YY_RULE_SETUP -#line 701 "ob_proxy_parser_utf8.l" +#line 706 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4195,15 +4238,15 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 181: +case 185: YY_RULE_SETUP -#line 711 "ob_proxy_parser_utf8.l" +#line 716 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_old_comment_expr); } YY_BREAK -case 182: -/* rule 182 can match eol */ +case 186: +/* rule 186 can match eol */ YY_RULE_SETUP -#line 712 "ob_proxy_parser_utf8.l" +#line 717 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_old_comment_expr); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4213,10 +4256,10 @@ YY_RULE_SETUP } } YY_BREAK -case 183: -/* rule 183 can match eol */ +case 187: +/* rule 187 can match eol */ YY_RULE_SETUP -#line 721 "ob_proxy_parser_utf8.l" +#line 726 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_old_comment_expr); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4228,9 +4271,9 @@ YY_RULE_SETUP } } YY_BREAK -case 184: +case 188: YY_RULE_SETUP -#line 732 "ob_proxy_parser_utf8.l" +#line 737 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4239,9 +4282,9 @@ YY_RULE_SETUP } } YY_BREAK -case 185: +case 189: YY_RULE_SETUP -#line 740 "ob_proxy_parser_utf8.l" +#line 745 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4252,29 +4295,29 @@ YY_RULE_SETUP } } YY_BREAK -case 186: +case 190: YY_RULE_SETUP -#line 750 "ob_proxy_parser_utf8.l" +#line 755 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 187: +case 191: YY_RULE_SETUP -#line 751 "ob_proxy_parser_utf8.l" +#line 756 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_SINGLE); } YY_BREAK -case 188: +case 192: YY_RULE_SETUP -#line 752 "ob_proxy_parser_utf8.l" +#line 757 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_DOUBLE); } YY_BREAK -case 189: +case 193: YY_RULE_SETUP -#line 753 "ob_proxy_parser_utf8.l" +#line 758 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD_WITH_QUOTE(NAME_OB, OBPROXY_QUOTE_T_BACK); } YY_BREAK -case 190: +case 194: YY_RULE_SETUP -#line 754 "ob_proxy_parser_utf8.l" +#line 759 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4286,160 +4329,160 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 191: -YY_RULE_SETUP -#line 764 "ob_proxy_parser_utf8.l" -{} - YY_BREAK -case 192: -YY_RULE_SETUP -#line 766 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(ODP_COMMENT); } - YY_BREAK -case 193: -/* rule 193 can match eol */ -YY_RULE_SETUP -#line 767 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(GROUP_ID); } - YY_BREAK -case 194: -/* rule 194 can match eol */ -YY_RULE_SETUP -#line 768 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TABLE_ID); } - YY_BREAK case 195: -/* rule 195 can match eol */ YY_RULE_SETUP #line 769 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TABLE_NAME); } +{} YY_BREAK case 196: -/* rule 196 can match eol */ YY_RULE_SETUP -#line 770 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(ELASTIC_ID); } +#line 771 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(ODP_COMMENT); } YY_BREAK case 197: /* rule 197 can match eol */ YY_RULE_SETUP -#line 771 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TESTLOAD); } +#line 772 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(GROUP_ID); } YY_BREAK case 198: /* rule 198 can match eol */ YY_RULE_SETUP -#line 772 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(DISASTER_STATUS); } +#line 773 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TABLE_ID); } YY_BREAK case 199: /* rule 199 can match eol */ YY_RULE_SETUP -#line 773 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TNT_ID); } +#line 774 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TABLE_NAME); } YY_BREAK case 200: /* rule 200 can match eol */ YY_RULE_SETUP -#line 774 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TRACE_ID); } +#line 775 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(ELASTIC_ID); } YY_BREAK case 201: /* rule 201 can match eol */ YY_RULE_SETUP -#line 775 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT_NAME_OB_DOT(NAME_OB_DOT); } +#line 776 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TESTLOAD); } YY_BREAK case 202: /* rule 202 can match eol */ YY_RULE_SETUP -#line 776 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(RPC_ID); } +#line 777 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(DISASTER_STATUS); } YY_BREAK case 203: +/* rule 203 can match eol */ YY_RULE_SETUP #line 778 "ob_proxy_parser_utf8.l" -{ return GROUP_ID; } +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TNT_ID); } YY_BREAK case 204: +/* rule 204 can match eol */ YY_RULE_SETUP #line 779 "ob_proxy_parser_utf8.l" -{ return TABLE_ID; } +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(TRACE_ID); } YY_BREAK case 205: +/* rule 205 can match eol */ YY_RULE_SETUP #line 780 "ob_proxy_parser_utf8.l" -{ return TABLE_NAME; } +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT_NAME_OB_DOT(NAME_OB_DOT); } YY_BREAK case 206: +/* rule 206 can match eol */ YY_RULE_SETUP #line 781 "ob_proxy_parser_utf8.l" -{ return ELASTIC_ID; } +{ PUSH_STATE(in_odp_comment_expr); RETURN_SHARD_COMMENT(RPC_ID); } YY_BREAK case 207: YY_RULE_SETUP -#line 782 "ob_proxy_parser_utf8.l" -{ return TESTLOAD; } +#line 783 "ob_proxy_parser_utf8.l" +{ return GROUP_ID; } YY_BREAK case 208: YY_RULE_SETUP -#line 783 "ob_proxy_parser_utf8.l" -{ return DISASTER_STATUS; } +#line 784 "ob_proxy_parser_utf8.l" +{ return TABLE_ID; } YY_BREAK case 209: YY_RULE_SETUP -#line 784 "ob_proxy_parser_utf8.l" -{ return TNT_ID; } +#line 785 "ob_proxy_parser_utf8.l" +{ return TABLE_NAME; } YY_BREAK case 210: YY_RULE_SETUP -#line 785 "ob_proxy_parser_utf8.l" -{ return TRACE_ID; } +#line 786 "ob_proxy_parser_utf8.l" +{ return ELASTIC_ID; } YY_BREAK case 211: YY_RULE_SETUP -#line 786 "ob_proxy_parser_utf8.l" -{ return RPC_ID; } +#line 787 "ob_proxy_parser_utf8.l" +{ return TESTLOAD; } YY_BREAK case 212: YY_RULE_SETUP -#line 787 "ob_proxy_parser_utf8.l" -{ ENTER_QUOTE_STATE(comment_sq); } +#line 788 "ob_proxy_parser_utf8.l" +{ return DISASTER_STATUS; } YY_BREAK case 213: YY_RULE_SETUP -#line 788 "ob_proxy_parser_utf8.l" -{ return ','; } +#line 789 "ob_proxy_parser_utf8.l" +{ return TNT_ID; } YY_BREAK case 214: YY_RULE_SETUP -#line 789 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(NAME_OB); } +#line 790 "ob_proxy_parser_utf8.l" +{ return TRACE_ID; } YY_BREAK case 215: YY_RULE_SETUP -#line 790 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_odp_comment_value_expr); return yytext[0]; } +#line 791 "ob_proxy_parser_utf8.l" +{ return RPC_ID; } YY_BREAK case 216: YY_RULE_SETUP -#line 791 "ob_proxy_parser_utf8.l" -{ return yytext[0]; } +#line 792 "ob_proxy_parser_utf8.l" +{ ENTER_QUOTE_STATE(comment_sq); } YY_BREAK case 217: -/* rule 217 can match eol */ YY_RULE_SETUP -#line 792 "ob_proxy_parser_utf8.l" -{} +#line 793 "ob_proxy_parser_utf8.l" +{ return ','; } YY_BREAK case 218: YY_RULE_SETUP -#line 793 "ob_proxy_parser_utf8.l" -{ return ERROR; } +#line 794 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK case 219: YY_RULE_SETUP -#line 794 "ob_proxy_parser_utf8.l" +#line 795 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_odp_comment_value_expr); return yytext[0]; } + YY_BREAK +case 220: +YY_RULE_SETUP +#line 796 "ob_proxy_parser_utf8.l" +{ return yytext[0]; } + YY_BREAK +case 221: +/* rule 221 can match eol */ +YY_RULE_SETUP +#line 797 "ob_proxy_parser_utf8.l" +{} + YY_BREAK +case 222: +YY_RULE_SETUP +#line 798 "ob_proxy_parser_utf8.l" +{ return ERROR; } + YY_BREAK +case 223: +YY_RULE_SETUP +#line 799 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4451,22 +4494,22 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 220: +case 224: YY_RULE_SETUP -#line 805 "ob_proxy_parser_utf8.l" +#line 810 "ob_proxy_parser_utf8.l" { ENTER_QUOTE_STATE(comment_sq); } YY_BREAK -case 221: +case 225: YY_RULE_SETUP -#line 806 "ob_proxy_parser_utf8.l" +#line 811 "ob_proxy_parser_utf8.l" { POP_STATE(); return ','; } YY_BREAK -case 222: +case 226: YY_RULE_SETUP -#line 810 "ob_proxy_parser_utf8.l" +#line 815 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4479,15 +4522,15 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 223: -/* rule 223 can match eol */ +case 227: +/* rule 227 can match eol */ YY_RULE_SETUP -#line 821 "ob_proxy_parser_utf8.l" +#line 826 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 224: +case 228: YY_RULE_SETUP -#line 822 "ob_proxy_parser_utf8.l" +#line 827 "ob_proxy_parser_utf8.l" { do { PUSH_STATE(in_odp_comment_value_expr_calc) @@ -4500,10 +4543,10 @@ YY_RULE_SETUP } while (0); } YY_BREAK -case 225: -/* rule 225 can match eol */ +case 229: +/* rule 229 can match eol */ YY_RULE_SETUP -#line 834 "ob_proxy_parser_utf8.l" +#line 839 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4516,9 +4559,9 @@ YY_RULE_SETUP } } YY_BREAK -case 226: +case 230: YY_RULE_SETUP -#line 846 "ob_proxy_parser_utf8.l" +#line 851 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4526,90 +4569,90 @@ YY_RULE_SETUP } } YY_BREAK -case 227: +case 231: YY_RULE_SETUP -#line 853 "ob_proxy_parser_utf8.l" +#line 858 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_dbp_comment_expr); RETURN_SHARD_COMMENT(DBP_COMMENT); } YY_BREAK -case 228: +case 232: YY_RULE_SETUP -#line 854 "ob_proxy_parser_utf8.l" +#line 859 "ob_proxy_parser_utf8.l" { return ROUTE_TAG; } YY_BREAK -case 229: +case 233: YY_RULE_SETUP -#line 855 "ob_proxy_parser_utf8.l" +#line 860 "ob_proxy_parser_utf8.l" { return SYS_TAG; } YY_BREAK -case 230: +case 234: YY_RULE_SETUP -#line 856 "ob_proxy_parser_utf8.l" +#line 861 "ob_proxy_parser_utf8.l" { return SCAN_ALL; } YY_BREAK -case 231: +case 235: YY_RULE_SETUP -#line 857 "ob_proxy_parser_utf8.l" +#line 862 "ob_proxy_parser_utf8.l" { return SHARD_KEY; } YY_BREAK -case 232: +case 236: YY_RULE_SETUP -#line 858 "ob_proxy_parser_utf8.l" +#line 863 "ob_proxy_parser_utf8.l" { return TABLE_NAME;} YY_BREAK -case 233: +case 237: YY_RULE_SETUP -#line 859 "ob_proxy_parser_utf8.l" +#line 864 "ob_proxy_parser_utf8.l" { return PARALL; } YY_BREAK -case 234: +case 238: YY_RULE_SETUP -#line 860 "ob_proxy_parser_utf8.l" +#line 865 "ob_proxy_parser_utf8.l" { return GROUP_ID; } YY_BREAK -case 235: +case 239: YY_RULE_SETUP -#line 861 "ob_proxy_parser_utf8.l" +#line 866 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_dbp_comment_trace_expr); return TRACE; } YY_BREAK -case 236: +case 240: YY_RULE_SETUP -#line 862 "ob_proxy_parser_utf8.l" +#line 867 "ob_proxy_parser_utf8.l" { ENTER_QUOTE_STATE(comment_sq); } YY_BREAK -case 237: +case 241: YY_RULE_SETUP -#line 863 "ob_proxy_parser_utf8.l" +#line 868 "ob_proxy_parser_utf8.l" { return yytext[0]; } YY_BREAK -case 238: +case 242: YY_RULE_SETUP -#line 864 "ob_proxy_parser_utf8.l" +#line 869 "ob_proxy_parser_utf8.l" { return '('; } YY_BREAK -case 239: +case 243: YY_RULE_SETUP -#line 865 "ob_proxy_parser_utf8.l" +#line 870 "ob_proxy_parser_utf8.l" { return ')'; } YY_BREAK -case 240: +case 244: YY_RULE_SETUP -#line 866 "ob_proxy_parser_utf8.l" +#line 871 "ob_proxy_parser_utf8.l" { return ','; } YY_BREAK -case 241: -/* rule 241 can match eol */ +case 245: +/* rule 245 can match eol */ YY_RULE_SETUP -#line 867 "ob_proxy_parser_utf8.l" +#line 872 "ob_proxy_parser_utf8.l" { } YY_BREAK -case 242: +case 246: YY_RULE_SETUP -#line 868 "ob_proxy_parser_utf8.l" +#line 873 "ob_proxy_parser_utf8.l" { RETURN_NON_RESERVED_KEYWORD(NAME_OB); } YY_BREAK -case 243: +case 247: YY_RULE_SETUP -#line 870 "ob_proxy_parser_utf8.l" +#line 875 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4621,39 +4664,39 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 244: +case 248: YY_RULE_SETUP -#line 880 "ob_proxy_parser_utf8.l" +#line 885 "ob_proxy_parser_utf8.l" {return yytext[0];} YY_BREAK -case 245: +case 249: YY_RULE_SETUP -#line 881 "ob_proxy_parser_utf8.l" +#line 886 "ob_proxy_parser_utf8.l" { PUSH_STATE(in_dbp_comment_trace_value_expr); return '('; } YY_BREAK -case 246: -/* rule 246 can match eol */ +case 250: +/* rule 250 can match eol */ YY_RULE_SETUP -#line 882 "ob_proxy_parser_utf8.l" +#line 887 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 247: +case 251: YY_RULE_SETUP -#line 884 "ob_proxy_parser_utf8.l" +#line 889 "ob_proxy_parser_utf8.l" { ENTER_QUOTE_STATE(comment_sq); } YY_BREAK -case 248: +case 252: YY_RULE_SETUP -#line 885 "ob_proxy_parser_utf8.l" +#line 890 "ob_proxy_parser_utf8.l" { POP_STATE(); POP_STATE(); return ')'; } YY_BREAK -case 249: +case 253: YY_RULE_SETUP -#line 890 "ob_proxy_parser_utf8.l" +#line 895 "ob_proxy_parser_utf8.l" { do { PUSH_STATE(in_dbp_comment_trace_value_expr_calc) @@ -4667,9 +4710,9 @@ YY_RULE_SETUP } while (0); } YY_BREAK -case 250: +case 254: YY_RULE_SETUP -#line 902 "ob_proxy_parser_utf8.l" +#line 907 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4683,15 +4726,15 @@ YY_RULE_SETUP return COMMENT_END; } YY_BREAK -case 251: -/* rule 251 can match eol */ +case 255: +/* rule 255 can match eol */ YY_RULE_SETUP -#line 914 "ob_proxy_parser_utf8.l" +#line 919 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 252: +case 256: YY_RULE_SETUP -#line 915 "ob_proxy_parser_utf8.l" +#line 920 "ob_proxy_parser_utf8.l" { do { PUSH_STATE(in_dbp_comment_trace_value_expr_calc) @@ -4704,10 +4747,10 @@ YY_RULE_SETUP } while (0); } YY_BREAK -case 253: -/* rule 253 can match eol */ +case 257: +/* rule 257 can match eol */ YY_RULE_SETUP -#line 927 "ob_proxy_parser_utf8.l" +#line 932 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4720,9 +4763,9 @@ YY_RULE_SETUP } } YY_BREAK -case 254: +case 258: YY_RULE_SETUP -#line 939 "ob_proxy_parser_utf8.l" +#line 944 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4731,9 +4774,9 @@ YY_RULE_SETUP } YY_BREAK /* quote */ -case 255: +case 259: YY_RULE_SETUP -#line 947 "ob_proxy_parser_utf8.l" +#line 952 "ob_proxy_parser_utf8.l" { PUSH_STATE(sq); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4744,10 +4787,10 @@ YY_RULE_SETUP } } YY_BREAK -case 256: -/* rule 256 can match eol */ +case 260: +/* rule 260 can match eol */ YY_RULE_SETUP -#line 957 "ob_proxy_parser_utf8.l" +#line 962 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4755,9 +4798,9 @@ YY_RULE_SETUP } } YY_BREAK -case 257: +case 261: YY_RULE_SETUP -#line 964 "ob_proxy_parser_utf8.l" +#line 969 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4774,32 +4817,32 @@ YY_RULE_SETUP } } YY_BREAK -case 258: +case 262: YY_RULE_SETUP -#line 980 "ob_proxy_parser_utf8.l" +#line 985 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 259: -/* rule 259 can match eol */ +case 263: +/* rule 263 can match eol */ YY_RULE_SETUP -#line 981 "ob_proxy_parser_utf8.l" +#line 986 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 260: -/* rule 260 can match eol */ +case 264: +/* rule 264 can match eol */ YY_RULE_SETUP -#line 982 "ob_proxy_parser_utf8.l" +#line 987 "ob_proxy_parser_utf8.l" {} YY_BREAK case YY_STATE_EOF(sq): -#line 983 "ob_proxy_parser_utf8.l" +#line 988 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK /* comment sq return name_str */ -case 261: -/* rule 261 can match eol */ +case 265: +/* rule 265 can match eol */ YY_RULE_SETUP -#line 986 "ob_proxy_parser_utf8.l" +#line 991 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4807,9 +4850,9 @@ YY_RULE_SETUP } } YY_BREAK -case 262: +case 266: YY_RULE_SETUP -#line 993 "ob_proxy_parser_utf8.l" +#line 998 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4821,31 +4864,31 @@ YY_RULE_SETUP } } YY_BREAK -case 263: +case 267: YY_RULE_SETUP -#line 1004 "ob_proxy_parser_utf8.l" +#line 1009 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 264: -/* rule 264 can match eol */ +case 268: +/* rule 268 can match eol */ YY_RULE_SETUP -#line 1005 "ob_proxy_parser_utf8.l" +#line 1010 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 265: -/* rule 265 can match eol */ +case 269: +/* rule 269 can match eol */ YY_RULE_SETUP -#line 1006 "ob_proxy_parser_utf8.l" +#line 1011 "ob_proxy_parser_utf8.l" {} YY_BREAK case YY_STATE_EOF(comment_sq): -#line 1007 "ob_proxy_parser_utf8.l" +#line 1012 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK /* dquote */ -case 266: +case 270: YY_RULE_SETUP -#line 1010 "ob_proxy_parser_utf8.l" +#line 1015 "ob_proxy_parser_utf8.l" { PUSH_STATE(dq); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4856,10 +4899,10 @@ YY_RULE_SETUP } } YY_BREAK -case 267: -/* rule 267 can match eol */ +case 271: +/* rule 271 can match eol */ YY_RULE_SETUP -#line 1020 "ob_proxy_parser_utf8.l" +#line 1025 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (OB_NOTNULL(p)) { @@ -4867,9 +4910,9 @@ YY_RULE_SETUP } } YY_BREAK -case 268: +case 272: YY_RULE_SETUP -#line 1027 "ob_proxy_parser_utf8.l" +#line 1032 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4886,31 +4929,31 @@ YY_RULE_SETUP } } YY_BREAK -case 269: +case 273: YY_RULE_SETUP -#line 1043 "ob_proxy_parser_utf8.l" +#line 1048 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 270: -/* rule 270 can match eol */ +case 274: +/* rule 274 can match eol */ YY_RULE_SETUP -#line 1044 "ob_proxy_parser_utf8.l" +#line 1049 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 271: -/* rule 271 can match eol */ +case 275: +/* rule 275 can match eol */ YY_RULE_SETUP -#line 1045 "ob_proxy_parser_utf8.l" +#line 1050 "ob_proxy_parser_utf8.l" {} YY_BREAK case YY_STATE_EOF(dq): -#line 1046 "ob_proxy_parser_utf8.l" +#line 1051 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK /* backtick */ -case 272: +case 276: YY_RULE_SETUP -#line 1050 "ob_proxy_parser_utf8.l" +#line 1055 "ob_proxy_parser_utf8.l" { PUSH_STATE(bt); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4921,9 +4964,9 @@ YY_RULE_SETUP } } YY_BREAK -case 273: +case 277: YY_RULE_SETUP -#line 1060 "ob_proxy_parser_utf8.l" +#line 1065 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (NULL != p && NULL != p->tmp_buf_ && p->tmp_len_ + 1 < OBPROXY_MAX_NAME_LENGTH) { @@ -4931,10 +4974,10 @@ YY_RULE_SETUP } } YY_BREAK -case 274: -/* rule 274 can match eol */ +case 278: +/* rule 278 can match eol */ YY_RULE_SETUP -#line 1067 "ob_proxy_parser_utf8.l" +#line 1072 "ob_proxy_parser_utf8.l" { ObProxyParseResult *p = (ObProxyParseResult *)yyextra; if (NULL != p && NULL != p->tmp_buf_ && p->tmp_len_ + yyleng < OBPROXY_MAX_NAME_LENGTH) { @@ -4943,9 +4986,9 @@ YY_RULE_SETUP } } YY_BREAK -case 275: +case 279: YY_RULE_SETUP -#line 1075 "ob_proxy_parser_utf8.l" +#line 1080 "ob_proxy_parser_utf8.l" { POP_STATE(); ObProxyParseResult *p = (ObProxyParseResult *)yyextra; @@ -4959,243 +5002,243 @@ YY_RULE_SETUP } YY_BREAK case YY_STATE_EOF(bt): -#line 1087 "ob_proxy_parser_utf8.l" +#line 1092 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK -case 276: +case 280: YY_RULE_SETUP -#line 1091 "ob_proxy_parser_utf8.l" +#line 1096 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 277: -/* rule 277 can match eol */ +case 281: +/* rule 281 can match eol */ YY_RULE_SETUP -#line 1092 "ob_proxy_parser_utf8.l" +#line 1097 "ob_proxy_parser_utf8.l" {} YY_BREAK -case 278: +case 282: YY_RULE_SETUP -#line 1093 "ob_proxy_parser_utf8.l" +#line 1098 "ob_proxy_parser_utf8.l" { POP_STATE(); RETURN_IGNORED_WORD(); } YY_BREAK case YY_STATE_EOF(bt_in_expr): -#line 1094 "ob_proxy_parser_utf8.l" +#line 1099 "ob_proxy_parser_utf8.l" { return ERROR; } YY_BREAK /* some useful keyword */ -case 279: -/* rule 279 can match eol */ -YY_RULE_SETUP -#line 1097 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD_FOR_DUAL(); } - YY_BREAK -case 280: -YY_RULE_SETUP -#line 1098 "ob_proxy_parser_utf8.l" -{ POP_STATE(); PUSH_STATE(INITIAL); return FROM; } - YY_BREAK -case 281: -YY_RULE_SETUP -#line 1099 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_subquery); return '('; } - YY_BREAK -case 282: -YY_RULE_SETUP -#line 1100 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(dq); } - YY_BREAK case 283: +/* rule 283 can match eol */ YY_RULE_SETUP -#line 1101 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(sq); } +#line 1102 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD_FOR_DUAL(); } YY_BREAK case 284: YY_RULE_SETUP -#line 1102 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(bt_in_expr); } +#line 1103 "ob_proxy_parser_utf8.l" +{ POP_STATE(); PUSH_STATE(INITIAL); return FROM; } YY_BREAK case 285: YY_RULE_SETUP -#line 1103 "ob_proxy_parser_utf8.l" -{ SET_FOUND_ROWS(); RETURN_IGNORED_WORD(); } +#line 1104 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_subquery); return '('; } YY_BREAK case 286: YY_RULE_SETUP -#line 1104 "ob_proxy_parser_utf8.l" -{ SET_ROW_COUNT(); RETURN_IGNORED_WORD(); } +#line 1105 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(dq); } YY_BREAK case 287: YY_RULE_SETUP -#line 1105 "ob_proxy_parser_utf8.l" -{ SET_LAST_INSERT_ID(); RETURN_IGNORED_WORD(); } +#line 1106 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(sq); } YY_BREAK case 288: YY_RULE_SETUP -#line 1106 "ob_proxy_parser_utf8.l" -{ SET_GLOBAL_SET_STMT(); RETURN_IGNORED_WORD(); } +#line 1107 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(bt_in_expr); } YY_BREAK case 289: -/* rule 289 can match eol */ YY_RULE_SETUP -#line 1107 "ob_proxy_parser_utf8.l" -{ RETURN_COL_NAME(TX_READ_ONLY); } +#line 1108 "ob_proxy_parser_utf8.l" +{ SET_FOUND_ROWS(); RETURN_IGNORED_WORD(); } YY_BREAK case 290: -/* rule 290 can match eol */ YY_RULE_SETUP -#line 1108 "ob_proxy_parser_utf8.l" -{ return AUTOCOMMIT_0; } +#line 1109 "ob_proxy_parser_utf8.l" +{ SET_ROW_COUNT(); RETURN_IGNORED_WORD(); } YY_BREAK case 291: YY_RULE_SETUP -#line 1109 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } - YY_BREAK -case YY_STATE_EOF(in_expr): #line 1110 "ob_proxy_parser_utf8.l" -{ return END_P; } +{ SET_LAST_INSERT_ID(); RETURN_IGNORED_WORD(); } YY_BREAK case 292: -/* rule 292 can match eol */ YY_RULE_SETUP #line 1111 "ob_proxy_parser_utf8.l" -{ } +{ SET_GLOBAL_SET_STMT(); RETURN_IGNORED_WORD(); } YY_BREAK case 293: +/* rule 293 can match eol */ YY_RULE_SETUP #line 1112 "ob_proxy_parser_utf8.l" -{ POP_STATE(); PUSH_STATE(INITIAL); return yytext[0]; } +{ RETURN_COL_NAME(TX_READ_ONLY); } YY_BREAK case 294: +/* rule 294 can match eol */ YY_RULE_SETUP #line 1113 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } +{ return AUTOCOMMIT_0; } YY_BREAK case 295: YY_RULE_SETUP +#line 1114 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD(); } + YY_BREAK +case YY_STATE_EOF(in_expr): #line 1115 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_expr); return SELECT; } +{ return END_P; } YY_BREAK case 296: +/* rule 296 can match eol */ YY_RULE_SETUP #line 1116 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_subquery); return '('; } +{ } YY_BREAK case 297: YY_RULE_SETUP #line 1117 "ob_proxy_parser_utf8.l" -{ POP_STATE(); return ')'; } +{ POP_STATE(); PUSH_STATE(INITIAL); return yytext[0]; } YY_BREAK case 298: YY_RULE_SETUP #line 1118 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(dq); } +{ RETURN_IGNORED_WORD(); } YY_BREAK case 299: YY_RULE_SETUP -#line 1119 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(sq); } - YY_BREAK -case YY_STATE_EOF(in_subquery): #line 1120 "ob_proxy_parser_utf8.l" -{ return END_P; } +{ PUSH_STATE(in_expr); return SELECT; } YY_BREAK case 300: -/* rule 300 can match eol */ YY_RULE_SETUP #line 1121 "ob_proxy_parser_utf8.l" -{ } +{ PUSH_STATE(in_subquery); return '('; } YY_BREAK case 301: YY_RULE_SETUP #line 1122 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_no_select_query); RETURN_IGNORED_WORD(); } +{ POP_STATE(); return ')'; } YY_BREAK case 302: YY_RULE_SETUP -#line 1124 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(in_subquery); return '('; } +#line 1123 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(dq); } YY_BREAK case 303: YY_RULE_SETUP +#line 1124 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(sq); } + YY_BREAK +case YY_STATE_EOF(in_subquery): #line 1125 "ob_proxy_parser_utf8.l" -{ POP_STATE(); POP_STATE(); return ')'; } +{ return END_P; } YY_BREAK case 304: +/* rule 304 can match eol */ YY_RULE_SETUP #line 1126 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(dq); } +{ } YY_BREAK case 305: YY_RULE_SETUP #line 1127 "ob_proxy_parser_utf8.l" -{ PUSH_STATE(sq); } +{ PUSH_STATE(in_no_select_query); RETURN_IGNORED_WORD(); } YY_BREAK case 306: YY_RULE_SETUP -#line 1128 "ob_proxy_parser_utf8.l" -{ SET_FOUND_ROWS(); RETURN_IGNORED_WORD(); } +#line 1129 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(in_subquery); return '('; } YY_BREAK case 307: YY_RULE_SETUP -#line 1129 "ob_proxy_parser_utf8.l" -{ SET_ROW_COUNT(); RETURN_IGNORED_WORD(); } +#line 1130 "ob_proxy_parser_utf8.l" +{ POP_STATE(); POP_STATE(); return ')'; } YY_BREAK case 308: YY_RULE_SETUP -#line 1130 "ob_proxy_parser_utf8.l" -{ SET_LAST_INSERT_ID(); RETURN_IGNORED_WORD(); } +#line 1131 "ob_proxy_parser_utf8.l" +{ PUSH_STATE(dq); } YY_BREAK case 309: YY_RULE_SETUP -#line 1131 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } - YY_BREAK -case YY_STATE_EOF(in_no_select_query): #line 1132 "ob_proxy_parser_utf8.l" -{ return END_P; } +{ PUSH_STATE(sq); } YY_BREAK case 310: -/* rule 310 can match eol */ YY_RULE_SETUP #line 1133 "ob_proxy_parser_utf8.l" -{ } +{ SET_FOUND_ROWS(); RETURN_IGNORED_WORD(); } YY_BREAK case 311: YY_RULE_SETUP #line 1134 "ob_proxy_parser_utf8.l" -{ RETURN_IGNORED_WORD(); } +{ SET_ROW_COUNT(); RETURN_IGNORED_WORD(); } YY_BREAK case 312: YY_RULE_SETUP -#line 1136 "ob_proxy_parser_utf8.l" -{ return FROM; } +#line 1135 "ob_proxy_parser_utf8.l" +{ SET_LAST_INSERT_ID(); RETURN_IGNORED_WORD(); } YY_BREAK case 313: YY_RULE_SETUP +#line 1136 "ob_proxy_parser_utf8.l" +{ RETURN_IGNORED_WORD(); } + YY_BREAK +case YY_STATE_EOF(in_no_select_query): #line 1137 "ob_proxy_parser_utf8.l" -{ RETURN_NON_RESERVED_KEYWORD(NAME_OB); } +{ return END_P; } YY_BREAK case 314: +/* rule 314 can match eol */ YY_RULE_SETUP #line 1138 "ob_proxy_parser_utf8.l" -{ POP_STATE(); } +{ } YY_BREAK case 315: YY_RULE_SETUP #line 1139 "ob_proxy_parser_utf8.l" -{ POP_STATE(); } +{ RETURN_IGNORED_WORD(); } YY_BREAK case 316: YY_RULE_SETUP #line 1141 "ob_proxy_parser_utf8.l" -{ POP_STATE(); return BEGI;} +{ return FROM; } YY_BREAK case 317: YY_RULE_SETUP #line 1142 "ob_proxy_parser_utf8.l" +{ RETURN_NON_RESERVED_KEYWORD(NAME_OB); } + YY_BREAK +case 318: +YY_RULE_SETUP +#line 1143 "ob_proxy_parser_utf8.l" +{ POP_STATE(); } + YY_BREAK +case 319: +YY_RULE_SETUP +#line 1144 "ob_proxy_parser_utf8.l" +{ POP_STATE(); } + YY_BREAK +case 320: +YY_RULE_SETUP +#line 1146 "ob_proxy_parser_utf8.l" +{ POP_STATE(); return BEGI;} + YY_BREAK +case 321: +YY_RULE_SETUP +#line 1147 "ob_proxy_parser_utf8.l" {} YY_BREAK case YY_STATE_EOF(INITIAL): @@ -5214,20 +5257,20 @@ case YY_STATE_EOF(in_dbp_comment_trace_value_expr_calc): case YY_STATE_EOF(in_anonymous_block): case YY_STATE_EOF(prepare): case YY_STATE_EOF(insert_all_expr): -#line 1144 "ob_proxy_parser_utf8.l" +#line 1149 "ob_proxy_parser_utf8.l" { return END_P; } YY_BREAK -case 318: +case 322: YY_RULE_SETUP -#line 1145 "ob_proxy_parser_utf8.l" +#line 1150 "ob_proxy_parser_utf8.l" { RETURN_IGNORED_WORD(); } YY_BREAK -case 319: +case 323: YY_RULE_SETUP -#line 1146 "ob_proxy_parser_utf8.l" +#line 1151 "ob_proxy_parser_utf8.l" ECHO; YY_BREAK -#line 5233 "ob_proxy_parser_utf8_lex.c" +#line 5276 "ob_proxy_parser_utf8_lex.c" case YY_END_OF_BUFFER: { @@ -5519,7 +5562,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 2073 ) + if ( yy_current_state >= 2102 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; @@ -5548,11 +5591,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 2073 ) + if ( yy_current_state >= 2102 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 2072); + yy_is_jam = (yy_current_state == 2101); return yy_is_jam ? 0 : yy_current_state; } @@ -6377,7 +6420,7 @@ static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner) #define YYTABLES_NAME "yytables" -#line 1146 "ob_proxy_parser_utf8.l" +#line 1151 "ob_proxy_parser_utf8.l" inline void *ob_proxy_parser_utf8_yyalloc(size_t bytes,void *yyscanner) @@ -6525,150 +6568,153 @@ extern int ob_proxy_parser_utf8_yydebug; ALTER = 271, TRUNCATE = 272, RENAME = 273, - GRANT = 274, - REVOKE = 275, - ANALYZE = 276, - PURGE = 277, - COMMENT = 278, - FROM = 279, - DUAL = 280, - PREPARE = 281, - EXECUTE = 282, - USING = 283, - SELECT_HINT_BEGIN = 284, - UPDATE_HINT_BEGIN = 285, - DELETE_HINT_BEGIN = 286, - INSERT_HINT_BEGIN = 287, - REPLACE_HINT_BEGIN = 288, - MERGE_HINT_BEGIN = 289, - HINT_END = 290, - COMMENT_BEGIN = 291, - COMMENT_END = 292, - ROUTE_TABLE = 293, - ROUTE_PART_KEY = 294, - QUERY_TIMEOUT = 295, - READ_CONSISTENCY = 296, - WEAK = 297, - STRONG = 298, - FROZEN = 299, - PLACE_HOLDER = 300, - END_P = 301, - ERROR = 302, - WHEN = 303, - FLASHBACK = 304, - AUDIT = 305, - NOAUDIT = 306, - BEGI = 307, - START = 308, - TRANSACTION = 309, - READ = 310, - ONLY = 311, - WITH = 312, - CONSISTENT = 313, - SNAPSHOT = 314, - INDEX = 315, - XA = 316, - WARNINGS = 317, - ERRORS = 318, - TRACE = 319, - QUICK = 320, - COUNT = 321, - AS = 322, - WHERE = 323, - VALUES = 324, - ORDER = 325, - GROUP = 326, - HAVING = 327, - INTO = 328, - UNION = 329, - FOR = 330, - TX_READ_ONLY = 331, - AUTOCOMMIT_0 = 332, - SELECT_OBPROXY_ROUTE_ADDR = 333, - SET_OBPROXY_ROUTE_ADDR = 334, - NAME_OB_DOT = 335, - NAME_OB = 336, - EXPLAIN = 337, - DESC = 338, - DESCRIBE = 339, - NAME_STR = 340, - USE = 341, - HELP = 342, - SET_NAMES = 343, - SET_CHARSET = 344, - SET_PASSWORD = 345, - SET_DEFAULT = 346, - SET_OB_READ_CONSISTENCY = 347, - SET_TX_READ_ONLY = 348, - GLOBAL = 349, - SESSION = 350, - NUMBER_VAL = 351, - GROUP_ID = 352, - TABLE_ID = 353, - ELASTIC_ID = 354, - TESTLOAD = 355, - ODP_COMMENT = 356, - TNT_ID = 357, - DISASTER_STATUS = 358, - TRACE_ID = 359, - RPC_ID = 360, - DBP_COMMENT = 361, - ROUTE_TAG = 362, - SYS_TAG = 363, - TABLE_NAME = 364, - SCAN_ALL = 365, - PARALL = 366, - SHARD_KEY = 367, - INT_NUM = 368, - SHOW_PROXYNET = 369, - THREAD = 370, - CONNECTION = 371, - LIMIT = 372, - OFFSET = 373, - SHOW_PROCESSLIST = 374, - SHOW_PROXYSESSION = 375, - SHOW_GLOBALSESSION = 376, - ATTRIBUTE = 377, - VARIABLES = 378, - ALL = 379, - STAT = 380, - SHOW_PROXYCONFIG = 381, - DIFF = 382, - USER = 383, - LIKE = 384, - SHOW_PROXYSM = 385, - SHOW_PROXYCLUSTER = 386, - SHOW_PROXYRESOURCE = 387, - SHOW_PROXYCONGESTION = 388, - SHOW_PROXYROUTE = 389, - PARTITION = 390, - ROUTINE = 391, - SHOW_PROXYVIP = 392, - SHOW_PROXYMEMORY = 393, - OBJPOOL = 394, - SHOW_SQLAUDIT = 395, - SHOW_WARNLOG = 396, - SHOW_PROXYSTAT = 397, - REFRESH = 398, - SHOW_PROXYTRACE = 399, - SHOW_PROXYINFO = 400, - BINARY = 401, - UPGRADE = 402, - IDC = 403, - SHOW_TOPOLOGY = 404, - GROUP_NAME = 405, - SHOW_DB_VERSION = 406, - SHOW_DATABASES = 407, - SHOW_TABLES = 408, - SELECT_DATABASE = 409, - SHOW_CREATE_TABLE = 410, - ALTER_PROXYCONFIG = 411, - ALTER_PROXYRESOURCE = 412, - PING_PROXY = 413, - KILL_PROXYSESSION = 414, - KILL_GLOBALSESSION = 415, - KILL = 416, - QUERY = 417 + TABLE = 274, + UNIQUE = 275, + GRANT = 276, + REVOKE = 277, + ANALYZE = 278, + PURGE = 279, + COMMENT = 280, + FROM = 281, + DUAL = 282, + PREPARE = 283, + EXECUTE = 284, + USING = 285, + SELECT_HINT_BEGIN = 286, + UPDATE_HINT_BEGIN = 287, + DELETE_HINT_BEGIN = 288, + INSERT_HINT_BEGIN = 289, + REPLACE_HINT_BEGIN = 290, + MERGE_HINT_BEGIN = 291, + HINT_END = 292, + COMMENT_BEGIN = 293, + COMMENT_END = 294, + ROUTE_TABLE = 295, + ROUTE_PART_KEY = 296, + QUERY_TIMEOUT = 297, + READ_CONSISTENCY = 298, + WEAK = 299, + STRONG = 300, + FROZEN = 301, + PLACE_HOLDER = 302, + END_P = 303, + ERROR = 304, + WHEN = 305, + FLASHBACK = 306, + AUDIT = 307, + NOAUDIT = 308, + BEGI = 309, + START = 310, + TRANSACTION = 311, + READ = 312, + ONLY = 313, + WITH = 314, + CONSISTENT = 315, + SNAPSHOT = 316, + INDEX = 317, + XA = 318, + WARNINGS = 319, + ERRORS = 320, + TRACE = 321, + QUICK = 322, + COUNT = 323, + AS = 324, + WHERE = 325, + VALUES = 326, + ORDER = 327, + GROUP = 328, + HAVING = 329, + INTO = 330, + UNION = 331, + FOR = 332, + TX_READ_ONLY = 333, + AUTOCOMMIT_0 = 334, + SELECT_OBPROXY_ROUTE_ADDR = 335, + SET_OBPROXY_ROUTE_ADDR = 336, + NAME_OB_DOT = 337, + NAME_OB = 338, + EXPLAIN = 339, + DESC = 340, + DESCRIBE = 341, + NAME_STR = 342, + USE = 343, + HELP = 344, + SET_NAMES = 345, + SET_CHARSET = 346, + SET_PASSWORD = 347, + SET_DEFAULT = 348, + SET_OB_READ_CONSISTENCY = 349, + SET_TX_READ_ONLY = 350, + GLOBAL = 351, + SESSION = 352, + NUMBER_VAL = 353, + GROUP_ID = 354, + TABLE_ID = 355, + ELASTIC_ID = 356, + TESTLOAD = 357, + ODP_COMMENT = 358, + TNT_ID = 359, + DISASTER_STATUS = 360, + TRACE_ID = 361, + RPC_ID = 362, + DBP_COMMENT = 363, + ROUTE_TAG = 364, + SYS_TAG = 365, + TABLE_NAME = 366, + SCAN_ALL = 367, + PARALL = 368, + SHARD_KEY = 369, + INT_NUM = 370, + SHOW_PROXYNET = 371, + THREAD = 372, + CONNECTION = 373, + LIMIT = 374, + OFFSET = 375, + SHOW_PROCESSLIST = 376, + SHOW_PROXYSESSION = 377, + SHOW_GLOBALSESSION = 378, + ATTRIBUTE = 379, + VARIABLES = 380, + ALL = 381, + STAT = 382, + SHOW_PROXYCONFIG = 383, + DIFF = 384, + USER = 385, + LIKE = 386, + SHOW_PROXYSM = 387, + SHOW_PROXYCLUSTER = 388, + SHOW_PROXYRESOURCE = 389, + SHOW_PROXYCONGESTION = 390, + SHOW_PROXYROUTE = 391, + PARTITION = 392, + ROUTINE = 393, + SHOW_PROXYVIP = 394, + SHOW_PROXYMEMORY = 395, + OBJPOOL = 396, + SHOW_SQLAUDIT = 397, + SHOW_WARNLOG = 398, + SHOW_PROXYSTAT = 399, + REFRESH = 400, + SHOW_PROXYTRACE = 401, + SHOW_PROXYINFO = 402, + BINARY = 403, + UPGRADE = 404, + IDC = 405, + SHOW_TOPOLOGY = 406, + GROUP_NAME = 407, + SHOW_DB_VERSION = 408, + SHOW_DATABASES = 409, + SHOW_TABLES = 410, + SELECT_DATABASE = 411, + SHOW_CREATE_TABLE = 412, + SELECT_PROXY_VERSION = 413, + ALTER_PROXYCONFIG = 414, + ALTER_PROXYRESOURCE = 415, + PING_PROXY = 416, + KILL_PROXYSESSION = 417, + KILL_GLOBALSESSION = 418, + KILL = 419, + QUERY = 420 }; #endif diff --git a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.h b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.h index d1f33dc3323ad52d85fb1564965ede7ab82a192f..17fecf15347cdf695f73fca4ca2d361c96a2ad82 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.h +++ b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_lex.h @@ -362,7 +362,7 @@ extern int ob_proxy_parser_utf8_yylex \ #undef YY_DECL #endif -#line 1146 "ob_proxy_parser_utf8.l" +#line 1151 "ob_proxy_parser_utf8.l" #line 369 "ob_proxy_parser_utf8_lex.h" diff --git a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.c b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.c index d900685d21dcc698d5d85e468cd0c7956020f688..0c60398f75312da01031df667017ae27d3a481a2 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.c +++ b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.c @@ -159,7 +159,6 @@ do {\ if ((OBPROXY_T_INVALID < result->cur_stmt_type_ && result->cur_stmt_type_ < OBPROXY_T_ICMD_MAX) || (OBPROXY_T_PING_PROXY == result->cur_stmt_type_)) {\ result->cmd_info_.err_type_ = OBPROXY_T_ERR_PARSE;\ }\ - result->sub_stmt_type_ = OBPROXY_T_SUB_INVALID;\ handle_stmt_end(result);\ HANDLE_ACCEPT();\ } while (0); @@ -398,150 +397,153 @@ extern int ob_proxy_parser_utf8_yydebug; ALTER = 271, TRUNCATE = 272, RENAME = 273, - GRANT = 274, - REVOKE = 275, - ANALYZE = 276, - PURGE = 277, - COMMENT = 278, - FROM = 279, - DUAL = 280, - PREPARE = 281, - EXECUTE = 282, - USING = 283, - SELECT_HINT_BEGIN = 284, - UPDATE_HINT_BEGIN = 285, - DELETE_HINT_BEGIN = 286, - INSERT_HINT_BEGIN = 287, - REPLACE_HINT_BEGIN = 288, - MERGE_HINT_BEGIN = 289, - HINT_END = 290, - COMMENT_BEGIN = 291, - COMMENT_END = 292, - ROUTE_TABLE = 293, - ROUTE_PART_KEY = 294, - QUERY_TIMEOUT = 295, - READ_CONSISTENCY = 296, - WEAK = 297, - STRONG = 298, - FROZEN = 299, - PLACE_HOLDER = 300, - END_P = 301, - ERROR = 302, - WHEN = 303, - FLASHBACK = 304, - AUDIT = 305, - NOAUDIT = 306, - BEGI = 307, - START = 308, - TRANSACTION = 309, - READ = 310, - ONLY = 311, - WITH = 312, - CONSISTENT = 313, - SNAPSHOT = 314, - INDEX = 315, - XA = 316, - WARNINGS = 317, - ERRORS = 318, - TRACE = 319, - QUICK = 320, - COUNT = 321, - AS = 322, - WHERE = 323, - VALUES = 324, - ORDER = 325, - GROUP = 326, - HAVING = 327, - INTO = 328, - UNION = 329, - FOR = 330, - TX_READ_ONLY = 331, - AUTOCOMMIT_0 = 332, - SELECT_OBPROXY_ROUTE_ADDR = 333, - SET_OBPROXY_ROUTE_ADDR = 334, - NAME_OB_DOT = 335, - NAME_OB = 336, - EXPLAIN = 337, - DESC = 338, - DESCRIBE = 339, - NAME_STR = 340, - USE = 341, - HELP = 342, - SET_NAMES = 343, - SET_CHARSET = 344, - SET_PASSWORD = 345, - SET_DEFAULT = 346, - SET_OB_READ_CONSISTENCY = 347, - SET_TX_READ_ONLY = 348, - GLOBAL = 349, - SESSION = 350, - NUMBER_VAL = 351, - GROUP_ID = 352, - TABLE_ID = 353, - ELASTIC_ID = 354, - TESTLOAD = 355, - ODP_COMMENT = 356, - TNT_ID = 357, - DISASTER_STATUS = 358, - TRACE_ID = 359, - RPC_ID = 360, - DBP_COMMENT = 361, - ROUTE_TAG = 362, - SYS_TAG = 363, - TABLE_NAME = 364, - SCAN_ALL = 365, - PARALL = 366, - SHARD_KEY = 367, - INT_NUM = 368, - SHOW_PROXYNET = 369, - THREAD = 370, - CONNECTION = 371, - LIMIT = 372, - OFFSET = 373, - SHOW_PROCESSLIST = 374, - SHOW_PROXYSESSION = 375, - SHOW_GLOBALSESSION = 376, - ATTRIBUTE = 377, - VARIABLES = 378, - ALL = 379, - STAT = 380, - SHOW_PROXYCONFIG = 381, - DIFF = 382, - USER = 383, - LIKE = 384, - SHOW_PROXYSM = 385, - SHOW_PROXYCLUSTER = 386, - SHOW_PROXYRESOURCE = 387, - SHOW_PROXYCONGESTION = 388, - SHOW_PROXYROUTE = 389, - PARTITION = 390, - ROUTINE = 391, - SHOW_PROXYVIP = 392, - SHOW_PROXYMEMORY = 393, - OBJPOOL = 394, - SHOW_SQLAUDIT = 395, - SHOW_WARNLOG = 396, - SHOW_PROXYSTAT = 397, - REFRESH = 398, - SHOW_PROXYTRACE = 399, - SHOW_PROXYINFO = 400, - BINARY = 401, - UPGRADE = 402, - IDC = 403, - SHOW_TOPOLOGY = 404, - GROUP_NAME = 405, - SHOW_DB_VERSION = 406, - SHOW_DATABASES = 407, - SHOW_TABLES = 408, - SELECT_DATABASE = 409, - SHOW_CREATE_TABLE = 410, - ALTER_PROXYCONFIG = 411, - ALTER_PROXYRESOURCE = 412, - PING_PROXY = 413, - KILL_PROXYSESSION = 414, - KILL_GLOBALSESSION = 415, - KILL = 416, - QUERY = 417 + TABLE = 274, + UNIQUE = 275, + GRANT = 276, + REVOKE = 277, + ANALYZE = 278, + PURGE = 279, + COMMENT = 280, + FROM = 281, + DUAL = 282, + PREPARE = 283, + EXECUTE = 284, + USING = 285, + SELECT_HINT_BEGIN = 286, + UPDATE_HINT_BEGIN = 287, + DELETE_HINT_BEGIN = 288, + INSERT_HINT_BEGIN = 289, + REPLACE_HINT_BEGIN = 290, + MERGE_HINT_BEGIN = 291, + HINT_END = 292, + COMMENT_BEGIN = 293, + COMMENT_END = 294, + ROUTE_TABLE = 295, + ROUTE_PART_KEY = 296, + QUERY_TIMEOUT = 297, + READ_CONSISTENCY = 298, + WEAK = 299, + STRONG = 300, + FROZEN = 301, + PLACE_HOLDER = 302, + END_P = 303, + ERROR = 304, + WHEN = 305, + FLASHBACK = 306, + AUDIT = 307, + NOAUDIT = 308, + BEGI = 309, + START = 310, + TRANSACTION = 311, + READ = 312, + ONLY = 313, + WITH = 314, + CONSISTENT = 315, + SNAPSHOT = 316, + INDEX = 317, + XA = 318, + WARNINGS = 319, + ERRORS = 320, + TRACE = 321, + QUICK = 322, + COUNT = 323, + AS = 324, + WHERE = 325, + VALUES = 326, + ORDER = 327, + GROUP = 328, + HAVING = 329, + INTO = 330, + UNION = 331, + FOR = 332, + TX_READ_ONLY = 333, + AUTOCOMMIT_0 = 334, + SELECT_OBPROXY_ROUTE_ADDR = 335, + SET_OBPROXY_ROUTE_ADDR = 336, + NAME_OB_DOT = 337, + NAME_OB = 338, + EXPLAIN = 339, + DESC = 340, + DESCRIBE = 341, + NAME_STR = 342, + USE = 343, + HELP = 344, + SET_NAMES = 345, + SET_CHARSET = 346, + SET_PASSWORD = 347, + SET_DEFAULT = 348, + SET_OB_READ_CONSISTENCY = 349, + SET_TX_READ_ONLY = 350, + GLOBAL = 351, + SESSION = 352, + NUMBER_VAL = 353, + GROUP_ID = 354, + TABLE_ID = 355, + ELASTIC_ID = 356, + TESTLOAD = 357, + ODP_COMMENT = 358, + TNT_ID = 359, + DISASTER_STATUS = 360, + TRACE_ID = 361, + RPC_ID = 362, + DBP_COMMENT = 363, + ROUTE_TAG = 364, + SYS_TAG = 365, + TABLE_NAME = 366, + SCAN_ALL = 367, + PARALL = 368, + SHARD_KEY = 369, + INT_NUM = 370, + SHOW_PROXYNET = 371, + THREAD = 372, + CONNECTION = 373, + LIMIT = 374, + OFFSET = 375, + SHOW_PROCESSLIST = 376, + SHOW_PROXYSESSION = 377, + SHOW_GLOBALSESSION = 378, + ATTRIBUTE = 379, + VARIABLES = 380, + ALL = 381, + STAT = 382, + SHOW_PROXYCONFIG = 383, + DIFF = 384, + USER = 385, + LIKE = 386, + SHOW_PROXYSM = 387, + SHOW_PROXYCLUSTER = 388, + SHOW_PROXYRESOURCE = 389, + SHOW_PROXYCONGESTION = 390, + SHOW_PROXYROUTE = 391, + PARTITION = 392, + ROUTINE = 393, + SHOW_PROXYVIP = 394, + SHOW_PROXYMEMORY = 395, + OBJPOOL = 396, + SHOW_SQLAUDIT = 397, + SHOW_WARNLOG = 398, + SHOW_PROXYSTAT = 399, + REFRESH = 400, + SHOW_PROXYTRACE = 401, + SHOW_PROXYINFO = 402, + BINARY = 403, + UPGRADE = 404, + IDC = 405, + SHOW_TOPOLOGY = 406, + GROUP_NAME = 407, + SHOW_DB_VERSION = 408, + SHOW_DATABASES = 409, + SHOW_TABLES = 410, + SELECT_DATABASE = 411, + SHOW_CREATE_TABLE = 412, + SELECT_PROXY_VERSION = 413, + ALTER_PROXYCONFIG = 414, + ALTER_PROXYRESOURCE = 415, + PING_PROXY = 416, + KILL_PROXYSESSION = 417, + KILL_GLOBALSESSION = 418, + KILL = 419, + QUERY = 420 }; #endif @@ -804,22 +806,22 @@ union yyalloc #endif /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 298 +#define YYFINAL 305 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 1404 +#define YYLAST 1423 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 174 +#define YYNTOKENS 177 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 126 +#define YYNNTS 128 /* YYNRULES -- Number of rules. */ -#define YYNRULES 420 +#define YYNRULES 427 /* YYNRULES -- Number of states. */ -#define YYNSTATES 672 +#define YYNSTATES 681 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 417 +#define YYMAXUTOK 420 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -830,16 +832,16 @@ static const yytype_uint8 yytranslate[] = 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 172, 2, 2, 2, 2, - 167, 168, 173, 2, 165, 2, 169, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 163, - 2, 166, 2, 2, 164, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 175, 2, 2, 2, 2, + 170, 171, 176, 2, 168, 2, 172, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 166, + 2, 169, 2, 2, 167, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 170, 2, 171, 2, 2, 2, 2, + 2, 2, 2, 173, 2, 174, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -868,7 +870,8 @@ static const yytype_uint8 yytranslate[] = 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162 + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 165 }; #if YYDEBUG @@ -881,221 +884,223 @@ static const yytype_uint16 yyprhs[] = 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 85, 88, 91, 94, 97, 100, 103, 106, 109, 112, 115, 118, - 121, 123, 125, 127, 129, 131, 133, 135, 137, 139, - 141, 143, 145, 147, 149, 151, 154, 159, 163, 166, - 169, 174, 176, 178, 180, 182, 184, 186, 188, 190, - 192, 195, 197, 199, 201, 202, 205, 206, 208, 211, - 217, 221, 224, 228, 230, 232, 234, 236, 238, 240, - 242, 244, 246, 248, 251, 254, 256, 258, 262, 268, - 276, 278, 282, 284, 286, 288, 290, 292, 294, 300, - 302, 306, 312, 313, 315, 319, 321, 323, 325, 328, - 332, 334, 336, 339, 341, 344, 348, 352, 354, 356, - 358, 359, 363, 365, 369, 373, 379, 382, 385, 390, - 393, 396, 400, 402, 407, 414, 419, 425, 432, 437, - 441, 443, 445, 447, 449, 452, 456, 462, 469, 476, - 483, 490, 497, 505, 512, 519, 526, 533, 542, 551, - 552, 555, 558, 561, 563, 567, 569, 574, 579, 583, - 590, 595, 600, 607, 611, 613, 617, 618, 622, 626, - 630, 634, 638, 642, 646, 650, 654, 658, 664, 668, - 669, 671, 672, 674, 676, 678, 680, 683, 685, 688, - 690, 693, 696, 700, 701, 703, 706, 708, 711, 713, - 716, 719, 720, 723, 728, 730, 735, 741, 743, 748, - 753, 759, 760, 762, 764, 766, 767, 769, 773, 777, - 780, 782, 784, 786, 788, 790, 792, 794, 796, 798, - 800, 802, 804, 806, 808, 810, 812, 814, 816, 818, - 820, 822, 824, 826, 827, 830, 835, 840, 841, 844, - 845, 848, 851, 853, 855, 859, 862, 866, 871, 873, - 876, 877, 880, 884, 887, 890, 893, 894, 897, 901, - 904, 908, 911, 915, 919, 924, 926, 929, 932, 936, - 939, 942, 943, 945, 947, 950, 953, 957, 960, 962, - 965, 967, 970, 973, 976, 979, 980, 982, 986, 992, - 995, 999, 1002, 1003, 1005, 1008, 1011, 1014, 1017, 1022, - 1028, 1034, 1038, 1040, 1043, 1047, 1051, 1054, 1057, 1061, - 1065, 1066, 1069, 1071, 1075, 1079, 1083, 1084, 1086, 1088, - 1092, 1095, 1099, 1102, 1105, 1107, 1108, 1111, 1116, 1119, - 1121, 1125, 1128, 1133, 1137, 1143, 1145, 1147, 1149, 1151, - 1153, 1155, 1157, 1159, 1161, 1163, 1165, 1167, 1169, 1171, - 1173, 1175, 1177, 1179, 1181, 1183, 1185, 1187, 1189, 1191, - 1193, 1195, 1197, 1199, 1201, 1203, 1205, 1207, 1209, 1211, - 1213, 1215, 1217, 1219, 1221, 1223, 1225, 1227, 1229, 1231, - 1233 + 121, 123, 125, 128, 130, 132, 134, 136, 137, 139, + 141, 144, 146, 148, 150, 152, 154, 156, 158, 160, + 163, 168, 172, 175, 178, 183, 185, 187, 189, 191, + 193, 195, 197, 199, 201, 204, 206, 208, 210, 211, + 214, 215, 217, 220, 226, 230, 232, 236, 239, 243, + 245, 247, 249, 251, 253, 255, 257, 259, 261, 263, + 265, 268, 271, 273, 275, 279, 285, 293, 295, 299, + 301, 303, 305, 307, 309, 311, 317, 319, 323, 329, + 330, 332, 336, 338, 340, 342, 345, 349, 351, 353, + 356, 358, 361, 365, 369, 371, 373, 375, 376, 380, + 382, 386, 390, 396, 399, 402, 407, 410, 413, 417, + 419, 424, 431, 436, 442, 449, 454, 458, 460, 462, + 464, 466, 469, 473, 479, 486, 493, 500, 507, 514, + 522, 529, 536, 543, 550, 559, 568, 569, 572, 575, + 578, 580, 584, 586, 591, 596, 600, 607, 612, 617, + 624, 628, 630, 634, 635, 639, 643, 647, 651, 655, + 659, 663, 667, 671, 675, 681, 685, 686, 688, 689, + 691, 693, 695, 697, 700, 702, 705, 707, 710, 713, + 717, 718, 720, 723, 725, 728, 730, 733, 736, 737, + 740, 745, 747, 752, 758, 760, 765, 770, 776, 777, + 779, 781, 783, 784, 786, 790, 794, 797, 799, 801, + 803, 805, 807, 809, 811, 813, 815, 817, 819, 821, + 823, 825, 827, 829, 831, 833, 835, 837, 839, 841, + 843, 844, 847, 852, 857, 858, 861, 862, 865, 868, + 870, 872, 876, 879, 883, 888, 890, 893, 894, 897, + 901, 904, 907, 910, 911, 914, 918, 921, 925, 928, + 932, 936, 941, 943, 946, 949, 953, 956, 959, 960, + 962, 964, 967, 970, 974, 977, 979, 982, 984, 987, + 990, 993, 996, 997, 999, 1003, 1009, 1012, 1016, 1019, + 1020, 1022, 1025, 1028, 1031, 1034, 1039, 1045, 1051, 1055, + 1057, 1060, 1064, 1068, 1071, 1074, 1078, 1082, 1083, 1086, + 1088, 1092, 1096, 1100, 1101, 1103, 1105, 1109, 1112, 1116, + 1119, 1122, 1124, 1125, 1128, 1133, 1136, 1138, 1142, 1145, + 1150, 1154, 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, + 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, 1192, 1194, + 1196, 1198, 1200, 1202, 1204, 1206, 1208, 1210, 1212, 1214, + 1216, 1218, 1220, 1222, 1224, 1226, 1228, 1230, 1232, 1234, + 1236, 1238, 1240, 1242, 1244, 1246, 1248, 1250 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { - 175, 0, -1, 176, -1, 1, -1, 177, -1, 176, - 177, -1, 178, 46, -1, 178, 163, -1, 178, 163, - 46, -1, 163, -1, 163, 46, -1, 52, 178, 163, - -1, 179, -1, 218, -1, 223, -1, 219, -1, 220, - -1, 221, -1, 180, -1, 181, -1, 288, -1, 253, - -1, 195, -1, 254, -1, 292, -1, 293, -1, 201, - -1, 202, -1, 203, -1, 204, -1, 205, -1, 206, - -1, 207, -1, 182, -1, 187, -1, 222, -1, 294, - -1, 241, 192, 191, -1, 189, 179, -1, 189, 218, - -1, 189, 220, -1, 189, 221, -1, 189, 219, -1, - 189, 222, -1, 227, 179, -1, 227, 218, -1, 227, - 220, -1, 227, 221, -1, 227, 219, -1, 227, 222, - -1, 183, -1, 188, -1, 14, -1, 15, -1, 16, - -1, 17, -1, 18, -1, 179, -1, 218, -1, 219, - -1, 221, -1, 220, -1, 294, -1, 207, -1, 222, - -1, 164, 81, -1, 185, 165, 164, 81, -1, 26, - 299, 24, -1, 186, 184, -1, 27, 299, -1, 27, - 299, 28, 185, -1, 19, -1, 20, -1, 21, -1, - 22, -1, 49, -1, 23, -1, 50, -1, 51, -1, - 190, -1, 190, 81, -1, 82, -1, 83, -1, 84, - -1, -1, 24, 214, -1, -1, 211, -1, 5, 76, - -1, 5, 76, 192, 24, 214, -1, 5, 76, 211, - -1, 12, 77, -1, 12, 77, 211, -1, 193, -1, - 194, -1, 199, -1, 200, -1, 196, -1, 198, -1, - 197, -1, 154, -1, 152, -1, 153, -1, 155, 208, - -1, 190, 208, -1, 151, -1, 149, -1, 149, 24, - 81, -1, 149, 68, 150, 166, 81, -1, 149, 24, - 81, 68, 150, 166, 81, -1, 78, -1, 79, 166, - 113, -1, 88, -1, 89, -1, 90, -1, 91, -1, - 92, -1, 93, -1, 13, 208, 167, 209, 168, -1, - 299, -1, 299, 169, 299, -1, 299, 169, 299, 169, - 299, -1, -1, 210, -1, 209, 165, 210, -1, 81, - -1, 113, -1, 96, -1, 164, 81, -1, 164, 164, - 81, -1, 45, -1, 212, -1, 211, 212, -1, 213, - -1, 167, 168, -1, 167, 179, 168, -1, 167, 211, - 168, -1, 296, -1, 215, -1, 179, -1, -1, 167, - 217, 168, -1, 81, -1, 217, 165, 81, -1, 244, - 297, 295, -1, 244, 297, 295, 216, 215, -1, 246, - 214, -1, 242, 214, -1, 243, 252, 24, 214, -1, - 247, 297, -1, 12, 224, -1, 225, 165, 224, -1, - 225, -1, 164, 81, 166, 226, -1, 164, 164, 94, - 81, 166, 226, -1, 94, 81, 166, 226, -1, 164, - 164, 81, 166, 226, -1, 164, 164, 95, 81, 166, - 226, -1, 95, 81, 166, 226, -1, 81, 166, 226, - -1, 81, -1, 113, -1, 96, -1, 228, -1, 228, - 227, -1, 36, 229, 37, -1, 36, 101, 237, 236, - 37, -1, 36, 98, 166, 240, 236, 37, -1, 36, - 109, 166, 240, 236, 37, -1, 36, 97, 166, 240, - 236, 37, -1, 36, 99, 166, 240, 236, 37, -1, - 36, 100, 166, 240, 236, 37, -1, 36, 80, 81, - 166, 239, 236, 37, -1, 36, 104, 166, 238, 236, - 37, -1, 36, 105, 166, 238, 236, 37, -1, 36, - 102, 166, 240, 236, 37, -1, 36, 103, 166, 240, - 236, 37, -1, 36, 106, 107, 166, 170, 231, 171, - 37, -1, 36, 106, 108, 166, 170, 233, 171, 37, - -1, -1, 229, 230, -1, 38, 81, -1, 39, 81, - -1, 81, -1, 232, 165, 231, -1, 232, -1, 97, - 167, 240, 168, -1, 109, 167, 240, 168, -1, 110, - 167, 168, -1, 110, 167, 111, 166, 240, 168, -1, - 112, 167, 234, 168, -1, 64, 167, 238, 168, -1, - 64, 167, 238, 172, 238, 168, -1, 235, 165, 234, - -1, 235, -1, 81, 166, 240, -1, -1, 236, 165, - 237, -1, 97, 166, 240, -1, 98, 166, 240, -1, - 109, 166, 240, -1, 99, 166, 240, -1, 100, 166, - 240, -1, 104, 166, 238, -1, 105, 166, 238, -1, - 102, 166, 240, -1, 103, 166, 240, -1, 81, 169, - 81, 166, 239, -1, 81, 166, 239, -1, -1, 240, - -1, -1, 240, -1, 81, -1, 85, -1, 5, -1, - 29, 248, -1, 8, -1, 30, 248, -1, 6, -1, - 31, 248, -1, 7, 245, -1, 32, 248, 245, -1, - -1, 124, -1, 124, 48, -1, 9, -1, 33, 248, - -1, 10, -1, 34, 248, -1, 249, 35, -1, -1, - 250, 249, -1, 40, 167, 113, 168, -1, 113, -1, - 41, 167, 251, 168, -1, 60, 167, 81, 81, 168, - -1, 81, -1, 81, 167, 113, 168, -1, 81, 167, - 81, 168, -1, 81, 167, 81, 81, 168, -1, -1, - 42, -1, 43, -1, 44, -1, -1, 65, -1, 11, - 287, 62, -1, 11, 287, 63, -1, 11, 64, -1, - 258, -1, 260, -1, 261, -1, 264, -1, 262, -1, - 266, -1, 267, -1, 268, -1, 269, -1, 271, -1, - 272, -1, 273, -1, 274, -1, 275, -1, 277, -1, - 278, -1, 280, -1, 281, -1, 282, -1, 283, -1, - 284, -1, 285, -1, 286, -1, -1, 117, 113, -1, - 117, 113, 165, 113, -1, 117, 113, 118, 113, -1, - -1, 129, 81, -1, -1, 129, 81, -1, 114, 259, - -1, 115, -1, 116, -1, 116, 113, 255, -1, 126, - 256, -1, 126, 127, 256, -1, 126, 127, 128, 256, - -1, 119, -1, 121, 263, -1, -1, 122, 81, -1, - 122, 129, 81, -1, 122, 124, -1, 129, 81, -1, - 120, 265, -1, -1, 122, 256, -1, 122, 113, 256, - -1, 125, 256, -1, 125, 113, 256, -1, 123, 256, - -1, 123, 113, 256, -1, 123, 124, 256, -1, 123, - 124, 113, 256, -1, 130, -1, 130, 113, -1, 131, - 256, -1, 131, 148, 256, -1, 132, 256, -1, 133, - 270, -1, -1, 81, -1, 124, -1, 124, 81, -1, - 134, 257, -1, 134, 136, 257, -1, 134, 135, -1, - 137, -1, 137, 81, -1, 138, -1, 138, 139, -1, - 140, 255, -1, 140, 113, -1, 141, 276, -1, -1, - 113, -1, 113, 165, 113, -1, 113, 165, 113, 165, - 81, -1, 142, 256, -1, 142, 143, 256, -1, 144, - 279, -1, -1, 113, -1, 113, 113, -1, 145, 146, - -1, 145, 147, -1, 145, 148, -1, 156, 12, 81, - 166, -1, 156, 12, 81, 166, 81, -1, 156, 12, - 81, 166, 113, -1, 157, 6, 81, -1, 158, -1, - 159, 113, -1, 159, 113, 113, -1, 160, 81, 113, - -1, 160, 81, -1, 161, 113, -1, 161, 116, 113, - -1, 161, 162, 113, -1, -1, 66, 173, -1, 52, - -1, 53, 54, 289, -1, 61, 52, 81, -1, 61, - 53, 81, -1, -1, 290, -1, 291, -1, 290, 165, - 291, -1, 55, 56, -1, 57, 58, 59, -1, 86, - 81, -1, 87, 81, -1, 81, -1, -1, 135, 81, - -1, 135, 167, 81, 168, -1, 297, 295, -1, 299, - -1, 299, 169, 299, -1, 299, 299, -1, 299, 169, - 299, 299, -1, 299, 67, 299, -1, 299, 169, 299, - 67, 299, -1, 53, -1, 61, -1, 52, -1, 54, - -1, 58, -1, 63, -1, 62, -1, 66, -1, 65, - -1, 64, -1, 115, -1, 116, -1, 118, -1, 122, - -1, 123, -1, 125, -1, 127, -1, 128, -1, 139, - -1, 143, -1, 147, -1, 148, -1, 162, -1, 97, - -1, 98, -1, 99, -1, 100, -1, 150, -1, 101, - -1, 102, -1, 103, -1, 104, -1, 105, -1, 106, - -1, 107, -1, 108, -1, 109, -1, 111, -1, 110, - -1, 112, -1, 60, -1, 49, -1, 50, -1, 51, - -1, 81, -1, 298, -1 + 178, 0, -1, 179, -1, 1, -1, 180, -1, 179, + 180, -1, 181, 48, -1, 181, 166, -1, 181, 166, + 48, -1, 166, -1, 166, 48, -1, 54, 181, 166, + -1, 182, -1, 223, -1, 228, -1, 224, -1, 225, + -1, 226, -1, 183, -1, 184, -1, 293, -1, 258, + -1, 200, -1, 259, -1, 297, -1, 298, -1, 206, + -1, 207, -1, 208, -1, 209, -1, 210, -1, 211, + -1, 212, -1, 185, -1, 191, -1, 227, -1, 299, + -1, 246, 196, 195, -1, 193, 182, -1, 193, 223, + -1, 193, 225, -1, 193, 226, -1, 193, 224, -1, + 193, 227, -1, 232, 182, -1, 232, 223, -1, 232, + 225, -1, 232, 226, -1, 232, 224, -1, 232, 227, + -1, 186, -1, 192, -1, 14, 187, -1, 15, -1, + 16, -1, 17, -1, 18, -1, -1, 19, -1, 62, + -1, 20, 62, -1, 182, -1, 223, -1, 224, -1, + 226, -1, 225, -1, 299, -1, 212, -1, 227, -1, + 167, 83, -1, 189, 168, 167, 83, -1, 28, 304, + 26, -1, 190, 188, -1, 29, 304, -1, 29, 304, + 30, 189, -1, 21, -1, 22, -1, 23, -1, 24, + -1, 51, -1, 25, -1, 52, -1, 53, -1, 194, + -1, 194, 83, -1, 84, -1, 85, -1, 86, -1, + -1, 26, 219, -1, -1, 216, -1, 5, 78, -1, + 5, 78, 196, 26, 219, -1, 5, 78, 216, -1, + 158, -1, 158, 69, 304, -1, 12, 79, -1, 12, + 79, 216, -1, 197, -1, 198, -1, 199, -1, 204, + -1, 205, -1, 201, -1, 203, -1, 202, -1, 156, + -1, 154, -1, 155, -1, 157, 213, -1, 194, 213, + -1, 153, -1, 151, -1, 151, 26, 83, -1, 151, + 70, 152, 169, 83, -1, 151, 26, 83, 70, 152, + 169, 83, -1, 80, -1, 81, 169, 115, -1, 90, + -1, 91, -1, 92, -1, 93, -1, 94, -1, 95, + -1, 13, 213, 170, 214, 171, -1, 304, -1, 304, + 172, 304, -1, 304, 172, 304, 172, 304, -1, -1, + 215, -1, 214, 168, 215, -1, 83, -1, 115, -1, + 98, -1, 167, 83, -1, 167, 167, 83, -1, 47, + -1, 217, -1, 216, 217, -1, 218, -1, 170, 171, + -1, 170, 182, 171, -1, 170, 216, 171, -1, 301, + -1, 220, -1, 182, -1, -1, 170, 222, 171, -1, + 83, -1, 222, 168, 83, -1, 249, 302, 300, -1, + 249, 302, 300, 221, 220, -1, 251, 219, -1, 247, + 219, -1, 248, 257, 26, 219, -1, 252, 302, -1, + 12, 229, -1, 230, 168, 229, -1, 230, -1, 167, + 83, 169, 231, -1, 167, 167, 96, 83, 169, 231, + -1, 96, 83, 169, 231, -1, 167, 167, 83, 169, + 231, -1, 167, 167, 97, 83, 169, 231, -1, 97, + 83, 169, 231, -1, 83, 169, 231, -1, 83, -1, + 115, -1, 98, -1, 233, -1, 233, 232, -1, 38, + 234, 39, -1, 38, 103, 242, 241, 39, -1, 38, + 100, 169, 245, 241, 39, -1, 38, 111, 169, 245, + 241, 39, -1, 38, 99, 169, 245, 241, 39, -1, + 38, 101, 169, 245, 241, 39, -1, 38, 102, 169, + 245, 241, 39, -1, 38, 82, 83, 169, 244, 241, + 39, -1, 38, 106, 169, 243, 241, 39, -1, 38, + 107, 169, 243, 241, 39, -1, 38, 104, 169, 245, + 241, 39, -1, 38, 105, 169, 245, 241, 39, -1, + 38, 108, 109, 169, 173, 236, 174, 39, -1, 38, + 108, 110, 169, 173, 238, 174, 39, -1, -1, 234, + 235, -1, 40, 83, -1, 41, 83, -1, 83, -1, + 237, 168, 236, -1, 237, -1, 99, 170, 245, 171, + -1, 111, 170, 245, 171, -1, 112, 170, 171, -1, + 112, 170, 113, 169, 245, 171, -1, 114, 170, 239, + 171, -1, 66, 170, 243, 171, -1, 66, 170, 243, + 175, 243, 171, -1, 240, 168, 239, -1, 240, -1, + 83, 169, 245, -1, -1, 241, 168, 242, -1, 99, + 169, 245, -1, 100, 169, 245, -1, 111, 169, 245, + -1, 101, 169, 245, -1, 102, 169, 245, -1, 106, + 169, 243, -1, 107, 169, 243, -1, 104, 169, 245, + -1, 105, 169, 245, -1, 83, 172, 83, 169, 244, + -1, 83, 169, 244, -1, -1, 245, -1, -1, 245, + -1, 83, -1, 87, -1, 5, -1, 31, 253, -1, + 8, -1, 32, 253, -1, 6, -1, 33, 253, -1, + 7, 250, -1, 34, 253, 250, -1, -1, 126, -1, + 126, 50, -1, 9, -1, 35, 253, -1, 10, -1, + 36, 253, -1, 254, 37, -1, -1, 255, 254, -1, + 42, 170, 115, 171, -1, 115, -1, 43, 170, 256, + 171, -1, 62, 170, 83, 83, 171, -1, 83, -1, + 83, 170, 115, 171, -1, 83, 170, 83, 171, -1, + 83, 170, 83, 83, 171, -1, -1, 44, -1, 45, + -1, 46, -1, -1, 67, -1, 11, 292, 64, -1, + 11, 292, 65, -1, 11, 66, -1, 263, -1, 265, + -1, 266, -1, 269, -1, 267, -1, 271, -1, 272, + -1, 273, -1, 274, -1, 276, -1, 277, -1, 278, + -1, 279, -1, 280, -1, 282, -1, 283, -1, 285, + -1, 286, -1, 287, -1, 288, -1, 289, -1, 290, + -1, 291, -1, -1, 119, 115, -1, 119, 115, 168, + 115, -1, 119, 115, 120, 115, -1, -1, 131, 83, + -1, -1, 131, 83, -1, 116, 264, -1, 117, -1, + 118, -1, 118, 115, 260, -1, 128, 261, -1, 128, + 129, 261, -1, 128, 129, 130, 261, -1, 121, -1, + 123, 268, -1, -1, 124, 83, -1, 124, 131, 83, + -1, 124, 126, -1, 131, 83, -1, 122, 270, -1, + -1, 124, 261, -1, 124, 115, 261, -1, 127, 261, + -1, 127, 115, 261, -1, 125, 261, -1, 125, 115, + 261, -1, 125, 126, 261, -1, 125, 126, 115, 261, + -1, 132, -1, 132, 115, -1, 133, 261, -1, 133, + 150, 261, -1, 134, 261, -1, 135, 275, -1, -1, + 83, -1, 126, -1, 126, 83, -1, 136, 262, -1, + 136, 138, 262, -1, 136, 137, -1, 139, -1, 139, + 83, -1, 140, -1, 140, 141, -1, 142, 260, -1, + 142, 115, -1, 143, 281, -1, -1, 115, -1, 115, + 168, 115, -1, 115, 168, 115, 168, 83, -1, 144, + 261, -1, 144, 145, 261, -1, 146, 284, -1, -1, + 115, -1, 115, 115, -1, 147, 148, -1, 147, 149, + -1, 147, 150, -1, 159, 12, 83, 169, -1, 159, + 12, 83, 169, 83, -1, 159, 12, 83, 169, 115, + -1, 160, 6, 83, -1, 161, -1, 162, 115, -1, + 162, 115, 115, -1, 163, 83, 115, -1, 163, 83, + -1, 164, 115, -1, 164, 118, 115, -1, 164, 165, + 115, -1, -1, 68, 176, -1, 54, -1, 55, 56, + 294, -1, 63, 54, 83, -1, 63, 55, 83, -1, + -1, 295, -1, 296, -1, 295, 168, 296, -1, 57, + 58, -1, 59, 60, 61, -1, 88, 83, -1, 89, + 83, -1, 83, -1, -1, 137, 83, -1, 137, 170, + 83, 171, -1, 302, 300, -1, 304, -1, 304, 172, + 304, -1, 304, 304, -1, 304, 172, 304, 304, -1, + 304, 69, 304, -1, 304, 172, 304, 69, 304, -1, + 55, -1, 63, -1, 54, -1, 56, -1, 60, -1, + 65, -1, 64, -1, 68, -1, 67, -1, 66, -1, + 117, -1, 118, -1, 120, -1, 124, -1, 125, -1, + 127, -1, 129, -1, 130, -1, 141, -1, 145, -1, + 149, -1, 150, -1, 165, -1, 99, -1, 100, -1, + 101, -1, 102, -1, 152, -1, 103, -1, 104, -1, + 105, -1, 106, -1, 107, -1, 108, -1, 109, -1, + 110, -1, 111, -1, 113, -1, 112, -1, 114, -1, + 62, -1, 51, -1, 52, -1, 53, -1, 83, -1, + 303, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 337, 337, 338, 340, 341, 343, 344, 345, 346, - 347, 348, 350, 351, 352, 353, 354, 355, 356, 357, - 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, - 368, 369, 370, 371, 372, 373, 374, 376, 378, 379, - 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, - 392, 393, 395, 396, 397, 398, 399, 401, 402, 403, - 404, 405, 406, 407, 408, 410, 417, 425, 431, 434, - 439, 445, 446, 447, 448, 449, 450, 451, 452, 454, - 455, 457, 458, 459, 461, 462, 464, 465, 467, 468, - 469, 471, 472, 474, 475, 476, 477, 478, 480, 481, - 482, 483, 484, 485, 486, 492, 494, 495, 500, 505, - 512, 515, 521, 522, 523, 524, 525, 526, 529, 531, - 535, 540, 548, 551, 556, 561, 566, 571, 576, 581, - 586, 594, 595, 597, 599, 600, 601, 603, 604, 606, - 608, 609, 611, 612, 614, 618, 619, 620, 621, 622, - 627, 629, 630, 632, 636, 640, 644, 648, 652, 656, - 660, 665, 670, 676, 677, 679, 680, 681, 682, 683, - 684, 685, 686, 692, 693, 694, 695, 696, 697, 699, - 700, 702, 703, 704, 706, 707, 709, 714, 719, 720, - 721, 723, 724, 726, 727, 729, 737, 738, 740, 741, - 742, 743, 744, 745, 746, 747, 748, 749, 755, 757, - 758, 760, 761, 763, 764, 766, 767, 768, 769, 770, - 771, 772, 773, 775, 776, 777, 779, 780, 781, 782, - 783, 784, 785, 787, 788, 789, 790, 795, 796, 797, - 798, 800, 801, 802, 803, 805, 806, 809, 810, 811, - 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, - 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, - 834, 835, 836, 841, 843, 847, 852, 860, 861, 865, - 866, 869, 871, 872, 873, 877, 878, 879, 883, 885, - 887, 888, 889, 890, 891, 894, 896, 897, 898, 899, - 900, 901, 902, 903, 904, 908, 909, 913, 914, 919, - 922, 924, 925, 926, 927, 931, 932, 933, 937, 938, - 942, 943, 947, 948, 951, 953, 954, 955, 956, 960, - 961, 964, 966, 967, 968, 972, 973, 974, 978, 979, - 980, 984, 988, 992, 993, 997, 998, 1002, 1003, 1004, - 1007, 1008, 1011, 1015, 1016, 1017, 1019, 1020, 1022, 1023, - 1026, 1027, 1030, 1036, 1039, 1041, 1042, 1043, 1045, 1050, - 1053, 1057, 1061, 1066, 1070, 1076, 1077, 1078, 1079, 1080, - 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, - 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, - 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, - 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, - 1122 + 0, 336, 336, 337, 339, 340, 342, 343, 344, 345, + 346, 347, 349, 350, 351, 352, 353, 354, 355, 356, + 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, + 367, 368, 369, 370, 371, 372, 373, 375, 377, 378, + 379, 380, 381, 382, 384, 385, 386, 387, 388, 389, + 391, 392, 394, 395, 396, 397, 398, 400, 401, 402, + 403, 405, 406, 407, 408, 409, 410, 411, 412, 414, + 421, 429, 435, 438, 443, 449, 450, 451, 452, 453, + 454, 455, 456, 458, 459, 461, 462, 463, 465, 466, + 468, 469, 471, 472, 473, 475, 476, 478, 479, 481, + 482, 483, 484, 485, 486, 488, 489, 490, 491, 492, + 493, 494, 500, 502, 503, 508, 513, 520, 523, 529, + 530, 531, 532, 533, 534, 537, 539, 543, 548, 556, + 559, 564, 569, 574, 579, 584, 589, 594, 602, 603, + 605, 607, 608, 609, 611, 612, 614, 616, 617, 619, + 620, 622, 626, 627, 628, 629, 630, 635, 637, 638, + 640, 644, 648, 652, 656, 660, 664, 668, 673, 678, + 684, 685, 687, 688, 689, 690, 691, 692, 693, 694, + 700, 701, 702, 703, 704, 705, 707, 708, 710, 711, + 712, 714, 715, 717, 722, 727, 728, 729, 731, 732, + 734, 735, 737, 745, 746, 748, 749, 750, 751, 752, + 753, 754, 755, 756, 757, 763, 765, 766, 768, 769, + 771, 772, 774, 775, 776, 777, 778, 779, 780, 781, + 783, 784, 785, 787, 788, 789, 790, 791, 792, 793, + 795, 796, 797, 798, 803, 804, 805, 806, 808, 809, + 810, 811, 813, 814, 817, 818, 819, 822, 823, 824, + 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, + 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, + 849, 851, 855, 860, 868, 869, 873, 874, 877, 879, + 880, 881, 885, 886, 887, 891, 893, 895, 896, 897, + 898, 899, 902, 904, 905, 906, 907, 908, 909, 910, + 911, 912, 916, 917, 921, 922, 927, 930, 932, 933, + 934, 935, 939, 940, 941, 945, 946, 950, 951, 955, + 956, 959, 961, 962, 963, 964, 968, 969, 972, 974, + 975, 976, 980, 981, 982, 986, 987, 988, 992, 996, + 1000, 1001, 1005, 1006, 1010, 1011, 1012, 1015, 1016, 1019, + 1023, 1024, 1025, 1027, 1028, 1030, 1031, 1034, 1035, 1038, + 1044, 1047, 1049, 1050, 1051, 1053, 1058, 1061, 1065, 1069, + 1074, 1078, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, + 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, + 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, + 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, + 1122, 1123, 1124, 1125, 1126, 1127, 1129, 1130 }; #endif @@ -1107,46 +1112,48 @@ static const char *const yytname[] = "$end", "error", "$undefined", "DUMMY_WHERE_CLAUSE", "DUMMY_INSERT_CLAUSE", "SELECT", "DELETE", "INSERT", "UPDATE", "REPLACE", "MERGE", "SHOW", "SET", "CALL", "CREATE", "DROP", "ALTER", "TRUNCATE", - "RENAME", "GRANT", "REVOKE", "ANALYZE", "PURGE", "COMMENT", "FROM", - "DUAL", "PREPARE", "EXECUTE", "USING", "SELECT_HINT_BEGIN", - "UPDATE_HINT_BEGIN", "DELETE_HINT_BEGIN", "INSERT_HINT_BEGIN", - "REPLACE_HINT_BEGIN", "MERGE_HINT_BEGIN", "HINT_END", "COMMENT_BEGIN", - "COMMENT_END", "ROUTE_TABLE", "ROUTE_PART_KEY", "QUERY_TIMEOUT", - "READ_CONSISTENCY", "WEAK", "STRONG", "FROZEN", "PLACE_HOLDER", "END_P", - "ERROR", "WHEN", "FLASHBACK", "AUDIT", "NOAUDIT", "BEGI", "START", - "TRANSACTION", "READ", "ONLY", "WITH", "CONSISTENT", "SNAPSHOT", "INDEX", - "XA", "WARNINGS", "ERRORS", "TRACE", "QUICK", "COUNT", "AS", "WHERE", - "VALUES", "ORDER", "GROUP", "HAVING", "INTO", "UNION", "FOR", - "TX_READ_ONLY", "AUTOCOMMIT_0", "SELECT_OBPROXY_ROUTE_ADDR", - "SET_OBPROXY_ROUTE_ADDR", "NAME_OB_DOT", "NAME_OB", "EXPLAIN", "DESC", - "DESCRIBE", "NAME_STR", "USE", "HELP", "SET_NAMES", "SET_CHARSET", - "SET_PASSWORD", "SET_DEFAULT", "SET_OB_READ_CONSISTENCY", - "SET_TX_READ_ONLY", "GLOBAL", "SESSION", "NUMBER_VAL", "GROUP_ID", - "TABLE_ID", "ELASTIC_ID", "TESTLOAD", "ODP_COMMENT", "TNT_ID", - "DISASTER_STATUS", "TRACE_ID", "RPC_ID", "DBP_COMMENT", "ROUTE_TAG", - "SYS_TAG", "TABLE_NAME", "SCAN_ALL", "PARALL", "SHARD_KEY", "INT_NUM", - "SHOW_PROXYNET", "THREAD", "CONNECTION", "LIMIT", "OFFSET", - "SHOW_PROCESSLIST", "SHOW_PROXYSESSION", "SHOW_GLOBALSESSION", - "ATTRIBUTE", "VARIABLES", "ALL", "STAT", "SHOW_PROXYCONFIG", "DIFF", - "USER", "LIKE", "SHOW_PROXYSM", "SHOW_PROXYCLUSTER", - "SHOW_PROXYRESOURCE", "SHOW_PROXYCONGESTION", "SHOW_PROXYROUTE", - "PARTITION", "ROUTINE", "SHOW_PROXYVIP", "SHOW_PROXYMEMORY", "OBJPOOL", - "SHOW_SQLAUDIT", "SHOW_WARNLOG", "SHOW_PROXYSTAT", "REFRESH", - "SHOW_PROXYTRACE", "SHOW_PROXYINFO", "BINARY", "UPGRADE", "IDC", - "SHOW_TOPOLOGY", "GROUP_NAME", "SHOW_DB_VERSION", "SHOW_DATABASES", - "SHOW_TABLES", "SELECT_DATABASE", "SHOW_CREATE_TABLE", - "ALTER_PROXYCONFIG", "ALTER_PROXYRESOURCE", "PING_PROXY", - "KILL_PROXYSESSION", "KILL_GLOBALSESSION", "KILL", "QUERY", "';'", "'@'", - "','", "'='", "'('", "')'", "'.'", "'{'", "'}'", "'#'", "'*'", "$accept", - "root", "sql_stmts", "sql_stmt", "stmt", "select_stmt", "explain_stmt", - "comment_stmt", "ddl_stmt", "mysql_ddl_stmt", "text_ps_from_stmt", + "RENAME", "TABLE", "UNIQUE", "GRANT", "REVOKE", "ANALYZE", "PURGE", + "COMMENT", "FROM", "DUAL", "PREPARE", "EXECUTE", "USING", + "SELECT_HINT_BEGIN", "UPDATE_HINT_BEGIN", "DELETE_HINT_BEGIN", + "INSERT_HINT_BEGIN", "REPLACE_HINT_BEGIN", "MERGE_HINT_BEGIN", + "HINT_END", "COMMENT_BEGIN", "COMMENT_END", "ROUTE_TABLE", + "ROUTE_PART_KEY", "QUERY_TIMEOUT", "READ_CONSISTENCY", "WEAK", "STRONG", + "FROZEN", "PLACE_HOLDER", "END_P", "ERROR", "WHEN", "FLASHBACK", "AUDIT", + "NOAUDIT", "BEGI", "START", "TRANSACTION", "READ", "ONLY", "WITH", + "CONSISTENT", "SNAPSHOT", "INDEX", "XA", "WARNINGS", "ERRORS", "TRACE", + "QUICK", "COUNT", "AS", "WHERE", "VALUES", "ORDER", "GROUP", "HAVING", + "INTO", "UNION", "FOR", "TX_READ_ONLY", "AUTOCOMMIT_0", + "SELECT_OBPROXY_ROUTE_ADDR", "SET_OBPROXY_ROUTE_ADDR", "NAME_OB_DOT", + "NAME_OB", "EXPLAIN", "DESC", "DESCRIBE", "NAME_STR", "USE", "HELP", + "SET_NAMES", "SET_CHARSET", "SET_PASSWORD", "SET_DEFAULT", + "SET_OB_READ_CONSISTENCY", "SET_TX_READ_ONLY", "GLOBAL", "SESSION", + "NUMBER_VAL", "GROUP_ID", "TABLE_ID", "ELASTIC_ID", "TESTLOAD", + "ODP_COMMENT", "TNT_ID", "DISASTER_STATUS", "TRACE_ID", "RPC_ID", + "DBP_COMMENT", "ROUTE_TAG", "SYS_TAG", "TABLE_NAME", "SCAN_ALL", + "PARALL", "SHARD_KEY", "INT_NUM", "SHOW_PROXYNET", "THREAD", + "CONNECTION", "LIMIT", "OFFSET", "SHOW_PROCESSLIST", "SHOW_PROXYSESSION", + "SHOW_GLOBALSESSION", "ATTRIBUTE", "VARIABLES", "ALL", "STAT", + "SHOW_PROXYCONFIG", "DIFF", "USER", "LIKE", "SHOW_PROXYSM", + "SHOW_PROXYCLUSTER", "SHOW_PROXYRESOURCE", "SHOW_PROXYCONGESTION", + "SHOW_PROXYROUTE", "PARTITION", "ROUTINE", "SHOW_PROXYVIP", + "SHOW_PROXYMEMORY", "OBJPOOL", "SHOW_SQLAUDIT", "SHOW_WARNLOG", + "SHOW_PROXYSTAT", "REFRESH", "SHOW_PROXYTRACE", "SHOW_PROXYINFO", + "BINARY", "UPGRADE", "IDC", "SHOW_TOPOLOGY", "GROUP_NAME", + "SHOW_DB_VERSION", "SHOW_DATABASES", "SHOW_TABLES", "SELECT_DATABASE", + "SHOW_CREATE_TABLE", "SELECT_PROXY_VERSION", "ALTER_PROXYCONFIG", + "ALTER_PROXYRESOURCE", "PING_PROXY", "KILL_PROXYSESSION", + "KILL_GLOBALSESSION", "KILL", "QUERY", "';'", "'@'", "','", "'='", "'('", + "')'", "'.'", "'{'", "'}'", "'#'", "'*'", "$accept", "root", "sql_stmts", + "sql_stmt", "stmt", "select_stmt", "explain_stmt", "comment_stmt", + "ddl_stmt", "mysql_ddl_stmt", "create_dll_expr", "text_ps_from_stmt", "text_ps_using_var_list", "text_ps_prepare_stmt", "text_ps_stmt", "oracle_ddl_stmt", "explain_or_desc_stmt", "explain_or_desc", "opt_from", - "select_expr_list", "select_tx_read_only_stmt", "set_autocommit_0_stmt", - "hooked_stmt", "shard_special_stmt", "show_db_version_stmt", - "show_es_id_stmt", "select_obproxy_route_addr_stmt", - "set_obproxy_route_addr_stmt", "set_names_stmt", "set_charset_stmt", - "set_password_stmt", "set_default_stmt", "set_ob_read_consistency_stmt", + "select_expr_list", "select_tx_read_only_stmt", + "select_proxy_version_stmt", "set_autocommit_0_stmt", "hooked_stmt", + "shard_special_stmt", "show_db_version_stmt", "show_es_id_stmt", + "select_obproxy_route_addr_stmt", "set_obproxy_route_addr_stmt", + "set_names_stmt", "set_charset_stmt", "set_password_stmt", + "set_default_stmt", "set_ob_read_consistency_stmt", "set_tx_read_only_stmt", "call_stmt", "routine_name_stmt", "call_expr_list", "call_expr", "expr_list", "expr", "clause", "fromlist", "sub_query", "opt_column_list", "column_list", "insert_stmt", @@ -1198,57 +1205,57 @@ static const yytype_uint16 yytoknum[] = 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, - 415, 416, 417, 59, 64, 44, 61, 40, 41, 46, - 123, 125, 35, 42 + 415, 416, 417, 418, 419, 420, 59, 64, 44, 61, + 40, 41, 46, 123, 125, 35, 42 }; # endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint16 yyr1[] = { - 0, 174, 175, 175, 176, 176, 177, 177, 177, 177, - 177, 177, 178, 178, 178, 178, 178, 178, 178, 178, - 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, - 178, 178, 178, 178, 178, 178, 178, 179, 180, 180, - 180, 180, 180, 180, 181, 181, 181, 181, 181, 181, - 182, 182, 183, 183, 183, 183, 183, 184, 184, 184, - 184, 184, 184, 184, 184, 185, 185, 186, 187, 187, + 0, 177, 178, 178, 179, 179, 180, 180, 180, 180, + 180, 180, 181, 181, 181, 181, 181, 181, 181, 181, + 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, + 181, 181, 181, 181, 181, 181, 181, 182, 183, 183, + 183, 183, 183, 183, 184, 184, 184, 184, 184, 184, + 185, 185, 186, 186, 186, 186, 186, 187, 187, 187, 187, 188, 188, 188, 188, 188, 188, 188, 188, 189, - 189, 190, 190, 190, 191, 191, 192, 192, 193, 193, - 193, 194, 194, 195, 195, 195, 195, 195, 196, 196, - 196, 196, 196, 196, 196, 197, 198, 198, 198, 198, - 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, - 208, 208, 209, 209, 209, 210, 210, 210, 210, 210, - 210, 211, 211, 212, 213, 213, 213, 214, 214, 215, - 216, 216, 217, 217, 218, 218, 219, 220, 221, 222, - 223, 224, 224, 225, 225, 225, 225, 225, 225, 225, - 226, 226, 226, 227, 227, 228, 228, 228, 228, 228, - 228, 228, 228, 228, 228, 228, 228, 228, 228, 229, - 229, 230, 230, 230, 231, 231, 232, 232, 232, 232, - 232, 233, 233, 234, 234, 235, 236, 236, 237, 237, - 237, 237, 237, 237, 237, 237, 237, 237, 237, 238, - 238, 239, 239, 240, 240, 241, 241, 242, 242, 243, - 243, 244, 244, 245, 245, 245, 246, 246, 247, 247, - 248, 249, 249, 250, 250, 250, 250, 250, 250, 250, - 250, 251, 251, 251, 251, 252, 252, 253, 253, 253, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 255, 255, 255, 255, 256, 256, 257, - 257, 258, 259, 259, 259, 260, 260, 260, 261, 262, - 263, 263, 263, 263, 263, 264, 265, 265, 265, 265, - 265, 265, 265, 265, 265, 266, 266, 267, 267, 268, - 269, 270, 270, 270, 270, 271, 271, 271, 272, 272, - 273, 273, 274, 274, 275, 276, 276, 276, 276, 277, - 277, 278, 279, 279, 279, 280, 280, 280, 281, 281, - 281, 282, 283, 284, 284, 285, 285, 286, 286, 286, - 287, 287, 288, 288, 288, 288, 289, 289, 290, 290, - 291, 291, 292, 293, 294, 295, 295, 295, 296, 297, - 297, 297, 297, 297, 297, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 298, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 298, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 298, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 298, 298, 298, 298, 298, 299, - 299 + 189, 190, 191, 191, 191, 192, 192, 192, 192, 192, + 192, 192, 192, 193, 193, 194, 194, 194, 195, 195, + 196, 196, 197, 197, 197, 198, 198, 199, 199, 200, + 200, 200, 200, 200, 200, 201, 201, 201, 201, 201, + 201, 201, 202, 203, 203, 203, 203, 204, 205, 206, + 207, 208, 209, 210, 211, 212, 213, 213, 213, 214, + 214, 214, 215, 215, 215, 215, 215, 215, 216, 216, + 217, 218, 218, 218, 219, 219, 220, 221, 221, 222, + 222, 223, 223, 224, 225, 226, 227, 228, 229, 229, + 230, 230, 230, 230, 230, 230, 230, 231, 231, 231, + 232, 232, 233, 233, 233, 233, 233, 233, 233, 233, + 233, 233, 233, 233, 233, 233, 234, 234, 235, 235, + 235, 236, 236, 237, 237, 237, 237, 237, 238, 238, + 239, 239, 240, 241, 241, 242, 242, 242, 242, 242, + 242, 242, 242, 242, 242, 242, 243, 243, 244, 244, + 245, 245, 246, 246, 247, 247, 248, 248, 249, 249, + 250, 250, 250, 251, 251, 252, 252, 253, 254, 254, + 255, 255, 255, 255, 255, 255, 255, 255, 256, 256, + 256, 256, 257, 257, 258, 258, 258, 259, 259, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 260, 260, 260, 260, 261, 261, 262, 262, 263, 264, + 264, 264, 265, 265, 265, 266, 267, 268, 268, 268, + 268, 268, 269, 270, 270, 270, 270, 270, 270, 270, + 270, 270, 271, 271, 272, 272, 273, 274, 275, 275, + 275, 275, 276, 276, 276, 277, 277, 278, 278, 279, + 279, 280, 281, 281, 281, 281, 282, 282, 283, 284, + 284, 284, 285, 285, 285, 286, 286, 286, 287, 288, + 289, 289, 290, 290, 291, 291, 291, 292, 292, 293, + 293, 293, 293, 294, 294, 295, 295, 296, 296, 297, + 298, 299, 300, 300, 300, 301, 302, 302, 302, 302, + 302, 302, 303, 303, 303, 303, 303, 303, 303, 303, + 303, 303, 303, 303, 303, 303, 303, 303, 303, 303, + 303, 303, 303, 303, 303, 303, 303, 303, 303, 303, + 303, 303, 303, 303, 303, 303, 303, 303, 303, 303, + 303, 303, 303, 303, 303, 303, 304, 304 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1259,44 +1266,44 @@ static const yytype_uint8 yyr2[] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 2, 1, 1, 1, 1, 0, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, + 4, 3, 2, 2, 4, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, 0, 2, + 0, 1, 2, 5, 3, 1, 3, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 2, 4, 3, 2, 2, - 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 1, 1, 1, 0, 2, 0, 1, 2, 5, - 3, 2, 3, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 2, 2, 1, 1, 3, 5, 7, - 1, 3, 1, 1, 1, 1, 1, 1, 5, 1, - 3, 5, 0, 1, 3, 1, 1, 1, 2, 3, - 1, 1, 2, 1, 2, 3, 3, 1, 1, 1, - 0, 3, 1, 3, 3, 5, 2, 2, 4, 2, - 2, 3, 1, 4, 6, 4, 5, 6, 4, 3, - 1, 1, 1, 1, 2, 3, 5, 6, 6, 6, - 6, 6, 7, 6, 6, 6, 6, 8, 8, 0, - 2, 2, 2, 1, 3, 1, 4, 4, 3, 6, - 4, 4, 6, 3, 1, 3, 0, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 5, 3, 0, - 1, 0, 1, 1, 1, 1, 2, 1, 2, 1, - 2, 2, 3, 0, 1, 2, 1, 2, 1, 2, - 2, 0, 2, 4, 1, 4, 5, 1, 4, 4, - 5, 0, 1, 1, 1, 0, 1, 3, 3, 2, + 2, 2, 1, 1, 3, 5, 7, 1, 3, 1, + 1, 1, 1, 1, 1, 5, 1, 3, 5, 0, + 1, 3, 1, 1, 1, 2, 3, 1, 1, 2, + 1, 2, 3, 3, 1, 1, 1, 0, 3, 1, + 3, 3, 5, 2, 2, 4, 2, 2, 3, 1, + 4, 6, 4, 5, 6, 4, 3, 1, 1, 1, + 1, 2, 3, 5, 6, 6, 6, 6, 6, 7, + 6, 6, 6, 6, 8, 8, 0, 2, 2, 2, + 1, 3, 1, 4, 4, 3, 6, 4, 4, 6, + 3, 1, 3, 0, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 5, 3, 0, 1, 0, 1, + 1, 1, 1, 2, 1, 2, 1, 2, 2, 3, + 0, 1, 2, 1, 2, 1, 2, 2, 0, 2, + 4, 1, 4, 5, 1, 4, 4, 5, 0, 1, + 1, 1, 0, 1, 3, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 0, 2, 4, 4, 0, 2, 0, - 2, 2, 1, 1, 3, 2, 3, 4, 1, 2, - 0, 2, 3, 2, 2, 2, 0, 2, 3, 2, - 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, - 2, 0, 1, 1, 2, 2, 3, 2, 1, 2, - 1, 2, 2, 2, 2, 0, 1, 3, 5, 2, - 3, 2, 0, 1, 2, 2, 2, 2, 4, 5, - 5, 3, 1, 2, 3, 3, 2, 2, 3, 3, - 0, 2, 1, 3, 3, 3, 0, 1, 1, 3, - 2, 3, 2, 2, 1, 0, 2, 4, 2, 1, - 3, 2, 4, 3, 5, 1, 1, 1, 1, 1, + 0, 2, 4, 4, 0, 2, 0, 2, 2, 1, + 1, 3, 2, 3, 4, 1, 2, 0, 2, 3, + 2, 2, 2, 0, 2, 3, 2, 3, 2, 3, + 3, 4, 1, 2, 2, 3, 2, 2, 0, 1, + 1, 2, 2, 3, 2, 1, 2, 1, 2, 2, + 2, 2, 0, 1, 3, 5, 2, 3, 2, 0, + 1, 2, 2, 2, 2, 4, 5, 5, 3, 1, + 2, 3, 3, 2, 2, 3, 3, 0, 2, 1, + 3, 3, 3, 0, 1, 1, 3, 2, 3, 2, + 2, 1, 0, 2, 4, 2, 1, 3, 2, 4, + 3, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1 + 1, 1, 1, 1, 1, 1, 1, 1 }; /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state @@ -1304,480 +1311,486 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint16 yydefact[] = { - 0, 3, 215, 219, 223, 217, 226, 228, 350, 0, - 0, 52, 53, 54, 55, 56, 71, 72, 73, 74, - 76, 0, 0, 231, 231, 231, 231, 231, 231, 179, - 75, 77, 78, 352, 0, 0, 110, 0, 364, 81, - 82, 83, 0, 0, 112, 113, 114, 115, 116, 117, - 0, 288, 296, 290, 277, 305, 277, 277, 311, 279, - 318, 320, 273, 325, 277, 332, 0, 106, 105, 101, - 102, 100, 0, 0, 0, 342, 0, 0, 0, 9, - 0, 2, 4, 0, 12, 18, 19, 33, 50, 0, - 34, 51, 0, 79, 93, 94, 22, 97, 99, 98, - 95, 96, 26, 27, 28, 29, 30, 31, 32, 13, - 15, 16, 17, 35, 14, 0, 163, 86, 0, 245, - 0, 0, 0, 21, 23, 250, 251, 252, 254, 253, - 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 20, 24, - 25, 36, 88, 224, 221, 249, 0, 0, 91, 0, - 0, 0, 0, 150, 152, 416, 417, 418, 377, 375, - 378, 379, 415, 376, 381, 380, 384, 383, 382, 419, - 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, - 409, 410, 411, 413, 412, 414, 385, 386, 387, 388, - 389, 390, 391, 392, 393, 394, 395, 396, 402, 397, - 0, 420, 119, 0, 69, 0, 0, 0, 237, 234, - 216, 0, 231, 218, 220, 223, 227, 229, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 352, 0, 356, 0, 0, 0, 362, 363, 282, - 283, 281, 277, 277, 277, 295, 0, 0, 289, 277, - 0, 285, 306, 277, 307, 309, 312, 313, 310, 0, - 317, 279, 315, 319, 321, 323, 0, 322, 326, 324, - 277, 329, 333, 331, 335, 336, 337, 0, 0, 103, - 0, 0, 343, 346, 347, 0, 0, 10, 1, 5, - 6, 7, 215, 57, 68, 63, 58, 59, 61, 60, - 64, 62, 38, 39, 42, 40, 41, 43, 80, 104, - 44, 45, 48, 46, 47, 49, 164, 0, 84, 87, - 131, 133, 139, 147, 138, 137, 365, 369, 246, 0, - 365, 146, 149, 0, 90, 225, 351, 247, 248, 92, - 0, 0, 0, 0, 0, 0, 122, 0, 67, 0, - 0, 241, 0, 0, 230, 232, 222, 0, 0, 0, + 0, 3, 222, 226, 230, 224, 233, 235, 357, 0, + 0, 57, 53, 54, 55, 56, 75, 76, 77, 78, + 80, 0, 0, 238, 238, 238, 238, 238, 238, 186, + 79, 81, 82, 359, 0, 0, 117, 0, 371, 85, + 86, 87, 0, 0, 119, 120, 121, 122, 123, 124, + 0, 295, 303, 297, 284, 312, 284, 284, 318, 286, + 325, 327, 280, 332, 284, 339, 0, 113, 112, 108, + 109, 107, 0, 95, 0, 0, 349, 0, 0, 0, + 9, 0, 2, 4, 0, 12, 18, 19, 33, 50, + 0, 34, 51, 0, 83, 99, 100, 101, 22, 104, + 106, 105, 102, 103, 26, 27, 28, 29, 30, 31, + 32, 13, 15, 16, 17, 35, 14, 0, 170, 90, + 0, 252, 0, 0, 0, 21, 23, 257, 258, 259, + 261, 260, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + 20, 24, 25, 36, 92, 231, 228, 256, 0, 0, + 97, 0, 0, 0, 0, 157, 159, 423, 424, 425, + 384, 382, 385, 386, 422, 383, 388, 387, 391, 390, + 389, 426, 405, 406, 407, 408, 410, 411, 412, 413, + 414, 415, 416, 417, 418, 420, 419, 421, 392, 393, + 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, + 409, 404, 0, 427, 126, 58, 0, 59, 52, 0, + 73, 0, 0, 0, 244, 241, 223, 0, 238, 225, + 227, 230, 234, 236, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 359, 0, 363, + 0, 0, 0, 369, 370, 289, 290, 288, 284, 284, + 284, 302, 0, 0, 296, 284, 0, 292, 313, 284, + 314, 316, 319, 320, 317, 0, 324, 286, 322, 326, + 328, 330, 0, 329, 333, 331, 284, 336, 340, 338, + 342, 343, 344, 0, 0, 110, 0, 0, 0, 350, + 353, 354, 0, 0, 10, 1, 5, 6, 7, 222, + 61, 72, 67, 62, 63, 65, 64, 68, 66, 38, + 39, 42, 40, 41, 43, 84, 111, 44, 45, 48, + 46, 47, 49, 171, 0, 88, 91, 138, 140, 146, + 154, 145, 144, 372, 376, 253, 0, 372, 153, 156, + 0, 94, 232, 358, 254, 255, 98, 0, 0, 0, + 0, 0, 0, 129, 0, 60, 71, 0, 0, 248, + 0, 0, 237, 239, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 196, 0, 0, 209, 209, 0, 0, 0, - 165, 0, 0, 183, 180, 11, 0, 0, 353, 357, - 358, 354, 355, 111, 273, 277, 297, 277, 277, 301, - 277, 299, 291, 293, 0, 294, 277, 286, 278, 308, - 314, 280, 316, 274, 0, 330, 334, 107, 0, 0, - 341, 344, 345, 348, 349, 8, 134, 0, 0, 0, - 37, 132, 0, 368, 0, 0, 371, 0, 140, 0, - 160, 162, 161, 159, 0, 0, 0, 0, 0, 0, - 151, 130, 125, 127, 126, 0, 0, 123, 120, 0, - 70, 0, 242, 243, 244, 0, 0, 0, 0, 211, - 213, 214, 196, 196, 196, 196, 211, 0, 0, 0, - 0, 0, 0, 0, 209, 209, 0, 0, 196, 196, - 196, 210, 196, 0, 0, 196, 181, 182, 360, 0, - 0, 284, 298, 302, 277, 303, 300, 292, 287, 0, - 0, 327, 0, 0, 338, 135, 136, 85, 366, 0, - 373, 370, 148, 0, 0, 89, 155, 158, 153, 0, - 0, 0, 128, 0, 0, 118, 0, 65, 0, 233, - 235, 0, 0, 239, 238, 196, 212, 0, 0, 0, - 0, 208, 0, 198, 199, 201, 202, 205, 206, 203, - 204, 200, 166, 0, 0, 0, 0, 0, 0, 0, - 0, 361, 359, 304, 276, 275, 0, 0, 108, 339, - 340, 0, 0, 372, 142, 0, 145, 156, 0, 0, - 129, 124, 121, 0, 236, 240, 0, 169, 167, 170, - 171, 211, 197, 175, 176, 173, 174, 0, 0, 0, - 0, 0, 185, 0, 0, 168, 328, 0, 367, 374, - 0, 141, 154, 157, 66, 172, 207, 0, 0, 0, - 0, 0, 0, 209, 0, 109, 143, 0, 0, 0, - 188, 0, 0, 194, 177, 184, 0, 178, 186, 187, - 0, 0, 190, 0, 191, 209, 0, 195, 193, 0, - 189, 192 + 203, 0, 0, 216, 216, 0, 0, 0, 172, 0, + 0, 190, 187, 11, 0, 0, 360, 364, 365, 361, + 362, 118, 280, 284, 304, 284, 284, 308, 284, 306, + 298, 300, 0, 301, 284, 293, 285, 315, 321, 287, + 323, 281, 0, 337, 341, 114, 0, 96, 0, 348, + 351, 352, 355, 356, 8, 141, 0, 0, 0, 37, + 139, 0, 375, 0, 0, 378, 0, 147, 0, 167, + 169, 168, 166, 0, 0, 0, 0, 0, 0, 158, + 137, 132, 134, 133, 0, 0, 130, 127, 0, 74, + 0, 249, 250, 251, 0, 0, 0, 0, 218, 220, + 221, 203, 203, 203, 203, 218, 0, 0, 0, 0, + 0, 0, 0, 216, 216, 0, 0, 203, 203, 203, + 217, 203, 0, 0, 203, 188, 189, 367, 0, 0, + 291, 305, 309, 284, 310, 307, 299, 294, 0, 0, + 334, 0, 0, 345, 142, 143, 89, 373, 0, 380, + 377, 155, 0, 0, 93, 162, 165, 160, 0, 0, + 0, 135, 0, 0, 125, 0, 69, 0, 240, 242, + 0, 0, 246, 245, 203, 219, 0, 0, 0, 0, + 215, 0, 205, 206, 208, 209, 212, 213, 210, 211, + 207, 173, 0, 0, 0, 0, 0, 0, 0, 0, + 368, 366, 311, 283, 282, 0, 0, 115, 346, 347, + 0, 0, 379, 149, 0, 152, 163, 0, 0, 136, + 131, 128, 0, 243, 247, 0, 176, 174, 177, 178, + 218, 204, 182, 183, 180, 181, 0, 0, 0, 0, + 0, 192, 0, 0, 175, 335, 0, 374, 381, 0, + 148, 161, 164, 70, 179, 214, 0, 0, 0, 0, + 0, 0, 216, 0, 116, 150, 0, 0, 0, 195, + 0, 0, 201, 184, 191, 0, 185, 193, 194, 0, + 0, 197, 0, 198, 216, 0, 202, 200, 0, 196, + 199 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 80, 81, 82, 83, 332, 85, 86, 87, 88, - 304, 470, 89, 90, 91, 92, 93, 440, 328, 94, + -1, 81, 82, 83, 84, 339, 86, 87, 88, 89, + 218, 311, 479, 90, 91, 92, 93, 94, 449, 335, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, - 105, 106, 107, 108, 210, 466, 467, 329, 330, 331, - 333, 334, 534, 595, 109, 110, 111, 112, 113, 114, - 163, 164, 453, 115, 116, 240, 394, 621, 622, 624, - 652, 653, 497, 382, 500, 555, 501, 117, 118, 119, - 120, 154, 121, 122, 220, 221, 222, 475, 339, 123, - 124, 277, 261, 272, 125, 251, 126, 127, 128, 258, - 129, 255, 130, 131, 132, 133, 268, 134, 135, 136, - 137, 138, 279, 139, 140, 283, 141, 142, 143, 144, - 145, 146, 147, 157, 148, 398, 399, 400, 149, 150, - 151, 443, 335, 336, 211, 337 + 105, 106, 107, 108, 109, 110, 212, 475, 476, 336, + 337, 338, 340, 341, 543, 604, 111, 112, 113, 114, + 115, 116, 165, 166, 462, 117, 118, 246, 402, 630, + 631, 633, 661, 662, 506, 390, 509, 564, 510, 119, + 120, 121, 122, 156, 123, 124, 226, 227, 228, 484, + 346, 125, 126, 283, 267, 278, 127, 257, 128, 129, + 130, 264, 131, 261, 132, 133, 134, 135, 274, 136, + 137, 138, 139, 140, 285, 141, 142, 289, 143, 144, + 145, 146, 147, 148, 149, 159, 150, 406, 407, 408, + 151, 152, 153, 452, 342, 343, 213, 344 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -466 +#define YYPACT_NINF -475 static const yytype_int16 yypact[] = { - 359, -466, 27, -466, 23, -466, -466, -466, 61, -35, - 1139, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, 1139, 1139, 36, 36, 36, 36, 36, 36, 135, - -466, -466, -466, 809, 78, 56, -466, 77, -466, -466, - -466, -466, 85, 97, -466, -466, -466, -466, -466, -466, - 86, -466, 166, 54, 28, 83, -63, 118, -20, -17, - 168, 129, 107, 171, 76, 190, 150, 25, -466, -466, - -466, -466, 1139, 282, 307, -466, 201, 234, -62, 270, - 317, 534, -466, 12, -466, -466, -466, -466, -466, 180, - -466, -466, 221, 1242, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, 221, 283, 151, 651, 255, - 1139, 651, 1139, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -16, 274, -466, -466, 152, 212, 151, 157, - 243, 245, -36, -466, 163, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - 169, -466, 161, 309, 306, 170, 172, 173, 174, -466, - -466, 300, 36, -466, -466, 23, -466, -466, 261, 177, - 178, 179, 182, 167, 183, 185, 188, 191, 200, 193, - 26, -466, 175, 247, 265, 275, 249, -466, -466, -466, - 250, -466, -8, -45, 87, -466, -42, 277, -466, 181, - 302, -466, -466, 118, -466, -466, -466, 303, -466, 313, - -466, 258, -466, -466, -466, -466, 284, -466, 231, -466, - 118, -466, 285, -466, -466, -466, -466, 318, 251, -466, - 319, 321, 290, 291, -466, 292, 293, -466, -466, -466, - -466, 361, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, 4, -466, - -466, -466, -466, -466, -466, -466, -466, 2, 389, 151, - -466, -466, -466, -466, -466, -466, 279, 922, -466, 391, - 279, -466, -466, 392, -4, -466, -466, -466, -466, 151, - 30, 252, 256, 259, 192, -25, 10, 1139, -466, 260, - 304, 257, 338, -3, -466, -466, -466, 262, -14, -14, - -14, -14, 124, 263, 264, 266, 267, 273, 278, 287, - 288, 289, -466, -14, -14, -14, -14, 294, 295, -14, - -466, 340, 342, -466, -466, -466, 370, 369, -466, 271, - -466, -466, -466, -466, 314, 118, -466, 118, 93, -466, - 118, -466, -466, -466, 375, -466, 118, -466, -466, -466, - -466, -466, -466, -66, 344, -466, -466, 390, 296, 297, - -466, -466, -466, -466, -466, -466, -466, 298, 144, 651, - -466, -466, -54, -466, 1139, 1139, -466, 651, -23, 651, - -466, -466, -466, -466, 30, 30, 30, 299, 383, 386, - -466, -466, -466, -466, -466, -34, 94, -466, 301, 387, - 310, 308, -466, -466, -466, 315, 388, -46, 316, -14, - -466, -466, -466, -466, -466, -466, -14, 393, -14, -14, - -14, -14, -14, -14, -14, -14, -14, -15, -466, -466, - -466, -466, -466, 311, 312, -466, -466, -466, -466, 412, - 247, -466, -466, -466, 118, -466, -466, -466, -466, 364, - 373, 322, 345, 407, 35, -466, -466, -466, -466, 413, - -466, 1036, -466, 417, 52, -466, -466, -466, -466, 30, - 336, 339, -466, 425, 10, -466, 1139, -466, 343, -466, - -466, 341, 355, -466, -466, -466, -466, -13, -12, -11, - -7, -466, 358, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, 167, -5, -1, 0, 3, 148, 408, - 6, -466, -466, -466, -466, -466, 444, 360, -466, -466, - -466, 362, 1139, -466, -466, 127, -466, -466, 30, 30, - -466, -466, -466, 446, -466, -466, 7, -466, -466, -466, - -466, -14, -466, -466, -466, -466, -466, 365, 366, 367, - 368, 357, 372, 371, 398, -466, -466, 448, -466, -466, - 450, -466, -466, -466, -466, -466, -466, -14, -14, -70, - 477, 522, 148, -14, 525, -466, -466, 403, 404, 409, - -466, 410, 405, 414, -466, -466, 74, -466, -466, -466, - -14, -14, -466, 477, -466, -14, 406, -466, -466, 420, - -466, -466 + 365, -475, 28, -475, -61, -475, -475, -475, 145, -39, + 1154, 35, -475, -475, -475, -475, -475, -475, -475, -475, + -475, 1154, 1154, 30, 30, 30, 30, 30, 30, 139, + -475, -475, -475, 821, 15, 201, -475, -83, -475, -475, + -475, -475, 41, 57, -475, -475, -475, -475, -475, -475, + 191, -475, 170, 55, 135, 42, -62, 46, -33, -11, + 98, 87, -7, 157, -63, 162, 150, 26, -475, -475, + -475, -475, 1154, 241, 303, 312, -475, 204, 237, -51, + 273, 322, 541, -475, 8, -475, -475, -475, -475, -475, + 184, -475, -475, 227, 1258, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, 227, 285, 154, + 660, 258, 1154, 660, 1154, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -18, 276, -475, -475, 151, 16, + 154, 160, 247, 248, -36, -475, 164, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, 163, -475, 165, -475, 278, -475, -475, 309, + 311, 172, 173, 174, 175, -475, -475, 310, 30, -475, + -475, -61, -475, -475, 263, 179, 181, 182, 186, 169, + 187, 190, 193, 195, 202, 196, 36, -475, 218, 244, + 266, 269, 238, -475, -475, -475, 242, -475, -30, 52, + -24, -475, -22, 284, -475, 183, 302, -475, -475, 46, + -475, -475, -475, 308, -475, 319, -475, 261, -475, -475, + -475, -475, 253, -475, 236, -475, 46, -475, 280, -475, + -475, -475, -475, 323, 255, -475, 1154, 325, 326, 290, + 295, -475, 296, 297, -475, -475, -475, -475, 366, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -25, -475, -475, -475, -475, + -475, -475, -475, -475, 1, 387, 154, -475, -475, -475, + -475, -475, -475, 286, 935, -475, 389, 286, -475, -475, + 395, -6, -475, -475, -475, -475, 154, 60, 256, 257, + 260, 99, -37, -4, 1154, -475, -475, 265, 307, 104, + 341, -5, -475, -475, -475, 262, 93, 93, 93, 93, + 79, 267, 268, 270, 271, 272, 283, 292, 293, 294, + -475, 93, 93, 93, 93, 298, 300, 93, -475, 344, + 347, -475, -475, -475, 375, 374, -475, 274, -475, -475, + -475, -475, 316, 46, -475, 46, -16, -475, 46, -475, + -475, -475, 355, -475, 46, -475, -475, -475, -475, -475, + -475, -50, 332, -475, -475, 394, 301, -475, 304, -475, + -475, -475, -475, -475, -475, -475, 305, 146, 660, -475, + -475, -41, -475, 1154, 1154, -475, 660, 3, 660, -475, + -475, -475, -475, 60, 60, 60, 306, 382, 383, -475, + -475, -475, -475, -475, -34, 114, -475, 299, 391, 314, + 313, -475, -475, -475, 318, 396, -48, 320, 93, -475, + -475, -475, -475, -475, -475, 93, 397, 93, 93, 93, + 93, 93, 93, 93, 93, 93, -17, -475, -475, -475, + -475, -475, 317, 321, -475, -475, -475, -475, 411, 244, + -475, -475, -475, 46, -475, -475, -475, -475, 362, 363, + 315, 333, 409, 6, -475, -475, -475, -475, 412, -475, + 1050, -475, 413, 69, -475, -475, -475, -475, 60, 334, + 337, -475, 419, -4, -475, 1154, -475, 343, -475, -475, + 342, 346, -475, -475, -475, -475, -15, -14, -13, -12, + -475, 345, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, 169, -9, -8, -3, -2, 88, 449, 0, + -475, -475, -475, -475, -475, 450, 367, -475, -475, -475, + 361, 1154, -475, -475, 125, -475, -475, 60, 60, -475, + -475, -475, 451, -475, -475, 2, -475, -475, -475, -475, + 93, -475, -475, -475, -475, -475, 368, 369, 370, 371, + 386, 376, 372, 393, -475, -475, 452, -475, -475, 454, + -475, -475, -475, -475, -475, -475, 93, 93, -68, 460, + 522, 88, 93, 529, -475, -475, 400, 407, 414, -475, + 415, 410, 417, -475, -475, 78, -475, -475, -475, 93, + 93, -475, 460, -475, 93, 416, -475, -475, 418, -475, + -475 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -466, -466, -466, 496, 545, 5, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, 428, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, 492, 8, -466, 45, -148, -296, -466, - -118, 57, -466, -466, 67, 88, 92, 102, 103, -466, - 235, -466, -437, 476, -466, -466, -466, -49, -466, -466, - -69, -466, -410, 24, -380, -465, -355, -466, -466, -466, - -466, 374, -466, -466, 253, 376, -466, -466, -466, -466, - -466, 196, -55, 325, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, -466, -466, -466, - -466, -466, -466, -466, -466, -466, -466, 91, -466, -466, - 513, 268, -466, 101, -466, -10 + -475, -475, -475, 498, 549, 5, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, 432, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, 500, 11, -475, 44, -150, + -303, -475, -120, 45, -475, -475, -27, 95, 108, 137, + 188, -475, 239, -475, -446, 480, -475, -475, -475, -52, + -475, -475, -72, -475, -285, 20, -387, -474, -363, -475, + -475, -475, -475, 377, -475, -475, 264, 378, -475, -475, + -475, -475, -475, 197, -55, 328, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, -475, + -475, -475, -475, -475, -475, -475, -475, -475, -475, 84, + -475, -475, 517, 281, -475, 180, -475, -10 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule which number is the opposite. If zero, do what YYDEFACT says. If YYTABLE_NINF, syntax error. */ -#define YYTABLE_NINF -420 +#define YYTABLE_NINF -427 static const yytype_int16 yytable[] = { - 212, 264, 265, 341, 344, 84, 502, 302, -86, 281, - 349, 213, 214, 482, 483, 484, 485, 536, 537, 538, - -87, 561, 572, -144, 607, 608, 609, 528, 498, 499, - 610, 23, 613, 441, 505, 552, 614, 615, 84, 412, - 616, 649, 158, 625, 635, 353, 159, 542, 441, 287, - -419, 294, 519, 441, 295, 461, 159, 302, 300, 160, - 161, 266, 212, 390, 391, 392, 260, 480, 407, 160, - 161, 481, 557, 558, 559, 560, 215, 216, 477, 408, - 289, 23, 413, 212, 260, 263, 84, 414, 574, 575, - 576, 462, 577, 288, 303, 580, 217, 312, 650, 520, - 296, 319, 597, 152, 267, 405, 463, 393, 244, 245, - 478, 450, 269, 529, 569, 570, 589, 218, 270, 271, - 320, 260, 553, 464, 556, 155, 451, 156, 354, 162, - 543, 556, 243, 563, 564, 565, 566, 567, 568, 162, - -144, 571, 441, 452, 533, 606, 636, 153, 590, 219, - 573, 327, 573, 573, 573, 259, 306, 260, 573, 313, - 573, 632, 633, 327, 573, 573, 247, -419, 573, 327, - 436, 573, 573, -419, 465, 301, 256, 307, 248, 438, - 314, 308, 321, 257, 315, 302, 3, 4, 5, 6, - 7, 309, 310, 10, 316, 317, 262, 406, 409, 411, - 410, 249, 250, 322, 417, 260, 514, 323, 419, 23, - 24, 25, 26, 27, 28, 228, 260, 324, 325, 280, - 275, 340, 260, 342, 276, 425, 302, 3, 4, 5, - 6, 7, 229, 230, 231, 232, 233, 234, 235, 236, - 237, 238, 664, 246, 239, 617, 665, 260, 372, 273, - 23, 24, 25, 26, 27, 28, 556, 618, 619, 544, - 620, 38, 545, 656, 373, 374, 375, 376, 274, 377, - 378, 379, 380, 457, 347, 348, 381, 223, 224, 225, - 226, 227, 647, 648, 278, 669, 458, 459, 252, 253, - 486, 254, 630, 487, 290, 631, 284, 285, 286, 472, - 473, 474, 396, 282, 397, 666, 667, 387, 388, 416, - 260, 327, 526, 291, 292, 293, 297, 298, 327, 29, - 338, 527, 345, 350, 351, 346, 352, 446, 355, 532, - 357, 535, 437, 358, 359, 364, 356, 360, 395, 361, - 362, 363, 367, 368, 369, 370, 401, 468, 371, 383, - 512, 384, 513, 515, 385, 516, 402, 386, 415, 389, - 1, 518, 403, 404, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 418, 420, 21, 22, 269, 23, 24, - 25, 26, 27, 28, 421, 29, 424, 423, 426, 427, - 429, 428, 430, 431, 432, 433, 434, 435, 30, 31, - 32, 33, 34, 439, 442, 447, 449, 471, 454, 476, - 35, 506, 455, 507, 469, 456, 508, 509, 479, 488, - 489, 276, 490, 491, 530, 531, 510, 36, 37, 492, - 38, 39, 40, 41, 493, 42, 43, 44, 45, 46, - 47, 48, 49, 494, 495, 496, 517, 521, 522, 583, - 503, 504, 523, 524, 540, 539, 525, 541, 547, 551, - 546, 581, 623, 50, 562, 548, 549, 584, 51, 52, - 53, 578, 579, 550, 554, 54, 585, 586, 588, 55, - 56, 57, 58, 59, 591, 587, 60, 61, 594, 62, - 63, 64, 598, 65, 66, 599, 600, 603, 67, 604, - 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, - 78, 593, 79, 605, 611, 626, 627, 634, 641, 645, - 628, 646, 637, 638, 639, 640, 602, 642, 643, 2, - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 16, 17, 18, 19, 20, 651, 654, - 21, 22, 657, 23, 24, 25, 26, 27, 28, 644, - 29, 658, 659, 662, 670, 660, 661, 299, 242, 663, - 343, 305, 629, 30, 31, 32, 33, 34, 671, 601, - 460, 596, 326, 655, 668, 35, 422, 612, 365, 366, - 511, 582, 311, 0, 0, 0, 0, 0, 448, 0, - 0, 0, 36, 37, 0, 38, 39, 40, 41, 0, - 42, 43, 44, 45, 46, 47, 48, 49, 0, 0, + 214, 270, 271, 348, 351, 85, 309, 511, -90, 287, + 356, 219, 220, 491, 492, 493, 494, 545, 546, 547, + -91, 570, 581, -426, 616, 617, 618, 619, 507, 508, + 622, 623, 23, 450, 514, 561, 624, 625, 85, 634, + 160, 644, 537, 470, 161, 658, 161, 360, 450, 551, + 272, -151, 293, 450, 215, 216, 307, 162, 163, 162, + 163, 420, 214, 313, 301, 155, 320, 302, 266, 266, + 528, 249, 221, 222, 309, 398, 399, 400, 486, 471, + 354, 355, 286, 295, 214, 413, 252, 85, 269, 598, + 328, 418, 223, 273, 472, 310, 294, 217, 319, 523, + 23, 266, 606, 659, 421, 326, 154, 266, 281, 422, + 487, 473, 282, 224, 303, 266, 578, 579, 529, 401, + 275, 599, 327, 562, 253, 565, 276, 277, 164, 538, + 164, 361, 565, 552, 572, 573, 574, 575, 576, 577, + 254, -426, 580, 459, 450, 225, 645, -426, 481, 482, + 483, 582, 334, 582, 582, 582, 582, 268, 460, 582, + 582, 641, 642, 474, 334, 582, 582, 415, 582, -151, + 582, 334, 445, 542, 308, 461, 489, 266, 416, 262, + 490, 279, 466, 266, 447, 314, 263, 626, 321, 309, + 3, 4, 5, 6, 7, 467, 468, 10, 315, 627, + 628, 322, 629, 414, 417, 419, 566, 567, 568, 569, + 425, 157, 329, 158, 427, 23, 24, 25, 26, 27, + 28, 234, 583, 584, 585, 330, 586, 316, 280, 589, + 323, 433, 309, 3, 4, 5, 6, 7, 235, 236, + 237, 238, 239, 240, 241, 242, 243, 244, 495, 673, + 245, 496, 380, 674, 331, 250, 251, 565, 23, 24, + 25, 26, 27, 28, 265, 665, 266, 38, 381, 382, + 383, 384, 284, 385, 386, 387, 388, 288, 317, 615, + 389, 324, 553, 656, 657, 554, 437, 678, 229, 230, + 231, 232, 233, 639, 258, 259, 640, 260, 290, 291, + 292, 404, 347, 405, 349, 332, 675, 676, 255, 256, + 296, 395, 396, 424, 266, 297, 334, 535, 298, 299, + 300, 304, 305, 29, 334, 345, 352, 353, 536, 357, + 358, 359, 362, 363, 455, 366, 541, 364, 544, 446, + 365, 367, 368, 369, 370, 371, 375, 372, 376, 409, + 377, 378, 410, 411, 477, 379, 391, 412, 521, 392, + 522, 524, 393, 525, 394, 397, 1, 423, 431, 527, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 403, 426, 16, 17, 18, 19, + 20, 428, 275, 21, 22, 434, 23, 24, 25, 26, + 27, 28, 429, 29, 432, 440, 435, 436, 438, 439, + 441, 442, 443, 448, 444, 456, 30, 31, 32, 33, + 34, 458, 480, 451, 485, 463, 464, 515, 35, 465, + 516, 488, 478, 517, 518, 282, 497, 498, 526, 499, + 500, 501, 519, 539, 540, 36, 37, 530, 38, 39, + 40, 41, 502, 42, 43, 44, 45, 46, 47, 48, + 49, 503, 504, 505, 531, 549, 550, 512, 592, 513, + 532, 555, 590, 533, 556, 548, 534, 593, 594, 560, + 571, 50, 557, 595, 558, 596, 51, 52, 53, 559, + 587, 563, 597, 54, 588, 600, 603, 55, 56, 57, + 58, 59, 609, 607, 60, 61, 608, 62, 63, 64, + 612, 65, 66, 613, 620, 632, 67, 614, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 602, 80, 637, 635, 643, 654, 636, 655, 646, 647, + 648, 649, 652, 660, 651, 611, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 650, 663, 16, 17, 18, 19, 20, 653, 666, 21, + 22, 667, 23, 24, 25, 26, 27, 28, 668, 29, + 306, 671, 248, 669, 670, 672, 350, 679, 605, 680, + 312, 638, 30, 31, 32, 33, 34, 610, 333, 664, + 677, 469, 621, 591, 35, 430, 373, 318, 374, 520, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 36, 37, 0, 38, 39, 40, 41, 457, 42, + 43, 44, 45, 46, 47, 48, 49, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, + 0, 0, 51, 52, 53, 309, 0, 0, 0, 54, + 0, 0, 0, 55, 56, 57, 58, 59, 0, 0, + 60, 61, 0, 62, 63, 64, 0, 65, 66, 0, + 0, 23, 67, 0, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 0, 80, 0, 0, + 0, 167, 168, 169, 170, 171, 172, 0, 0, 0, + 173, 0, 174, 175, 176, 177, 178, 179, 180, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 0, 0, 198, 199, 0, + 200, 0, 0, 0, 201, 202, 0, 203, 0, 204, + 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 206, 0, 0, 0, 207, 0, 0, 0, 208, + 209, 0, 210, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 211, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 16, 17, 18, 19, 20, 0, 0, 21, + 22, 0, 23, 24, 25, 26, 27, 28, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, - 0, 0, 0, 51, 52, 53, 302, 0, 0, 0, - 54, 0, 0, 0, 55, 56, 57, 58, 59, 0, - 0, 60, 61, 0, 62, 63, 64, 0, 65, 66, - 23, 0, 0, 67, 0, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 0, 79, 0, 0, - 165, 166, 167, 168, 169, 170, 0, 0, 0, 171, - 0, 172, 173, 174, 175, 176, 177, 178, 0, 0, + 0, 0, 30, 31, 32, 247, 34, 0, 0, 0, + 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - 192, 193, 194, 195, 0, 0, 196, 197, 0, 198, - 0, 0, 0, 199, 200, 0, 201, 0, 202, 203, + 0, 36, 37, 0, 38, 39, 40, 41, 0, 42, + 43, 44, 45, 46, 47, 48, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 0, 0, 0, 205, 0, 0, 0, 206, 207, - 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 209, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 0, 0, 21, 22, 0, 23, 24, - 25, 26, 27, 28, 0, 29, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 30, 31, - 32, 241, 34, 0, 0, 0, 0, 0, 0, 0, - 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 36, 37, 0, - 38, 39, 40, 41, 0, 42, 43, 44, 45, 46, - 47, 48, 49, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, + 0, 0, 51, 52, 53, 0, 0, 0, 0, 54, + 0, 0, 0, 55, 56, 57, 58, 59, 0, 0, + 60, 61, 0, 62, 63, 64, 0, 65, 66, 0, + 0, 0, 67, 0, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 167, 168, 169, 170, + 171, 172, 0, 0, 0, 173, 0, 174, 175, 176, + 177, 178, 179, 180, 453, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 50, 0, 0, 0, 0, 51, 52, - 53, 0, 0, 0, 0, 54, 0, 0, 0, 55, - 56, 57, 58, 59, 0, 0, 60, 61, 0, 62, - 63, 64, 0, 65, 66, 0, 0, 0, 67, 0, - 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, - 78, 165, 166, 167, 168, 169, 170, 0, 0, 0, - 171, 0, 172, 173, 174, 175, 176, 177, 178, 444, + 0, 0, 0, 0, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 0, 0, 198, 199, 0, 200, 0, 0, 0, 201, + 202, 0, 203, 0, 204, 205, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, + 207, 0, 0, 0, 208, 209, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 0, 196, 197, 0, - 198, 0, 0, 0, 199, 200, 0, 201, 0, 202, - 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 204, 0, 0, 0, 205, 0, 0, 0, 206, - 207, 0, 208, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 209, 165, 166, 167, 168, 169, - 170, 445, 0, 0, 171, 0, 172, 173, 174, 175, - 176, 177, 178, 592, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, + 211, 167, 168, 169, 170, 171, 172, 454, 0, 0, + 173, 0, 174, 175, 176, 177, 178, 179, 180, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 0, - 0, 196, 197, 0, 198, 0, 0, 0, 199, 200, - 0, 201, 0, 202, 203, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 204, 0, 0, 0, 205, - 0, 0, 0, 206, 207, 0, 208, 0, 165, 166, - 167, 168, 169, 170, 0, 0, 0, 171, 209, 172, - 173, 174, 175, 176, 177, 178, 0, 0, 0, 0, + 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 0, 0, 198, 199, 0, + 200, 0, 0, 0, 201, 202, 0, 203, 0, 204, + 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 206, 0, 0, 0, 207, 0, 0, 0, 208, + 209, 0, 210, 0, 0, 167, 168, 169, 170, 171, + 172, 0, 0, 0, 173, 211, 174, 175, 176, 177, + 178, 179, 180, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 0, 0, 196, 197, 0, 198, 0, 0, - 0, 199, 200, 0, 201, 0, 202, 203, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, - 0, 0, 205, 0, 0, 0, 206, 207, 0, 208, - 0, 165, 166, 167, 168, 169, 170, 0, 0, 0, - 171, 209, 172, 173, 174, 175, 176, 177, 178, 0, + 0, 0, 0, 182, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 195, 196, 197, 0, + 0, 198, 199, 0, 200, 0, 0, 0, 201, 202, + 0, 203, 0, 204, 205, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 206, 0, 0, 0, 207, + 0, 0, 0, 208, 209, 0, 210, 0, 0, 167, + 168, 169, 170, 171, 172, 0, 0, 0, 173, 211, + 174, 175, 176, 177, 178, 179, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, - 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, - 191, 192, 193, 194, 195, 0, 0, 196, 197, 0, - 198, 0, 0, 0, 199, 200, 0, 201, 0, 202, - 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 204, 0, 0, 0, 205, 0, 0, 0, 206, - 207, 0, 208, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 209 + 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 0, 0, 198, 199, 0, 200, 0, + 0, 0, 201, 202, 0, 203, 0, 204, 205, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, + 0, 0, 0, 207, 0, 0, 0, 208, 209, 0, + 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 211 }; static const yytype_int16 yycheck[] = { - 10, 56, 57, 121, 152, 0, 386, 5, 24, 64, - 158, 21, 22, 368, 369, 370, 371, 454, 455, 456, - 24, 486, 37, 46, 37, 37, 37, 81, 383, 384, - 37, 29, 37, 329, 389, 81, 37, 37, 33, 81, - 37, 111, 77, 37, 37, 81, 81, 81, 344, 24, - 46, 113, 118, 349, 116, 45, 81, 5, 46, 94, - 95, 81, 72, 37, 38, 39, 129, 81, 113, 94, - 95, 85, 482, 483, 484, 485, 40, 41, 81, 124, - 72, 29, 124, 93, 129, 148, 81, 129, 498, 499, - 500, 81, 502, 68, 89, 505, 60, 92, 168, 165, - 162, 93, 539, 76, 124, 113, 96, 81, 52, 53, - 113, 81, 129, 167, 494, 495, 81, 81, 135, 136, - 115, 129, 168, 113, 479, 64, 96, 66, 164, 164, - 164, 486, 54, 488, 489, 490, 491, 492, 493, 164, - 163, 496, 438, 113, 167, 555, 611, 124, 113, 113, - 165, 167, 165, 165, 165, 127, 89, 129, 165, 92, - 165, 598, 599, 167, 165, 165, 81, 163, 165, 167, - 168, 165, 165, 169, 164, 163, 122, 89, 81, 327, - 92, 89, 115, 129, 92, 5, 6, 7, 8, 9, - 10, 89, 89, 13, 92, 92, 113, 252, 253, 254, - 113, 115, 116, 115, 259, 129, 113, 115, 263, 29, - 30, 31, 32, 33, 34, 80, 129, 115, 115, 143, - 113, 120, 129, 122, 117, 280, 5, 6, 7, 8, - 9, 10, 97, 98, 99, 100, 101, 102, 103, 104, - 105, 106, 168, 166, 109, 97, 172, 129, 81, 81, - 29, 30, 31, 32, 33, 34, 611, 109, 110, 165, - 112, 81, 168, 643, 97, 98, 99, 100, 139, 102, - 103, 104, 105, 81, 62, 63, 109, 24, 25, 26, - 27, 28, 637, 638, 113, 665, 94, 95, 122, 123, - 166, 125, 165, 169, 12, 168, 146, 147, 148, 42, - 43, 44, 55, 113, 57, 660, 661, 107, 108, 128, - 129, 167, 168, 6, 113, 81, 46, 0, 167, 36, - 65, 439, 48, 166, 81, 173, 81, 337, 165, 447, - 169, 449, 327, 24, 28, 35, 167, 167, 163, 167, - 167, 167, 81, 166, 166, 166, 81, 357, 166, 166, - 405, 166, 407, 408, 166, 410, 81, 166, 81, 166, - 1, 416, 113, 113, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 81, 81, 26, 27, 129, 29, 30, - 31, 32, 33, 34, 81, 36, 165, 113, 113, 81, - 81, 150, 81, 113, 113, 113, 113, 46, 49, 50, - 51, 52, 53, 24, 135, 24, 24, 113, 166, 81, - 61, 81, 166, 81, 164, 166, 56, 58, 166, 166, - 166, 117, 166, 166, 444, 445, 165, 78, 79, 166, - 81, 82, 83, 84, 166, 86, 87, 88, 89, 90, - 91, 92, 93, 166, 166, 166, 81, 113, 68, 514, - 166, 166, 166, 166, 81, 166, 168, 81, 81, 81, - 169, 59, 64, 114, 81, 165, 168, 113, 119, 120, - 121, 170, 170, 168, 168, 126, 113, 165, 81, 130, - 131, 132, 133, 134, 81, 150, 137, 138, 81, 140, - 141, 142, 166, 144, 145, 166, 81, 164, 149, 168, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 531, 163, 168, 166, 81, 166, 81, 171, 81, - 168, 81, 167, 167, 167, 167, 546, 165, 167, 5, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 81, 37, - 26, 27, 37, 29, 30, 31, 32, 33, 34, 171, - 36, 168, 168, 168, 168, 166, 166, 81, 33, 165, - 152, 89, 592, 49, 50, 51, 52, 53, 168, 544, - 355, 534, 116, 642, 663, 61, 271, 573, 222, 225, - 404, 510, 89, -1, -1, -1, -1, -1, 340, -1, - -1, -1, 78, 79, -1, 81, 82, 83, 84, -1, - 86, 87, 88, 89, 90, 91, 92, 93, -1, -1, + 10, 56, 57, 123, 154, 0, 5, 394, 26, 64, + 160, 21, 22, 376, 377, 378, 379, 463, 464, 465, + 26, 495, 39, 48, 39, 39, 39, 39, 391, 392, + 39, 39, 31, 336, 397, 83, 39, 39, 33, 39, + 79, 39, 83, 47, 83, 113, 83, 83, 351, 83, + 83, 48, 26, 356, 19, 20, 48, 96, 97, 96, + 97, 83, 72, 90, 115, 126, 93, 118, 131, 131, + 120, 56, 42, 43, 5, 39, 40, 41, 83, 83, + 64, 65, 145, 72, 94, 115, 169, 82, 150, 83, + 117, 115, 62, 126, 98, 90, 70, 62, 93, 115, + 31, 131, 548, 171, 126, 94, 78, 131, 115, 131, + 115, 115, 119, 83, 165, 131, 503, 504, 168, 83, + 131, 115, 117, 171, 83, 488, 137, 138, 167, 170, + 167, 167, 495, 167, 497, 498, 499, 500, 501, 502, + 83, 166, 505, 83, 447, 115, 620, 172, 44, 45, + 46, 168, 170, 168, 168, 168, 168, 115, 98, 168, + 168, 607, 608, 167, 170, 168, 168, 115, 168, 166, + 168, 170, 171, 170, 166, 115, 83, 131, 126, 124, + 87, 83, 83, 131, 334, 90, 131, 99, 93, 5, + 6, 7, 8, 9, 10, 96, 97, 13, 90, 111, + 112, 93, 114, 258, 259, 260, 491, 492, 493, 494, + 265, 66, 117, 68, 269, 31, 32, 33, 34, 35, + 36, 82, 507, 508, 509, 117, 511, 90, 141, 514, + 93, 286, 5, 6, 7, 8, 9, 10, 99, 100, + 101, 102, 103, 104, 105, 106, 107, 108, 169, 171, + 111, 172, 83, 175, 117, 54, 55, 620, 31, 32, + 33, 34, 35, 36, 129, 652, 131, 83, 99, 100, + 101, 102, 115, 104, 105, 106, 107, 115, 90, 564, + 111, 93, 168, 646, 647, 171, 296, 674, 24, 25, + 26, 27, 28, 168, 124, 125, 171, 127, 148, 149, + 150, 57, 122, 59, 124, 117, 669, 670, 117, 118, + 69, 109, 110, 130, 131, 12, 170, 171, 6, 115, + 83, 48, 0, 38, 170, 67, 50, 176, 448, 169, + 83, 83, 168, 170, 344, 26, 456, 172, 458, 334, + 62, 30, 170, 170, 170, 170, 83, 37, 169, 83, + 169, 169, 83, 115, 364, 169, 169, 115, 413, 169, + 415, 416, 169, 418, 169, 169, 1, 83, 115, 424, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 166, 83, 21, 22, 23, 24, + 25, 83, 131, 28, 29, 115, 31, 32, 33, 34, + 35, 36, 83, 38, 168, 115, 83, 152, 83, 83, + 115, 115, 115, 26, 48, 26, 51, 52, 53, 54, + 55, 26, 115, 137, 83, 169, 169, 83, 63, 169, + 83, 169, 167, 58, 60, 119, 169, 169, 83, 169, + 169, 169, 168, 453, 454, 80, 81, 115, 83, 84, + 85, 86, 169, 88, 89, 90, 91, 92, 93, 94, + 95, 169, 169, 169, 70, 83, 83, 169, 523, 169, + 169, 172, 61, 169, 83, 169, 171, 115, 115, 83, + 83, 116, 168, 168, 171, 152, 121, 122, 123, 171, + 173, 171, 83, 128, 173, 83, 83, 132, 133, 134, + 135, 136, 83, 169, 139, 140, 169, 142, 143, 144, + 167, 146, 147, 171, 169, 66, 151, 171, 153, 154, + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 540, 166, 171, 83, 83, 83, 169, 83, 170, 170, + 170, 170, 170, 83, 168, 555, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 174, 39, 21, 22, 23, 24, 25, 174, 39, 28, + 29, 171, 31, 32, 33, 34, 35, 36, 171, 38, + 82, 171, 33, 169, 169, 168, 154, 171, 543, 171, + 90, 601, 51, 52, 53, 54, 55, 553, 118, 651, + 672, 362, 582, 519, 63, 277, 228, 90, 231, 412, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 114, -1, - -1, -1, -1, 119, 120, 121, 5, -1, -1, -1, - 126, -1, -1, -1, 130, 131, 132, 133, 134, -1, - -1, 137, 138, -1, 140, 141, 142, -1, 144, 145, - 29, -1, -1, 149, -1, 151, 152, 153, 154, 155, - 156, 157, 158, 159, 160, 161, -1, 163, -1, -1, - 49, 50, 51, 52, 53, 54, -1, -1, -1, 58, - -1, 60, 61, 62, 63, 64, 65, 66, -1, -1, + -1, 80, 81, -1, 83, 84, 85, 86, 347, 88, + 89, 90, 91, 92, 93, 94, 95, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 97, 98, - 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, - 109, 110, 111, 112, -1, -1, 115, 116, -1, 118, - -1, -1, -1, 122, 123, -1, 125, -1, 127, 128, + -1, -1, -1, -1, -1, -1, -1, 116, -1, -1, + -1, -1, 121, 122, 123, 5, -1, -1, -1, 128, + -1, -1, -1, 132, 133, 134, 135, 136, -1, -1, + 139, 140, -1, 142, 143, 144, -1, 146, 147, -1, + -1, 31, 151, -1, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, -1, 166, -1, -1, + -1, 51, 52, 53, 54, 55, 56, -1, -1, -1, + 60, -1, 62, 63, 64, 65, 66, 67, 68, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 139, -1, -1, -1, 143, -1, -1, -1, 147, 148, - -1, 150, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 162, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, -1, -1, 26, 27, -1, 29, 30, - 31, 32, 33, 34, -1, 36, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 49, 50, - 51, 52, 53, -1, -1, -1, -1, -1, -1, -1, - 61, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 78, 79, -1, - 81, 82, 83, 84, -1, 86, 87, 88, 89, 90, - 91, 92, 93, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 83, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 99, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, -1, -1, 117, 118, -1, + 120, -1, -1, -1, 124, 125, -1, 127, -1, 129, + 130, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 141, -1, -1, -1, 145, -1, -1, -1, 149, + 150, -1, 152, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 165, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + -1, -1, 21, 22, 23, 24, 25, -1, -1, 28, + 29, -1, 31, 32, 33, 34, 35, 36, -1, 38, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 114, -1, -1, -1, -1, 119, 120, - 121, -1, -1, -1, -1, 126, -1, -1, -1, 130, - 131, 132, 133, 134, -1, -1, 137, 138, -1, 140, - 141, 142, -1, 144, 145, -1, -1, -1, 149, -1, - 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 49, 50, 51, 52, 53, 54, -1, -1, -1, - 58, -1, 60, 61, 62, 63, 64, 65, 66, 67, + -1, -1, 51, 52, 53, 54, 55, -1, -1, -1, + -1, -1, -1, -1, 63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, -1, -1, 115, 116, -1, - 118, -1, -1, -1, 122, 123, -1, 125, -1, 127, - 128, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 139, -1, -1, -1, 143, -1, -1, -1, 147, - 148, -1, 150, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 162, 49, 50, 51, 52, 53, - 54, 169, -1, -1, 58, -1, 60, 61, 62, 63, - 64, 65, 66, 67, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 81, -1, -1, + -1, 80, 81, -1, 83, 84, 85, 86, -1, 88, + 89, 90, 91, 92, 93, 94, 95, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 116, -1, -1, + -1, -1, 121, 122, 123, -1, -1, -1, -1, 128, + -1, -1, -1, 132, 133, 134, 135, 136, -1, -1, + 139, 140, -1, 142, 143, 144, -1, 146, 147, -1, + -1, -1, 151, -1, 153, 154, 155, 156, 157, 158, + 159, 160, 161, 162, 163, 164, 51, 52, 53, 54, + 55, 56, -1, -1, -1, 60, -1, 62, 63, 64, + 65, 66, 67, 68, 69, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 83, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 99, 100, 101, 102, 103, 104, + 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + -1, -1, 117, 118, -1, 120, -1, -1, -1, 124, + 125, -1, 127, -1, 129, 130, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 141, -1, -1, -1, + 145, -1, -1, -1, 149, 150, -1, 152, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 97, 98, 99, 100, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, -1, - -1, 115, 116, -1, 118, -1, -1, -1, 122, 123, - -1, 125, -1, 127, 128, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 139, -1, -1, -1, 143, - -1, -1, -1, 147, 148, -1, 150, -1, 49, 50, - 51, 52, 53, 54, -1, -1, -1, 58, 162, 60, - 61, 62, 63, 64, 65, 66, -1, -1, -1, -1, + 165, 51, 52, 53, 54, 55, 56, 172, -1, -1, + 60, -1, 62, 63, 64, 65, 66, 67, 68, 69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 97, 98, 99, 100, - 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, -1, -1, 115, 116, -1, 118, -1, -1, - -1, 122, 123, -1, 125, -1, 127, 128, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 139, -1, - -1, -1, 143, -1, -1, -1, 147, 148, -1, 150, - -1, 49, 50, 51, 52, 53, 54, -1, -1, -1, - 58, 162, 60, 61, 62, 63, 64, 65, 66, -1, + -1, -1, -1, 83, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 99, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, -1, -1, 117, 118, -1, + 120, -1, -1, -1, 124, 125, -1, 127, -1, 129, + 130, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 141, -1, -1, -1, 145, -1, -1, -1, 149, + 150, -1, 152, -1, -1, 51, 52, 53, 54, 55, + 56, -1, -1, -1, 60, 165, 62, 63, 64, 65, + 66, 67, 68, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 83, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, - 108, 109, 110, 111, 112, -1, -1, 115, 116, -1, - 118, -1, -1, -1, 122, 123, -1, 125, -1, 127, - 128, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 139, -1, -1, -1, 143, -1, -1, -1, 147, - 148, -1, 150, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 162 + -1, -1, -1, 99, 100, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 114, -1, + -1, 117, 118, -1, 120, -1, -1, -1, 124, 125, + -1, 127, -1, 129, 130, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 141, -1, -1, -1, 145, + -1, -1, -1, 149, 150, -1, 152, -1, -1, 51, + 52, 53, 54, 55, 56, -1, -1, -1, 60, 165, + 62, 63, 64, 65, 66, 67, 68, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 83, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, -1, -1, 117, 118, -1, 120, -1, + -1, -1, 124, 125, -1, 127, -1, 129, 130, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 141, + -1, -1, -1, 145, -1, -1, -1, 149, 150, -1, + 152, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 165 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing @@ -1785,73 +1798,74 @@ static const yytype_int16 yycheck[] = static const yytype_uint16 yystos[] = { 0, 1, 5, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 26, 27, 29, 30, 31, 32, 33, 34, 36, - 49, 50, 51, 52, 53, 61, 78, 79, 81, 82, - 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, - 114, 119, 120, 121, 126, 130, 131, 132, 133, 134, - 137, 138, 140, 141, 142, 144, 145, 149, 151, 152, - 153, 154, 155, 156, 157, 158, 159, 160, 161, 163, - 175, 176, 177, 178, 179, 180, 181, 182, 183, 186, - 187, 188, 189, 190, 193, 194, 195, 196, 197, 198, - 199, 200, 201, 202, 203, 204, 205, 206, 207, 218, - 219, 220, 221, 222, 223, 227, 228, 241, 242, 243, - 244, 246, 247, 253, 254, 258, 260, 261, 262, 264, - 266, 267, 268, 269, 271, 272, 273, 274, 275, 277, - 278, 280, 281, 282, 283, 284, 285, 286, 288, 292, - 293, 294, 76, 124, 245, 64, 66, 287, 77, 81, - 94, 95, 164, 224, 225, 49, 50, 51, 52, 53, - 54, 58, 60, 61, 62, 63, 64, 65, 66, 81, - 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 115, 116, 118, 122, - 123, 125, 127, 128, 139, 143, 147, 148, 150, 162, - 208, 298, 299, 299, 299, 40, 41, 60, 81, 113, - 248, 249, 250, 248, 248, 248, 248, 248, 80, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 109, - 229, 52, 178, 54, 52, 53, 166, 81, 81, 115, - 116, 259, 122, 123, 125, 265, 122, 129, 263, 127, - 129, 256, 113, 148, 256, 256, 81, 124, 270, 129, - 135, 136, 257, 81, 139, 113, 117, 255, 113, 276, - 143, 256, 113, 279, 146, 147, 148, 24, 68, 208, - 12, 6, 113, 81, 113, 116, 162, 46, 0, 177, - 46, 163, 5, 179, 184, 207, 218, 219, 220, 221, - 222, 294, 179, 218, 219, 220, 221, 222, 81, 208, - 179, 218, 219, 220, 221, 222, 227, 167, 192, 211, - 212, 213, 179, 214, 215, 296, 297, 299, 65, 252, - 297, 214, 297, 192, 211, 48, 173, 62, 63, 211, - 166, 81, 81, 81, 164, 165, 167, 169, 24, 28, - 167, 167, 167, 167, 35, 249, 245, 81, 166, 166, - 166, 166, 81, 97, 98, 99, 100, 102, 103, 104, - 105, 109, 237, 166, 166, 166, 166, 107, 108, 166, - 37, 38, 39, 81, 230, 163, 55, 57, 289, 290, - 291, 81, 81, 113, 113, 113, 256, 113, 124, 256, - 113, 256, 81, 124, 129, 81, 128, 256, 81, 256, - 81, 81, 257, 113, 165, 256, 113, 81, 150, 81, - 81, 113, 113, 113, 113, 46, 168, 179, 211, 24, - 191, 212, 135, 295, 67, 169, 299, 24, 295, 24, - 81, 96, 113, 226, 166, 166, 166, 81, 94, 95, - 224, 45, 81, 96, 113, 164, 209, 210, 299, 164, - 185, 113, 42, 43, 44, 251, 81, 81, 113, 166, - 81, 85, 240, 240, 240, 240, 166, 169, 166, 166, - 166, 166, 166, 166, 166, 166, 166, 236, 240, 240, - 238, 240, 238, 166, 166, 240, 81, 81, 56, 58, - 165, 255, 256, 256, 113, 256, 256, 81, 256, 118, - 165, 113, 68, 166, 166, 168, 168, 214, 81, 167, - 299, 299, 214, 167, 216, 214, 226, 226, 226, 166, - 81, 81, 81, 164, 165, 168, 169, 81, 165, 168, - 168, 81, 81, 168, 168, 239, 240, 236, 236, 236, - 236, 239, 81, 240, 240, 240, 240, 240, 240, 238, - 238, 240, 37, 165, 236, 236, 236, 236, 170, 170, - 236, 59, 291, 256, 113, 113, 165, 150, 81, 81, - 113, 81, 67, 299, 81, 217, 215, 226, 166, 166, - 81, 210, 299, 164, 168, 168, 236, 37, 37, 37, - 37, 166, 237, 37, 37, 37, 37, 97, 109, 110, - 112, 231, 232, 64, 233, 37, 81, 166, 168, 299, - 165, 168, 226, 226, 81, 37, 239, 167, 167, 167, - 167, 171, 165, 167, 171, 81, 81, 240, 240, 111, - 168, 81, 234, 235, 37, 231, 238, 37, 168, 168, - 166, 166, 168, 165, 168, 172, 240, 240, 234, 238, - 168, 168 + 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, + 25, 28, 29, 31, 32, 33, 34, 35, 36, 38, + 51, 52, 53, 54, 55, 63, 80, 81, 83, 84, + 85, 86, 88, 89, 90, 91, 92, 93, 94, 95, + 116, 121, 122, 123, 128, 132, 133, 134, 135, 136, + 139, 140, 142, 143, 144, 146, 147, 151, 153, 154, + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, + 166, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 190, 191, 192, 193, 194, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, + 212, 223, 224, 225, 226, 227, 228, 232, 233, 246, + 247, 248, 249, 251, 252, 258, 259, 263, 265, 266, + 267, 269, 271, 272, 273, 274, 276, 277, 278, 279, + 280, 282, 283, 285, 286, 287, 288, 289, 290, 291, + 293, 297, 298, 299, 78, 126, 250, 66, 68, 292, + 79, 83, 96, 97, 167, 229, 230, 51, 52, 53, + 54, 55, 56, 60, 62, 63, 64, 65, 66, 67, + 68, 83, 99, 100, 101, 102, 103, 104, 105, 106, + 107, 108, 109, 110, 111, 112, 113, 114, 117, 118, + 120, 124, 125, 127, 129, 130, 141, 145, 149, 150, + 152, 165, 213, 303, 304, 19, 20, 62, 187, 304, + 304, 42, 43, 62, 83, 115, 253, 254, 255, 253, + 253, 253, 253, 253, 82, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 111, 234, 54, 181, 56, + 54, 55, 169, 83, 83, 117, 118, 264, 124, 125, + 127, 270, 124, 131, 268, 129, 131, 261, 115, 150, + 261, 261, 83, 126, 275, 131, 137, 138, 262, 83, + 141, 115, 119, 260, 115, 281, 145, 261, 115, 284, + 148, 149, 150, 26, 70, 213, 69, 12, 6, 115, + 83, 115, 118, 165, 48, 0, 180, 48, 166, 5, + 182, 188, 212, 223, 224, 225, 226, 227, 299, 182, + 223, 224, 225, 226, 227, 83, 213, 182, 223, 224, + 225, 226, 227, 232, 170, 196, 216, 217, 218, 182, + 219, 220, 301, 302, 304, 67, 257, 302, 219, 302, + 196, 216, 50, 176, 64, 65, 216, 169, 83, 83, + 83, 167, 168, 170, 172, 62, 26, 30, 170, 170, + 170, 170, 37, 254, 250, 83, 169, 169, 169, 169, + 83, 99, 100, 101, 102, 104, 105, 106, 107, 111, + 242, 169, 169, 169, 169, 109, 110, 169, 39, 40, + 41, 83, 235, 166, 57, 59, 294, 295, 296, 83, + 83, 115, 115, 115, 261, 115, 126, 261, 115, 261, + 83, 126, 131, 83, 130, 261, 83, 261, 83, 83, + 262, 115, 168, 261, 115, 83, 152, 304, 83, 83, + 115, 115, 115, 115, 48, 171, 182, 216, 26, 195, + 217, 137, 300, 69, 172, 304, 26, 300, 26, 83, + 98, 115, 231, 169, 169, 169, 83, 96, 97, 229, + 47, 83, 98, 115, 167, 214, 215, 304, 167, 189, + 115, 44, 45, 46, 256, 83, 83, 115, 169, 83, + 87, 245, 245, 245, 245, 169, 172, 169, 169, 169, + 169, 169, 169, 169, 169, 169, 241, 245, 245, 243, + 245, 243, 169, 169, 245, 83, 83, 58, 60, 168, + 260, 261, 261, 115, 261, 261, 83, 261, 120, 168, + 115, 70, 169, 169, 171, 171, 219, 83, 170, 304, + 304, 219, 170, 221, 219, 231, 231, 231, 169, 83, + 83, 83, 167, 168, 171, 172, 83, 168, 171, 171, + 83, 83, 171, 171, 244, 245, 241, 241, 241, 241, + 244, 83, 245, 245, 245, 245, 245, 245, 243, 243, + 245, 39, 168, 241, 241, 241, 241, 173, 173, 241, + 61, 296, 261, 115, 115, 168, 152, 83, 83, 115, + 83, 69, 304, 83, 222, 220, 231, 169, 169, 83, + 215, 304, 167, 171, 171, 241, 39, 39, 39, 39, + 169, 242, 39, 39, 39, 39, 99, 111, 112, 114, + 236, 237, 66, 238, 39, 83, 169, 171, 304, 168, + 171, 231, 231, 83, 39, 244, 170, 170, 170, 170, + 174, 168, 170, 174, 83, 83, 245, 245, 113, 171, + 83, 239, 240, 39, 236, 243, 39, 171, 171, 169, + 169, 171, 168, 171, 175, 245, 245, 239, 243, 171, + 171 }; #define yyerrok (yyerrstatus = 0) @@ -2892,24 +2906,19 @@ yyreduce: { result->cur_stmt_type_ = OBPROXY_T_RENAME; ;} break; - case 57: - - {;} - break; - case 58: - {;} + { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_TABLE; ;} break; case 59: - {;} + { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_INDEX; ;} break; case 60: - {;} + { result->sub_stmt_type_ = OBPROXY_T_SUB_CREATE_INDEX; ;} break; case 61: @@ -2934,6 +2943,26 @@ yyreduce: case 65: + {;} + break; + + case 66: + + {;} + break; + + case 67: + + {;} + break; + + case 68: + + {;} + break; + + case 69: + { ObProxyTextPsExecuteParseNode *node = NULL; malloc_execute_parse_node(node); @@ -2942,7 +2971,7 @@ yyreduce: ;} break; - case 66: + case 70: { ObProxyTextPsExecuteParseNode *node = NULL; @@ -2952,7 +2981,7 @@ yyreduce: ;} break; - case 67: + case 71: { result->text_ps_inner_stmt_type_ = OBPROXY_T_TEXT_PS_PREPARE; @@ -2960,13 +2989,13 @@ yyreduce: ;} break; - case 68: + case 72: { ;} break; - case 69: + case 73: { result->cur_stmt_type_ = OBPROXY_T_TEXT_PS_EXECUTE; @@ -2974,7 +3003,7 @@ yyreduce: ;} break; - case 70: + case 74: { result->cur_stmt_type_ = OBPROXY_T_TEXT_PS_EXECUTE; @@ -2982,127 +3011,137 @@ yyreduce: ;} break; - case 71: + case 75: { result->cur_stmt_type_ = OBPROXY_T_GRANT; ;} break; - case 72: + case 76: { result->cur_stmt_type_ = OBPROXY_T_REVOKE; ;} break; - case 73: + case 77: { result->cur_stmt_type_ = OBPROXY_T_ANALYZE; ;} break; - case 74: + case 78: { result->cur_stmt_type_ = OBPROXY_T_PURGE; ;} break; - case 75: + case 79: { result->cur_stmt_type_ = OBPROXY_T_FLASHBACK; ;} break; - case 76: + case 80: { result->cur_stmt_type_ = OBPROXY_T_COMMENT; ;} break; - case 77: + case 81: { result->cur_stmt_type_ = OBPROXY_T_AUDIT; ;} break; - case 78: + case 82: { result->cur_stmt_type_ = OBPROXY_T_NOAUDIT; ;} break; - case 81: + case 85: {;} break; - case 82: + case 86: {;} break; - case 83: + case 87: {;} break; - case 88: + case 92: { result->cur_stmt_type_ = OBPROXY_T_SELECT_TX_RO; ;} break; - case 91: + case 96: + + { result->col_name_ = (yyvsp[(3) - (3)].str); ;} + break; + + case 97: { result->cur_stmt_type_ = OBPROXY_T_SET_AC_0; ;} break; - case 93: + case 99: {;} break; - case 94: + case 100: {;} break; - case 95: + case 101: {;} break; - case 96: + case 102: {;} break; - case 97: + case 103: + + {;} + break; + + case 104: {;} break; - case 98: + case 105: {;} break; - case 99: + case 106: {;} break; - case 100: + case 107: { result->sub_stmt_type_ = OBPROXY_T_SUB_SELECT_DATABASE; ;} break; - case 101: + case 108: { result->sub_stmt_type_ = OBPROXY_T_SUB_SHOW_DATABASES; ;} break; - case 102: + case 109: { result->sub_stmt_type_ = OBPROXY_T_SUB_SHOW_TABLES; ;} break; - case 103: + case 110: { result->sub_stmt_type_ = OBPROXY_T_SUB_SHOW_CREATE_TABLE; ;} break; - case 104: + case 111: { result->cur_stmt_type_ = OBPROXY_T_DESC; @@ -3110,17 +3149,17 @@ yyreduce: ;} break; - case 105: + case 112: { result->sub_stmt_type_ = OBPROXY_T_SUB_SHOW_DB_VERSION; ;} break; - case 106: + case 113: { result->sub_stmt_type_ = OBPROXY_T_SUB_SHOW_TOPOLOGY; ;} break; - case 107: + case 114: { SET_ICMD_ONE_STRING((yyvsp[(3) - (3)].str)); @@ -3128,7 +3167,7 @@ yyreduce: ;} break; - case 108: + case 115: { SET_ICMD_SECOND_STRING((yyvsp[(5) - (5)].str)); @@ -3136,7 +3175,7 @@ yyreduce: ;} break; - case 109: + case 116: { SET_ICMD_ONE_STRING((yyvsp[(3) - (7)].str)); @@ -3145,12 +3184,12 @@ yyreduce: ;} break; - case 110: + case 117: { result->cur_stmt_type_ = OBPROXY_T_SELECT_ROUTE_ADDR; ;} break; - case 111: + case 118: { result->cur_stmt_type_ = OBPROXY_T_SET_ROUTE_ADDR; @@ -3158,44 +3197,44 @@ yyreduce: ;} break; - case 112: + case 119: {;} break; - case 113: + case 120: {;} break; - case 114: + case 121: {;} break; - case 115: + case 122: {;} break; - case 116: + case 123: {;} break; - case 117: + case 124: {;} break; - case 119: + case 126: { result->table_info_.table_name_ = (yyvsp[(1) - (1)].str); ;} break; - case 120: + case 127: { result->table_info_.package_name_ = (yyvsp[(1) - (3)].str); @@ -3203,7 +3242,7 @@ yyreduce: ;} break; - case 121: + case 128: { result->table_info_.database_name_ = (yyvsp[(1) - (5)].str); @@ -3212,14 +3251,14 @@ yyreduce: ;} break; - case 122: + case 129: { result->call_parse_info_.node_count_ = 0; ;} break; - case 123: + case 130: { result->call_parse_info_.node_count_ = 0; @@ -3227,14 +3266,14 @@ yyreduce: ;} break; - case 124: + case 131: { add_call_node(result->call_parse_info_, (yyvsp[(3) - (3)].node)); ;} break; - case 125: + case 132: { malloc_call_node((yyval.node), CALL_TOKEN_STR_VAL); @@ -3242,7 +3281,7 @@ yyreduce: ;} break; - case 126: + case 133: { malloc_call_node((yyval.node), CALL_TOKEN_INT_VAL); @@ -3250,7 +3289,7 @@ yyreduce: ;} break; - case 127: + case 134: { malloc_call_node((yyval.node), CALL_TOKEN_NUMBER_VAL); @@ -3258,7 +3297,7 @@ yyreduce: ;} break; - case 128: + case 135: { malloc_call_node((yyval.node), CALL_TOKEN_USER_VAR); @@ -3266,7 +3305,7 @@ yyreduce: ;} break; - case 129: + case 136: { malloc_call_node((yyval.node), CALL_TOKEN_SYS_VAR); @@ -3274,7 +3313,7 @@ yyreduce: ;} break; - case 130: + case 137: { result->placeholder_list_idx_++; @@ -3283,7 +3322,7 @@ yyreduce: ;} break; - case 144: + case 151: { handle_stmt_end(result); @@ -3291,7 +3330,7 @@ yyreduce: ;} break; - case 149: + case 156: { handle_stmt_end(result); @@ -3299,56 +3338,56 @@ yyreduce: ;} break; - case 153: + case 160: { add_set_var_node(result->set_parse_info_, (yyvsp[(4) - (4)].var_node), (yyvsp[(2) - (4)].str), SET_VAR_USER); ;} break; - case 154: + case 161: { add_set_var_node(result->set_parse_info_, (yyvsp[(6) - (6)].var_node), (yyvsp[(4) - (6)].str), SET_VAR_SYS); ;} break; - case 155: + case 162: { add_set_var_node(result->set_parse_info_, (yyvsp[(4) - (4)].var_node), (yyvsp[(2) - (4)].str), SET_VAR_SYS); ;} break; - case 156: + case 163: { add_set_var_node(result->set_parse_info_, (yyvsp[(5) - (5)].var_node), (yyvsp[(3) - (5)].str), SET_VAR_SYS); ;} break; - case 157: + case 164: { add_set_var_node(result->set_parse_info_, (yyvsp[(6) - (6)].var_node), (yyvsp[(4) - (6)].str), SET_VAR_SYS); ;} break; - case 158: + case 165: { add_set_var_node(result->set_parse_info_, (yyvsp[(4) - (4)].var_node), (yyvsp[(2) - (4)].str), SET_VAR_SYS); ;} break; - case 159: + case 166: { add_set_var_node(result->set_parse_info_, (yyvsp[(3) - (3)].var_node), (yyvsp[(1) - (3)].str), SET_VAR_SYS); ;} break; - case 160: + case 167: { malloc_set_var_node((yyval.var_node), SET_VALUE_TYPE_STR); @@ -3356,7 +3395,7 @@ yyreduce: ;} break; - case 161: + case 168: { malloc_set_var_node((yyval.var_node), SET_VALUE_TYPE_INT); @@ -3364,7 +3403,7 @@ yyreduce: ;} break; - case 162: + case 169: { malloc_set_var_node((yyval.var_node), SET_VALUE_TYPE_NUMBER); @@ -3372,42 +3411,42 @@ yyreduce: ;} break; - case 165: + case 172: {;} break; - case 166: + case 173: {;} break; - case 167: + case 174: { result->dbmesh_route_info_.tb_idx_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 168: + case 175: { result->dbmesh_route_info_.table_name_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 169: + case 176: { result->dbmesh_route_info_.group_idx_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 170: + case 177: { result->dbmesh_route_info_.es_idx_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 171: + case 178: { result->dbmesh_route_info_.testload_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 172: + case 179: { malloc_shard_column_node((yyval.shard_node), (yyvsp[(2) - (7)].str), (yyvsp[(3) - (7)].str), DBMESH_TOKEN_STR_VAL); @@ -3416,52 +3455,52 @@ yyreduce: ;} break; - case 173: + case 180: { result->trace_id_ = (yyvsp[(4) - (6)].str); ;} break; - case 174: + case 181: { result->rpc_id_ = (yyvsp[(4) - (6)].str); ;} break; - case 175: + case 182: { result->dbmesh_route_info_.tnt_id_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 176: + case 183: { result->dbmesh_route_info_.disaster_status_str_ = (yyvsp[(4) - (6)].str); ;} break; - case 177: + case 184: {;} break; - case 178: + case 185: {;} break; - case 179: + case 186: {;} break; - case 181: + case 188: { result->has_simple_route_info_ = true; result->simple_route_info_.table_name_ = (yyvsp[(2) - (2)].str); ;} break; - case 182: + case 189: { result->simple_route_info_.part_key_ = (yyvsp[(2) - (2)].str); ;} break; - case 186: + case 193: { result->dbp_route_info_.has_group_info_ = true; @@ -3469,7 +3508,7 @@ yyreduce: ;} break; - case 187: + case 194: { result->dbp_route_info_.has_group_info_ = true; @@ -3477,37 +3516,37 @@ yyreduce: ;} break; - case 188: + case 195: { result->dbp_route_info_.scan_all_ = true; ;} break; - case 189: + case 196: { result->dbp_route_info_.scan_all_ = true; ;} break; - case 190: + case 197: {result->dbp_route_info_.has_shard_key_ = true;;} break; - case 191: + case 198: { result->trace_id_ = (yyvsp[(3) - (4)].str); ;} break; - case 192: + case 199: { result->trace_id_ = (yyvsp[(3) - (6)].str); result->rpc_id_ = (yyvsp[(5) - (6)].str); ;} break; - case 193: + case 200: {;} break; - case 195: + case 202: { if (result->dbp_route_info_.shard_key_count_ < OBPROXY_MAX_DBP_SHARD_KEY_NUM) { @@ -3518,52 +3557,52 @@ yyreduce: ;} break; - case 198: + case 205: { result->dbmesh_route_info_.group_idx_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 199: + case 206: { result->dbmesh_route_info_.tb_idx_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 200: + case 207: { result->dbmesh_route_info_.table_name_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 201: + case 208: { result->dbmesh_route_info_.es_idx_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 202: + case 209: { result->dbmesh_route_info_.testload_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 203: + case 210: { result->trace_id_ = (yyvsp[(3) - (3)].str); ;} break; - case 204: + case 211: { result->rpc_id_ = (yyvsp[(3) - (3)].str); ;} break; - case 205: + case 212: { result->dbmesh_route_info_.tnt_id_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 206: + case 213: { result->dbmesh_route_info_.disaster_status_str_ = (yyvsp[(3) - (3)].str); ;} break; - case 207: + case 214: { malloc_shard_column_node((yyval.shard_node), (yyvsp[(1) - (5)].str), (yyvsp[(3) - (5)].str), DBMESH_TOKEN_STR_VAL); @@ -3572,32 +3611,32 @@ yyreduce: ;} break; - case 208: + case 215: {;} break; - case 209: + case 216: { (yyval.str).str_ = NULL; (yyval.str).str_len_ = 0; ;} break; - case 211: + case 218: { (yyval.str).str_ = NULL; (yyval.str).str_len_ = 0; ;} break; - case 233: + case 240: { result->query_timeout_ = (yyvsp[(3) - (4)].num); ;} break; - case 234: + case 241: {;} break; - case 236: + case 243: { add_hint_index(result->dbmesh_route_info_, (yyvsp[(3) - (5)].str)); @@ -3605,75 +3644,75 @@ yyreduce: ;} break; - case 237: + case 244: {;} break; - case 238: + case 245: {;} break; - case 239: + case 246: {;} break; - case 240: + case 247: {;} break; - case 241: + case 248: {;} break; - case 242: + case 249: { SET_READ_CONSISTENCY(OBPROXY_READ_CONSISTENCY_WEAK); ;} break; - case 243: + case 250: { SET_READ_CONSISTENCY(OBPROXY_READ_CONSISTENCY_STRONG); ;} break; - case 244: + case 251: { SET_READ_CONSISTENCY(OBPROXY_READ_CONSISTENCY_FROZEN); ;} break; - case 247: + case 254: { result->cur_stmt_type_ = OBPROXY_T_SHOW_WARNINGS; ;} break; - case 248: + case 255: { result->cur_stmt_type_ = OBPROXY_T_SHOW_ERRORS; ;} break; - case 249: + case 256: { result->cur_stmt_type_ = OBPROXY_T_SHOW_TRACE; ;} break; - case 273: + case 280: { ;} break; - case 274: + case 281: { result->cmd_info_.integer_[2] = (yyvsp[(2) - (2)].num);/*row*/ ;} break; - case 275: + case 282: { result->cmd_info_.integer_[1] = (yyvsp[(2) - (4)].num);/*offset*/ @@ -3681,7 +3720,7 @@ yyreduce: ;} break; - case 276: + case 283: { result->cmd_info_.integer_[1] = (yyvsp[(4) - (4)].num);/*offset*/ @@ -3689,342 +3728,342 @@ yyreduce: ;} break; - case 277: + case 284: {;} break; - case 278: + case 285: { result->cmd_info_.string_[0] = (yyvsp[(2) - (2)].str);;} break; - case 279: + case 286: {;} break; - case 280: + case 287: { result->cmd_info_.string_[1] = (yyvsp[(2) - (2)].str);;} break; - case 282: + case 289: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_NET_THREAD); ;} break; - case 283: + case 290: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_NET_CONNECTION); ;} break; - case 284: + case 291: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_NET_CONNECTION, (yyvsp[(2) - (3)].num)); ;} break; - case 285: + case 292: {;} break; - case 286: + case 293: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_CONFIG_DIFF); ;} break; - case 287: + case 294: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_CONFIG_DIFF_USER); ;} break; - case 288: + case 295: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_LIST); ;} break; - case 290: + case 297: {SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_GLOBAL_SESSION_LIST);;} break; - case 291: + case 298: {SET_ICMD_SUB_AND_ONE_STRING(OBPROXY_T_SUB_GLOBAL_SESSION_LIST_INFO, (yyvsp[(2) - (2)].str));;} break; - case 292: + case 299: {SET_ICMD_SUB_AND_ONE_STRING(OBPROXY_T_SUB_GLOBAL_SESSION_LIST_LIKE, (yyvsp[(3) - (3)].str));;} break; - case 293: + case 300: {SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_GLOBAL_SESSION_LIST_INFO_ALL);;} break; - case 294: + case 301: {result->cmd_info_.string_[0] = (yyvsp[(2) - (2)].str);;} break; - case 296: + case 303: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_LIST_INTERNAL); ;} break; - case 297: + case 304: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_ATTRIBUTE); ;} break; - case 298: + case 305: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_SESSION_ATTRIBUTE, (yyvsp[(2) - (3)].num)); ;} break; - case 299: + case 306: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_STAT); ;} break; - case 300: + case 307: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_SESSION_STAT, (yyvsp[(2) - (3)].num)); ;} break; - case 301: + case 308: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_VARIABLES_LOCAL); ;} break; - case 302: + case 309: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_SESSION_VARIABLES_LOCAL, (yyvsp[(2) - (3)].num)); ;} break; - case 303: + case 310: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SESSION_VARIABLES_ALL); ;} break; - case 304: + case 311: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_SESSION_VARIABLES_ALL, (yyvsp[(3) - (4)].num)); ;} break; - case 305: + case 312: {;} break; - case 306: + case 313: { SET_ICMD_ONE_ID((yyvsp[(2) - (2)].num)); ;} break; - case 307: + case 314: {;} break; - case 308: + case 315: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_INFO_IDC); ;} break; - case 309: + case 316: {;} break; - case 311: + case 318: {;} break; - case 312: + case 319: { SET_ICMD_ONE_STRING((yyvsp[(1) - (1)].str)); ;} break; - case 313: + case 320: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_CONGEST_ALL);;} break; - case 314: + case 321: { SET_ICMD_SUB_AND_ONE_STRING(OBPROXY_T_SUB_CONGEST_ALL, (yyvsp[(2) - (2)].str));;} break; - case 315: + case 322: {;} break; - case 316: + case 323: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_ROUTE_ROUTINE); ;} break; - case 317: + case 324: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_ROUTE_PARTITION); ;} break; - case 318: + case 325: {;} break; - case 319: + case 326: { SET_ICMD_ONE_STRING((yyvsp[(2) - (2)].str)); ;} break; - case 320: + case 327: {;} break; - case 321: + case 328: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_MEMORY_OBJPOOL); ;} break; - case 322: + case 329: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_SQLAUDIT_AUDIT_ID); ;} break; - case 323: + case 330: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_SQLAUDIT_SM_ID, (yyvsp[(2) - (2)].num)); ;} break; - case 325: + case 332: {;} break; - case 326: + case 333: { SET_ICMD_SECOND_ID((yyvsp[(1) - (1)].num)); ;} break; - case 327: + case 334: { SET_ICMD_TWO_ID((yyvsp[(3) - (3)].num), (yyvsp[(1) - (3)].num)); ;} break; - case 328: + case 335: { SET_ICMD_TWO_ID((yyvsp[(3) - (5)].num), (yyvsp[(1) - (5)].num)); SET_ICMD_ONE_STRING((yyvsp[(5) - (5)].str)); ;} break; - case 329: + case 336: {;} break; - case 330: + case 337: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_STAT_REFRESH); ;} break; - case 332: + case 339: {;} break; - case 333: + case 340: { SET_ICMD_ONE_ID((yyvsp[(1) - (1)].num)); ;} break; - case 334: + case 341: { SET_ICMD_SUB_AND_TWO_ID(OBPROXY_T_SUB_TRACE_LIMIT, (yyvsp[(1) - (2)].num),(yyvsp[(2) - (2)].num)); ;} break; - case 335: + case 342: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_INFO_BINARY); ;} break; - case 336: + case 343: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_INFO_UPGRADE); ;} break; - case 337: + case 344: { SET_ICMD_SUB_TYPE(OBPROXY_T_SUB_INFO_IDC); ;} break; - case 338: + case 345: { SET_ICMD_ONE_STRING((yyvsp[(3) - (4)].str)); ;} break; - case 339: + case 346: { SET_ICMD_TWO_STRING((yyvsp[(3) - (5)].str), (yyvsp[(5) - (5)].str)); ;} break; - case 340: + case 347: { SET_ICMD_CONFIG_INT_VALUE((yyvsp[(3) - (5)].str), (yyvsp[(5) - (5)].num)); ;} break; - case 341: + case 348: { SET_ICMD_ONE_STRING((yyvsp[(3) - (3)].str)); ;} break; - case 342: + case 349: {;} break; - case 343: + case 350: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_KILL_CS, (yyvsp[(2) - (2)].num)); ;} break; - case 344: + case 351: { SET_ICMD_SUB_AND_TWO_ID(OBPROXY_T_SUB_KILL_SS, (yyvsp[(2) - (3)].num), (yyvsp[(3) - (3)].num)); ;} break; - case 345: + case 352: {SET_ICMD_TYPE_STRING_INT_VALUE(OBPROXY_T_SUB_KILL_GLOBAL_SS_ID, (yyvsp[(2) - (3)].str),(yyvsp[(3) - (3)].num));;} break; - case 346: + case 353: {SET_ICMD_SUB_AND_ONE_STRING(OBPROXY_T_SUB_KILL_GLOBAL_SS_DBKEY, (yyvsp[(2) - (2)].str));;} break; - case 347: + case 354: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_KILL_CONNECTION, (yyvsp[(2) - (2)].num)); ;} break; - case 348: + case 355: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_KILL_CONNECTION, (yyvsp[(3) - (3)].num)); ;} break; - case 349: + case 356: { SET_ICMD_SUB_AND_ONE_ID(OBPROXY_T_SUB_KILL_QUERY, (yyvsp[(3) - (3)].num)); ;} break; - case 352: + case 359: { result->has_anonymous_block_ = false ; @@ -4032,22 +4071,22 @@ yyreduce: ;} break; - case 353: + case 360: { result->cur_stmt_type_ = OBPROXY_T_BEGIN; ;} break; - case 354: + case 361: { result->cur_stmt_type_ = OBPROXY_T_BEGIN; ;} break; - case 355: + case 362: { result->cur_stmt_type_ = OBPROXY_T_BEGIN; ;} break; - case 362: + case 369: { result->cur_stmt_type_ = OBPROXY_T_USE_DB; @@ -4055,27 +4094,27 @@ yyreduce: ;} break; - case 363: + case 370: { result->cur_stmt_type_ = OBPROXY_T_HELP; ;} break; - case 365: + case 372: {;} break; - case 366: + case 373: { result->part_name_ = (yyvsp[(2) - (2)].str); ;} break; - case 367: + case 374: { result->part_name_ = (yyvsp[(3) - (4)].str); ;} break; - case 368: + case 375: { handle_stmt_end(result); @@ -4083,14 +4122,14 @@ yyreduce: ;} break; - case 369: + case 376: { result->table_info_.table_name_ = (yyvsp[(1) - (1)].str); ;} break; - case 370: + case 377: { result->table_info_.database_name_ = (yyvsp[(1) - (3)].str); @@ -4098,7 +4137,7 @@ yyreduce: ;} break; - case 371: + case 378: { UPDATE_ALIAS_NAME((yyvsp[(2) - (2)].str)); @@ -4106,7 +4145,7 @@ yyreduce: ;} break; - case 372: + case 379: { UPDATE_ALIAS_NAME((yyvsp[(4) - (4)].str)); @@ -4115,7 +4154,7 @@ yyreduce: ;} break; - case 373: + case 380: { UPDATE_ALIAS_NAME((yyvsp[(3) - (3)].str)); @@ -4123,7 +4162,7 @@ yyreduce: ;} break; - case 374: + case 381: { UPDATE_ALIAS_NAME((yyvsp[(5) - (5)].str)); diff --git a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.h b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.h index 3e9ba4a7a1c68dfb2496cee346c5432d51360c02..6740aaa8a50ee6e4ac795e51bf5d70cffd39e6c0 100644 --- a/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.h +++ b/src/obproxy/opsql/parser/ob_proxy_parser_utf8_tab.h @@ -72,150 +72,153 @@ extern int ob_proxy_parser_utf8_yydebug; ALTER = 271, TRUNCATE = 272, RENAME = 273, - GRANT = 274, - REVOKE = 275, - ANALYZE = 276, - PURGE = 277, - COMMENT = 278, - FROM = 279, - DUAL = 280, - PREPARE = 281, - EXECUTE = 282, - USING = 283, - SELECT_HINT_BEGIN = 284, - UPDATE_HINT_BEGIN = 285, - DELETE_HINT_BEGIN = 286, - INSERT_HINT_BEGIN = 287, - REPLACE_HINT_BEGIN = 288, - MERGE_HINT_BEGIN = 289, - HINT_END = 290, - COMMENT_BEGIN = 291, - COMMENT_END = 292, - ROUTE_TABLE = 293, - ROUTE_PART_KEY = 294, - QUERY_TIMEOUT = 295, - READ_CONSISTENCY = 296, - WEAK = 297, - STRONG = 298, - FROZEN = 299, - PLACE_HOLDER = 300, - END_P = 301, - ERROR = 302, - WHEN = 303, - FLASHBACK = 304, - AUDIT = 305, - NOAUDIT = 306, - BEGI = 307, - START = 308, - TRANSACTION = 309, - READ = 310, - ONLY = 311, - WITH = 312, - CONSISTENT = 313, - SNAPSHOT = 314, - INDEX = 315, - XA = 316, - WARNINGS = 317, - ERRORS = 318, - TRACE = 319, - QUICK = 320, - COUNT = 321, - AS = 322, - WHERE = 323, - VALUES = 324, - ORDER = 325, - GROUP = 326, - HAVING = 327, - INTO = 328, - UNION = 329, - FOR = 330, - TX_READ_ONLY = 331, - AUTOCOMMIT_0 = 332, - SELECT_OBPROXY_ROUTE_ADDR = 333, - SET_OBPROXY_ROUTE_ADDR = 334, - NAME_OB_DOT = 335, - NAME_OB = 336, - EXPLAIN = 337, - DESC = 338, - DESCRIBE = 339, - NAME_STR = 340, - USE = 341, - HELP = 342, - SET_NAMES = 343, - SET_CHARSET = 344, - SET_PASSWORD = 345, - SET_DEFAULT = 346, - SET_OB_READ_CONSISTENCY = 347, - SET_TX_READ_ONLY = 348, - GLOBAL = 349, - SESSION = 350, - NUMBER_VAL = 351, - GROUP_ID = 352, - TABLE_ID = 353, - ELASTIC_ID = 354, - TESTLOAD = 355, - ODP_COMMENT = 356, - TNT_ID = 357, - DISASTER_STATUS = 358, - TRACE_ID = 359, - RPC_ID = 360, - DBP_COMMENT = 361, - ROUTE_TAG = 362, - SYS_TAG = 363, - TABLE_NAME = 364, - SCAN_ALL = 365, - PARALL = 366, - SHARD_KEY = 367, - INT_NUM = 368, - SHOW_PROXYNET = 369, - THREAD = 370, - CONNECTION = 371, - LIMIT = 372, - OFFSET = 373, - SHOW_PROCESSLIST = 374, - SHOW_PROXYSESSION = 375, - SHOW_GLOBALSESSION = 376, - ATTRIBUTE = 377, - VARIABLES = 378, - ALL = 379, - STAT = 380, - SHOW_PROXYCONFIG = 381, - DIFF = 382, - USER = 383, - LIKE = 384, - SHOW_PROXYSM = 385, - SHOW_PROXYCLUSTER = 386, - SHOW_PROXYRESOURCE = 387, - SHOW_PROXYCONGESTION = 388, - SHOW_PROXYROUTE = 389, - PARTITION = 390, - ROUTINE = 391, - SHOW_PROXYVIP = 392, - SHOW_PROXYMEMORY = 393, - OBJPOOL = 394, - SHOW_SQLAUDIT = 395, - SHOW_WARNLOG = 396, - SHOW_PROXYSTAT = 397, - REFRESH = 398, - SHOW_PROXYTRACE = 399, - SHOW_PROXYINFO = 400, - BINARY = 401, - UPGRADE = 402, - IDC = 403, - SHOW_TOPOLOGY = 404, - GROUP_NAME = 405, - SHOW_DB_VERSION = 406, - SHOW_DATABASES = 407, - SHOW_TABLES = 408, - SELECT_DATABASE = 409, - SHOW_CREATE_TABLE = 410, - ALTER_PROXYCONFIG = 411, - ALTER_PROXYRESOURCE = 412, - PING_PROXY = 413, - KILL_PROXYSESSION = 414, - KILL_GLOBALSESSION = 415, - KILL = 416, - QUERY = 417 + TABLE = 274, + UNIQUE = 275, + GRANT = 276, + REVOKE = 277, + ANALYZE = 278, + PURGE = 279, + COMMENT = 280, + FROM = 281, + DUAL = 282, + PREPARE = 283, + EXECUTE = 284, + USING = 285, + SELECT_HINT_BEGIN = 286, + UPDATE_HINT_BEGIN = 287, + DELETE_HINT_BEGIN = 288, + INSERT_HINT_BEGIN = 289, + REPLACE_HINT_BEGIN = 290, + MERGE_HINT_BEGIN = 291, + HINT_END = 292, + COMMENT_BEGIN = 293, + COMMENT_END = 294, + ROUTE_TABLE = 295, + ROUTE_PART_KEY = 296, + QUERY_TIMEOUT = 297, + READ_CONSISTENCY = 298, + WEAK = 299, + STRONG = 300, + FROZEN = 301, + PLACE_HOLDER = 302, + END_P = 303, + ERROR = 304, + WHEN = 305, + FLASHBACK = 306, + AUDIT = 307, + NOAUDIT = 308, + BEGI = 309, + START = 310, + TRANSACTION = 311, + READ = 312, + ONLY = 313, + WITH = 314, + CONSISTENT = 315, + SNAPSHOT = 316, + INDEX = 317, + XA = 318, + WARNINGS = 319, + ERRORS = 320, + TRACE = 321, + QUICK = 322, + COUNT = 323, + AS = 324, + WHERE = 325, + VALUES = 326, + ORDER = 327, + GROUP = 328, + HAVING = 329, + INTO = 330, + UNION = 331, + FOR = 332, + TX_READ_ONLY = 333, + AUTOCOMMIT_0 = 334, + SELECT_OBPROXY_ROUTE_ADDR = 335, + SET_OBPROXY_ROUTE_ADDR = 336, + NAME_OB_DOT = 337, + NAME_OB = 338, + EXPLAIN = 339, + DESC = 340, + DESCRIBE = 341, + NAME_STR = 342, + USE = 343, + HELP = 344, + SET_NAMES = 345, + SET_CHARSET = 346, + SET_PASSWORD = 347, + SET_DEFAULT = 348, + SET_OB_READ_CONSISTENCY = 349, + SET_TX_READ_ONLY = 350, + GLOBAL = 351, + SESSION = 352, + NUMBER_VAL = 353, + GROUP_ID = 354, + TABLE_ID = 355, + ELASTIC_ID = 356, + TESTLOAD = 357, + ODP_COMMENT = 358, + TNT_ID = 359, + DISASTER_STATUS = 360, + TRACE_ID = 361, + RPC_ID = 362, + DBP_COMMENT = 363, + ROUTE_TAG = 364, + SYS_TAG = 365, + TABLE_NAME = 366, + SCAN_ALL = 367, + PARALL = 368, + SHARD_KEY = 369, + INT_NUM = 370, + SHOW_PROXYNET = 371, + THREAD = 372, + CONNECTION = 373, + LIMIT = 374, + OFFSET = 375, + SHOW_PROCESSLIST = 376, + SHOW_PROXYSESSION = 377, + SHOW_GLOBALSESSION = 378, + ATTRIBUTE = 379, + VARIABLES = 380, + ALL = 381, + STAT = 382, + SHOW_PROXYCONFIG = 383, + DIFF = 384, + USER = 385, + LIKE = 386, + SHOW_PROXYSM = 387, + SHOW_PROXYCLUSTER = 388, + SHOW_PROXYRESOURCE = 389, + SHOW_PROXYCONGESTION = 390, + SHOW_PROXYROUTE = 391, + PARTITION = 392, + ROUTINE = 393, + SHOW_PROXYVIP = 394, + SHOW_PROXYMEMORY = 395, + OBJPOOL = 396, + SHOW_SQLAUDIT = 397, + SHOW_WARNLOG = 398, + SHOW_PROXYSTAT = 399, + REFRESH = 400, + SHOW_PROXYTRACE = 401, + SHOW_PROXYINFO = 402, + BINARY = 403, + UPGRADE = 404, + IDC = 405, + SHOW_TOPOLOGY = 406, + GROUP_NAME = 407, + SHOW_DB_VERSION = 408, + SHOW_DATABASES = 409, + SHOW_TABLES = 410, + SELECT_DATABASE = 411, + SHOW_CREATE_TABLE = 412, + SELECT_PROXY_VERSION = 413, + ALTER_PROXYCONFIG = 414, + ALTER_PROXYRESOURCE = 415, + PING_PROXY = 416, + KILL_PROXYSESSION = 417, + KILL_GLOBALSESSION = 418, + KILL = 419, + QUERY = 420 }; #endif diff --git a/src/obproxy/optimizer/ob_sharding_select_log_plan.cpp b/src/obproxy/optimizer/ob_sharding_select_log_plan.cpp index ff72df38c3245501019b203fe653531c964ac6be..a4e80fa871d5514329d8804fcb9417de388f68d2 100644 --- a/src/obproxy/optimizer/ob_sharding_select_log_plan.cpp +++ b/src/obproxy/optimizer/ob_sharding_select_log_plan.cpp @@ -22,6 +22,7 @@ using namespace oceanbase::obproxy::engine; using namespace oceanbase::common; using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy::opsql; namespace oceanbase { @@ -30,6 +31,8 @@ namespace obproxy namespace optimizer { +const static char DERIVED_COLUMN[] = "DERIVED_COLUMN"; + template int create_operator_and_input(ObIAllocator *allocator, O *&op, I *&input) { @@ -59,47 +62,58 @@ int create_operator_and_input(ObIAllocator *allocator, O *&op, I *&input) return ret; } -ObShardingSelectLogPlan::ObShardingSelectLogPlan(obutils::ObSqlParseResult &parse_result, - ObIAllocator *allocator, - ObIArray &physical_table_name_array) - : parse_result_(parse_result), allocator_(allocator), plan_root_(NULL), - shard_connector_array_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE), - physical_table_name_array_(physical_table_name_array), - table_scan_(NULL), output_expr_array_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE) +ObShardingSelectLogPlan::ObShardingSelectLogPlan(proxy::ObProxyMysqlRequest &client_request, + ObIAllocator *allocator) + : client_request_(client_request), allocator_(allocator), plan_root_(NULL), + derived_column_count_(0), is_set_limit_(false), + agg_exprs_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE), + calc_exprs_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE), + derived_exprs_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE), + derived_columns_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE), + derived_orders_(ObModIds::OB_PROXY_SHARDING_OPTIMIZER, OB_MALLOC_NORMAL_BLOCK_SIZE) { } -ObShardingSelectLogPlan::~ObShardingSelectLogPlan() { +ObShardingSelectLogPlan::~ObShardingSelectLogPlan() +{ if (NULL != plan_root_) { plan_root_->~ObProxyOperator(); } plan_root_ = NULL; - table_scan_ = NULL; } -int ObShardingSelectLogPlan::generate_plan() +int ObShardingSelectLogPlan::generate_plan(ObIArray &shard_connector_array, + ObIArray &shard_prop_array, + ObIArray > &table_name_map_array) { int ret = OB_SUCCESS; + ObSqlString new_sql; + bool is_same_group_and_order = false; LOG_DEBUG("begin to generate plan"); - if (OB_FAIL(analyze_from_clause())) { - LOG_WARN("analyze from clause failed", K(ret)); - } else if (OB_FAIL(analyze_where_clause())) { - LOG_WARN("analyze where clause failed", K(ret)); + if (OB_FAIL(analyze_select_clause())) { + LOG_WARN("analyze select clause failed", K(ret)); } else if (OB_FAIL(analyze_group_by_clause())) { - LOG_WARN("analyze group by clause failed", K(ret)); - } else if (OB_FAIL(analyze_order_by_clause())) { LOG_WARN("analyze order by clause failed", K(ret)); - } else if (OB_FAIL(analyze_select_clause())) { - LOG_WARN("analyze select clause failed", K(ret)); + } else if (OB_FAIL(analyze_order_by_clause())) { + LOG_WARN("analyze group by clause failed", K(ret)); + } else if (OB_FAIL(append_derived_order_by(is_same_group_and_order))) { + LOG_WARN("fail to append derived order by", K(ret)); + } else if (OB_FAIL(rewrite_sql(new_sql))) { + LOG_WARN("fail to rewrite sql", K(ret)); } if (OB_SUCC(ret)) { - if (OB_FAIL(add_projection_operator())) { - LOG_WARN("add projection operator failed", K(ret)); - } else if (OB_FAIL(traverse_plan_tree(plan_root_))) { - LOG_WARN("traverse plan tree failed", K(ret)); + if (OB_FAIL(add_table_scan_operator(shard_connector_array, + shard_prop_array, + table_name_map_array, + new_sql))) { + LOG_WARN("fail to add table scan operator", K(ret)); + } else if (OB_FAIL(add_agg_and_sort_operator(is_same_group_and_order))) { + LOG_WARN("fail to add agg operator", K(ret)); + } else if (OB_FAIL(add_projection_operator())) { + LOG_WARN("fail to add projection operator", K(ret)); } else { print_plan_info(); } @@ -108,48 +122,111 @@ int ObShardingSelectLogPlan::generate_plan() return ret; } -int ObShardingSelectLogPlan::analyze_from_clause() +template +int ObShardingSelectLogPlan::do_handle_avg_expr(ObProxyExprAvg *agg_expr, T *&expr, + const char* op, ObProxyExprType expr_type) { int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - // generate table scan operator - ObProxyTableScanOp *table_scan_op = NULL; - ObProxyTableScanInput *table_scan_input = NULL; - if (OB_FAIL(create_operator_and_input(allocator_, table_scan_op, table_scan_input))) { - LOG_WARN("create operator and input failed", K(ret)); + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_->alloc(sizeof(T)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc sum expr buf", K(ret)); } else { - table_scan_ = table_scan_op; - table_scan_input->set_logical_table_name(parse_result_.get_origin_table_name()); - table_scan_input->set_logical_database_name(parse_result_.get_origin_database_name()); - table_scan_input->set_db_key_names(shard_connector_array_); - table_scan_input->set_phy_db_table_names(physical_table_name_array_); - table_scan_input->set_hint_string(select_stmt->hint_string_); - - plan_root_ = table_scan_op; + expr = new (ptr) T(); + expr->set_expr_type(expr_type); + expr->has_agg_ = 1; + expr->set_param_array(agg_expr->get_param_array()); + ObSqlString sql_string; + char *buf = NULL; + if (OB_FAIL(sql_string.append(op))) { + LOG_WARN("append failed", K(ret)); + } else if (OB_FAIL(sql_string.append("("))) { + LOG_WARN("append failed", K(ret)); + } else if (OB_FAIL(agg_expr->get_param_array().at(0)->to_column_string(sql_string))) { + LOG_WARN("to sql_string failed", K(ret)); + } else if (OB_FAIL(sql_string.append(")"))) { + LOG_WARN("append failed", K(ret)); + } else if (OB_ISNULL(buf = (char*)allocator_->alloc(sql_string.length()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc sum expr buf", K(ret)); + } else { + MEMCPY(buf, sql_string.ptr(), sql_string.length()); + expr->set_expr_name(buf, sql_string.length()); + if (OB_FAIL(handle_derived(expr))) { + LOG_WARN("fail to handle derived", K(ret)); + } + } } return ret; } -int ObShardingSelectLogPlan::analyze_where_clause() { +int ObShardingSelectLogPlan::handle_avg_expr(ObProxyExprAvg *agg_expr) +{ int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - ObIArray &condition_expr_array = select_stmt->condition_exprs_; + ObProxyExprSum *sum_expr = NULL; + ObProxyExprCount *count_expr = NULL; + if (OB_FAIL(do_handle_avg_expr(agg_expr, sum_expr, "SUM", OB_PROXY_EXPR_TYPE_FUNC_SUM))) { + LOG_WARN("fail to do handle avg expr", K(ret)); + } else if (OB_FAIL(do_handle_avg_expr(agg_expr, count_expr, "COUNT", OB_PROXY_EXPR_TYPE_FUNC_COUNT))) { + LOG_WARN("fail to do handle avg expr", K(ret)); + } else { + agg_expr->set_sum_expr(sum_expr); + agg_expr->set_count_expr(count_expr); + } + + return ret; +} - if (condition_expr_array.count() > 0) { - if (1 != condition_expr_array.count()) { +int ObShardingSelectLogPlan::handle_agg_expr(ObProxyExpr *expr, bool need_add_calc) +{ + int ret = OB_SUCCESS; + + if (expr->has_agg()) { + // If itself is an aggregate function + if (expr->is_agg()) { + // If it is an avg aggregate function, it is actually a sum/count calculation function + if (OB_PROXY_EXPR_TYPE_FUNC_AVG == expr->get_expr_type()) { + ObProxyExprAvg *avg_expr = NULL; + if (OB_ISNULL(avg_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else if (OB_FAIL(handle_avg_expr(avg_expr))) { + LOG_WARN("handle avg expr failed", KPC(expr), K(ret)); + } else if (need_add_calc && OB_FAIL(calc_exprs_.push_back(avg_expr))) { + LOG_WARN("fail to push back to calc expr", KP(avg_expr), K(ret)); + } + } else { + // Other aggregate functions, put into the aggregate function array + if (OB_FAIL(agg_exprs_.push_back(expr))) { + LOG_WARN("fail to add agg expr to array", K(ret)); + } + } + } else if (!expr->is_func_expr()) { ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KPC(expr), K(ret)); } else { - ObProxyExpr* condition_expr = condition_expr_array.at(0); - if (condition_expr->has_agg()) { + ObProxyFuncExpr *func_expr = NULL; + if (OB_ISNULL(func_expr = dynamic_cast(expr))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("where expr has agg unexpected", K(ret)); + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); } else { - ObProxyTableScanInput *table_scan_input = static_cast(table_scan_->get_input()); - if (OB_FAIL(table_scan_input->get_condition_exprs().push_back(condition_expr))) { - LOG_WARN("push back condition expr failed", K(ret)); + ObSEArray& param_array = func_expr->get_param_array(); + + // Parse each parameter for further processing + for (int64_t i = 0; OB_SUCC(ret) && i < param_array.count(); i++) { + ObProxyExpr* param_expr = param_array.at(i); + if (OB_FAIL(handle_derived(param_expr))) { + LOG_WARN("fail to handle derived", K(ret)); + } + } + + if (OB_SUCC(ret) && need_add_calc) { + if (OB_FAIL(calc_exprs_.push_back(expr))) { + LOG_WARN("fail to push back calc expr", KP(expr), K(ret)); + } } } } @@ -158,121 +235,204 @@ int ObShardingSelectLogPlan::analyze_where_clause() { return ret; } -int ObShardingSelectLogPlan::analyze_group_by_clause() +int ObShardingSelectLogPlan::add_derived_column(ObProxyExpr *expr) { int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - ObIArray &group_by_expr_array = select_stmt->group_by_exprs_; - bool has_agg = false; - ObProxyAggOp *agg_op = NULL; - ObProxyAggInput *agg_input = NULL; - - if (group_by_expr_array.count() > 0) { - has_agg = true; - - ObProxyMergeAggOp *merge_agg = NULL; - if (OB_FAIL(create_operator_and_input(allocator_, merge_agg, agg_input))) { - LOG_WARN("create operator and input for agg failed", K(ret)); + ObSqlString sql_string; + char *buf = NULL; + if (OB_FAIL(sql_string.append_fmt("%s_%ld", DERIVED_COLUMN, derived_column_count_++))) { + LOG_WARN("fail to append derived column name", K(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(sql_string.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc alias name buf", "len", sql_string.length(), K(ret)); + } else { + MEMCPY(buf, sql_string.ptr(), sql_string.length()); + expr->set_alias_name(buf, sql_string.length()); + buf = NULL; + sql_string.reuse(); + if (OB_FAIL(expr->to_sql_string(sql_string))) { + LOG_WARN("fail to get sql string", K(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(sql_string.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc sql string buf", K(ret)); } else { - agg_op = merge_agg; - ObProxyTableScanInput *table_scan_input = static_cast(table_scan_->get_input()); - for (int64_t i = 0; OB_SUCC(ret) && i < group_by_expr_array.count(); i++) { - ObProxyExpr* expr = group_by_expr_array.at(i); - // group by must not follow agg func - if (expr->has_agg()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group by expr is agg unexpected", K(ret)); - } else if (expr->has_alias()) { - if (!expr->is_alias()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("has alias but alias is a sub expr", K(ret)); - } else if (OB_FAIL(table_scan_input->get_group_by_exprs().push_back(expr))) { - LOG_WARN("table scan add group by expr failed", K(ret)); - } else if (OB_FAIL(agg_input->get_group_by_exprs().push_back(expr))) { - LOG_WARN("agg add group by expr failed", K(ret)); - } else { - //expr->set_index(); - //do nothing, expr->index will be updated in ObProxyAggOp::init_group_by_columns - } - } else if (OB_FAIL(output_expr_array_.push_back(expr))) { - LOG_WARN("output expr array push back failed", K(ret)); - } else if (OB_FAIL(table_scan_input->get_group_by_exprs().push_back(expr))) { - LOG_WARN("table scan add group by expr failed", K(ret)); - } else if (OB_FAIL(agg_input->get_group_by_exprs().push_back(expr))) { - LOG_WARN("agg add group by expr failed", K(ret)); - } else { - expr->set_index(output_expr_array_.count() - 1); - } + MEMCPY(buf, sql_string.ptr(), sql_string.length()); + ObString derived_column(sql_string.length(), buf); + if (OB_FAIL(derived_columns_.push_back(derived_column))) { + LOG_WARN("fail to add derived column", K(ret)); + } else if (OB_FAIL(derived_exprs_.push_back(expr))) { + LOG_WARN("fail to add derived expr", K(ret)); } } - } else { - for (int64_t i = 0; !has_agg && i < select_stmt->order_by_exprs_.count(); i++) { - if (select_stmt->order_by_exprs_.at(i)->has_agg()) { - has_agg = true; + } + + return ret; +} + +int ObShardingSelectLogPlan::do_column_need_derived_with_star(ObIArray &expr_array, ObProxyExpr *expr, bool &bret) +{ + int ret = OB_SUCCESS; + bret = true; + + ObProxyExprColumn* expr_column = dynamic_cast(expr); + if (OB_ISNULL(expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && bret && i < expr_array.count(); i++) { + ObProxyExpr *tmp_expr = expr_array.at(i); + ObProxyExprType tmp_expr_type = tmp_expr->get_expr_type(); + if (OB_PROXY_EXPR_TYPE_STAR == tmp_expr_type) { + ObString &table_name = expr_column->get_table_name(); + ObProxyExprStar* expr_star = dynamic_cast(tmp_expr); + if (OB_ISNULL(expr_star)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(ret)); + } else { + ObString &tmp_table_name = expr_star->get_table_name(); + if (tmp_table_name.empty()) { + // select * from t1, t3 order by c3; + bret = false; + } else if (tmp_table_name == table_name) { + // bad case: select t1.* from t1, t3 order by c3; + // c3 may be a column in t3, so the table must be the same + bret = false; + } } } + } + + return ret; +} + +int ObShardingSelectLogPlan::do_column_need_derived_with_alias(ObIArray &expr_array, ObProxyExpr *expr, bool &bret) +{ + int ret = OB_SUCCESS; + bret = true; - for (int64_t i = 0; !has_agg && i < select_stmt->select_exprs_.count(); i++) { - if (select_stmt->select_exprs_.at(i)->has_agg()) { - has_agg = true; + ObProxyExprColumn* expr_column = dynamic_cast(expr); + if (OB_ISNULL(expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(ret)); + } + + for (int64_t i = 0; OB_SUCC(ret) && bret && i < expr_array.count(); i++) { + ObProxyExpr *tmp_expr = expr_array.at(i); + ObString &column_name = expr_column->get_column_name(); + ObString &tmp_alias_name = tmp_expr->get_alias_name(); + if (!tmp_alias_name.empty()) { + if (0 == tmp_alias_name.case_compare(column_name)) { + bret = false; + expr->set_alias_name(tmp_alias_name); } } + } - if (has_agg && OB_FAIL(create_operator_and_input(allocator_, agg_op, agg_input))) { - LOG_WARN("create operator and input for agg failed", K(ret)); - } + return ret; +} + +int ObShardingSelectLogPlan::do_column_need_derived(ObIArray &expr_array, ObProxyExpr *expr, bool &bret) +{ + int ret = OB_SUCCESS; + bret = true; + + ObProxyExprColumn* expr_column = dynamic_cast(expr); + if (OB_ISNULL(expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(ret)); } - if (OB_SUCC(ret) && has_agg) { - if (OB_FAIL(agg_op->set_child(0, plan_root_))) { - LOG_WARN("set child failed", K(ret)); - } else { - plan_root_ = agg_op; + for (int64_t i = 0; OB_SUCC(ret) && bret && i < expr_array.count(); i++) { + ObProxyExpr *tmp_expr = expr_array.at(i); + ObProxyExprType tmp_expr_type = tmp_expr->get_expr_type(); + if (OB_PROXY_EXPR_TYPE_COLUMN == tmp_expr_type) { + ObString &table_name = expr_column->get_table_name(); + ObString &column_name = expr_column->get_column_name(); + ObString &tmp_alias_name = tmp_expr->get_alias_name(); + ObProxyExprColumn* tmp_expr_column = dynamic_cast(tmp_expr); + if (OB_ISNULL(tmp_expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(ret)); + } else { + ObString &tmp_column_name = tmp_expr_column->get_column_name(); + ObString &tmp_table_name = tmp_expr_column->get_table_name(); + if (tmp_table_name.empty() || table_name.empty()) { + // When there is no table name in the select column: + // select c1 from t1, t3 order by t1.c1; + // select c1 from t1, t3 order by c1; + // If both t1 and t3 have c1 column, the execution will report an error + // When there is no table name in the order column: + // select t1.c1 from t1, t3 order by c1 + // This can be executed correctly, and it is also sorted according to the t1.c1 column + if (0 == tmp_column_name.case_compare(column_name)) { + bret = false; + expr->set_alias_name(tmp_alias_name); + } + } else if ((tmp_table_name == table_name) + && (0 == tmp_column_name.case_compare(column_name))) { + bret = false; + expr->set_alias_name(tmp_alias_name); + } + } } } return ret; } -int ObShardingSelectLogPlan::handle_avg_expr(opsql::ObProxyExprAvg *agg_expr) +int ObShardingSelectLogPlan::do_other_need_derived_for_avg(ObProxyExpr *tmp_expr, ObProxyExpr *expr) { int ret = OB_SUCCESS; + ObProxyExprType tmp_expr_type = tmp_expr->get_expr_type(); - ObProxyExprSum *sum_expr = NULL; - void *ptr = NULL; - - if (OB_ISNULL(ptr = allocator_->alloc(sizeof(ObProxyExprSum)))) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; + if (OB_PROXY_EXPR_TYPE_FUNC_AVG != tmp_expr_type) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("avg expr match non avg expr", K(tmp_expr), K(expr), K(ret)); } else { - sum_expr = new (ptr) ObProxyExprSum(); - sum_expr->set_expr_type(OB_PROXY_EXPR_TYPE_FUNC_SUM); - sum_expr->set_param_array(agg_expr->get_param_array()); - sum_expr->has_agg_ = 1; - if (OB_FAIL(output_expr_array_.push_back(sum_expr))) { - LOG_WARN("output expr push back failed", K(ret)); + ObProxyExprAvg *avg_expr = NULL; + ObProxyExprAvg *tmp_avg_expr = NULL; + if (OB_ISNULL(avg_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else if (OB_ISNULL(tmp_avg_expr = dynamic_cast(tmp_expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(tmp_expr), K(ret)); } else { - agg_expr->set_sum_index(output_expr_array_.count() - 1); - sum_expr->set_index(output_expr_array_.count() - 1); + avg_expr->set_sum_expr(tmp_avg_expr->get_sum_expr()); + avg_expr->set_count_expr(tmp_avg_expr->get_count_expr()); } } + return ret; +} + +int ObShardingSelectLogPlan::do_other_need_derived(ObProxyExpr *tmp_expr, ObProxyExpr *expr, bool &bret) +{ + int ret = OB_SUCCESS; + bret = true; - if (OB_SUCC(ret)) - { - ObProxyExprCount *count_expr = NULL; - void *ptr = allocator_->alloc(sizeof(ObProxyExprCount)); + ObString &expr_name = expr->get_expr_name(); + ObProxyExprType expr_type = expr->get_expr_type(); + ObProxyExprType tmp_expr_type = tmp_expr->get_expr_type(); + ObString &tmp_alias_name = tmp_expr->get_alias_name(); - if (OB_UNLIKELY(NULL == ptr)) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - } else { - count_expr = new (ptr) ObProxyExprCount(); - count_expr->set_expr_type(OB_PROXY_EXPR_TYPE_FUNC_COUNT); - count_expr->set_param_array(agg_expr->get_param_array()); - count_expr->has_agg_ = 1; - if (OB_FAIL(output_expr_array_.push_back(count_expr))) { - LOG_WARN("output expr push back failed", K(ret)); - } else { - agg_expr->set_count_index(output_expr_array_.count() - 1); - count_expr->set_index(output_expr_array_.count() - 1); + if (0 == tmp_alias_name.case_compare(expr_name)) { + bret = false; + expr->set_alias_name(tmp_alias_name); + if (OB_PROXY_EXPR_TYPE_FUNC_AVG == expr_type) { + if (OB_FAIL(do_other_need_derived_for_avg(tmp_expr, expr))) { + LOG_WARN("fail to do other need derived for avg", K(tmp_expr), K(expr), K(ret)); + } + } + } else if (OB_PROXY_EXPR_TYPE_SHARDING_CONST == tmp_expr_type || tmp_expr->is_func_expr()) { + ObString &tmp_expr_name = tmp_expr->get_expr_name(); + if (0 == tmp_expr_name.case_compare(expr_name)) { + bret = false; + expr->set_alias_name(tmp_alias_name); + if (OB_PROXY_EXPR_TYPE_FUNC_AVG == expr_type) { + if (OB_FAIL(do_other_need_derived_for_avg(tmp_expr, expr))) { + LOG_WARN("fail to do other need derived for avg", K(tmp_expr), K(expr), K(ret)); + } } } } @@ -280,219 +440,598 @@ int ObShardingSelectLogPlan::handle_avg_expr(opsql::ObProxyExprAvg *agg_expr) return ret; } -int ObShardingSelectLogPlan::analyze_order_by_clause() +int ObShardingSelectLogPlan::do_need_derived(ObIArray &expr_array, ObProxyExpr *expr, bool column_first, bool &bret) { int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - ObIArray *order_by_expr_array = &select_stmt->order_by_exprs_; - ObProxyOperator *sort_op = NULL; - ObProxySortInput *sort_input = NULL; - ObProxyTableScanInput *table_scan_input = static_cast(table_scan_->get_input()); + ObProxyExprType expr_type = expr->get_expr_type(); + bret = true; - if (order_by_expr_array->count() > 0) { - if (select_stmt->limit_start_ > 0) { - ObProxyTopKOp *topK = NULL; - if (OB_FAIL(create_operator_and_input(allocator_, topK, sort_input))) { - LOG_WARN("create operator and input for topk failed", K(ret)); - } else { - sort_op = topK; + if (OB_PROXY_EXPR_TYPE_COLUMN == expr_type) { + if (column_first) { + if (OB_FAIL(do_column_need_derived(expr_array, expr, bret))) { + LOG_WARN("fail to do column need derived", K(ret)); + } else if (bret && OB_FAIL(do_column_need_derived_with_alias(expr_array, expr, bret))) { + LOG_WARN("fail to do column need derived with alias", K(ret)); } } else { - ObProxyMemSortOp *mem_sort = NULL; - if (OB_FAIL(create_operator_and_input(allocator_, mem_sort, sort_input))) { - LOG_WARN("create operator and input for mem sort failed", K(ret)); - } else { - sort_op = mem_sort; + if (OB_FAIL(do_column_need_derived_with_alias(expr_array, expr, bret))) { + LOG_WARN("fail to do column need derived with alias", K(ret)); + } else if (bret && OB_FAIL(do_column_need_derived(expr_array, expr, bret))) { + LOG_WARN("fail to do column need derived", K(ret)); } } - if (OB_SUCC(ret)) { - if (OB_FAIL(sort_op->set_child(0, plan_root_))) { - LOG_WARN("set child failed", K(ret)); - } else { - plan_root_ = sort_op; + if (bret && OB_FAIL(do_column_need_derived_with_star(expr_array, expr, bret))) { + LOG_WARN("fail to do column need derived with star", K(ret)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && bret && i < expr_array.count(); i++) { + ObProxyExpr *tmp_expr = expr_array.at(i); + if (OB_FAIL(do_other_need_derived(tmp_expr, expr, bret))) { + LOG_WARN("fail to do other need derived", K(ret)); } } + } - for (int64_t i = 0; OB_SUCC(ret) && i < order_by_expr_array->count(); i++) { - ObProxyOrderItem *order_item = order_by_expr_array->at(i); - ObProxyExpr *expr = order_item->expr_; - if ((!expr->has_agg() && !expr->has_alias()) || expr->is_agg()) { - if (OB_FAIL(output_expr_array_.push_back(expr))) { - LOG_WARN("output expr push failed", K(ret)); - } else if (OB_FAIL(sort_input->get_order_by_expr().push_back(order_item))) { - LOG_WARN("add order by expr failed", K(ret)); - } else if (OB_FAIL(table_scan_input->get_order_by_exprs().push_back(order_item))) { - LOG_WARN("table scan add order by expr failed", K(ret)); - } else { - expr->set_index(output_expr_array_.count() - 1); + return ret; +} + +int ObShardingSelectLogPlan::handle_derived(ObProxyExpr *expr, bool column_first /*false*/) +{ + int ret = OB_SUCCESS; + bool bret = true; + + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expr is NULL", K(ret)); + } else { + ObProxyExprType expr_type = expr->get_expr_type(); + + if (OB_PROXY_EXPR_TYPE_COLUMN == expr_type || OB_PROXY_EXPR_TYPE_SHARDING_CONST == expr_type || expr->is_func_expr()) { + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &select_expr_array = select_stmt->select_exprs_; + + // Priority is given to finding derived columns, all derived columns have aliases and can be uniquely located + if (bret && do_need_derived(derived_exprs_, expr, column_first, bret)) { + LOG_WARN("fail to do need derived from select expr", K(ret)); + } else if (OB_FAIL(do_need_derived(select_expr_array, expr, column_first, bret))) { + LOG_WARN("fail to do need derived from select expr", K(ret)); + } else if (bret) { + if (OB_FAIL(add_derived_column(expr))) { + LOG_WARN("fail to add derived column", K(ret)); + } else if (OB_FAIL(handle_agg_expr(expr))) { + LOG_WARN("fail to handle agg expr", K(ret)); } - } else if (expr->is_alias()) { - if (expr->has_agg() && !expr->is_agg()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("has agg not support", K(ret), K(expr)); - } else if (OB_FAIL(sort_input->get_order_by_expr().push_back(order_item))) { - LOG_WARN("add order by expr failed", K(ret)); - } else if (OB_FAIL(table_scan_input->get_order_by_exprs().push_back(order_item))) { - LOG_WARN("table scan add order by expr failed", K(ret)); + } + } + } + + return ret; +} + +int ObShardingSelectLogPlan::do_handle_select_derived(ObProxyExpr *expr, bool &bret, bool is_root) +{ + int ret = OB_SUCCESS; + bret = false; + + if (expr->get_alias_name().empty()) { + ObProxyExprType expr_type = expr->get_expr_type(); + if (expr->is_func_expr()) { + ObProxyFuncExpr *func_expr = NULL; + if (OB_ISNULL(func_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); + } else { + ObSEArray& param_array = func_expr->get_param_array(); + for (int64_t i = 0; OB_SUCC(ret) && !bret && i < param_array.count(); i++) { + ObProxyExpr* param_expr = param_array.at(i); + if (OB_FAIL(do_handle_select_derived(param_expr, bret))) { + LOG_WARN("fail to handle derived", K(ret)); + } } + } + } else if (!is_root && OB_PROXY_EXPR_TYPE_COLUMN == expr_type) { + ObProxyExprColumn* expr_column = dynamic_cast(expr); + if (OB_ISNULL(expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", K(expr), K(ret)); } else { - ret = OB_NOT_SUPPORTED; - ObProxyExpr::print_proxy_expr(expr); - LOG_WARN("not support", K(ret), K(*expr)); + if (!expr_column->get_table_name().empty() + && expr_column->get_table_name() == expr_column->get_real_table_name()) { + bret = true; + } } } } + + return ret; +} + +int ObShardingSelectLogPlan::handle_select_derived(ObProxyExpr *expr) +{ + int ret = OB_SUCCESS; + bool bret = false; + + if (OB_ISNULL(expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("expr is NULL", K(ret)); + } else if (OB_FAIL(do_handle_select_derived(expr, bret, true))) { + LOG_WARN("fail to do handle select derived", K(ret)); + } else if (bret && OB_FAIL(add_derived_column(expr))) { + LOG_WARN("fail to add derived column", K(ret)); + } else if (OB_FAIL(handle_agg_expr(expr, true))) { + LOG_WARN("fail to handle agg expr", K(ret)); + } + + return ret; +} + +int ObShardingSelectLogPlan::analyze_group_by_clause() +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &group_by_exprs = select_stmt->group_by_exprs_; + + for (int64_t i = 0; OB_SUCC(ret) && i < group_by_exprs.count(); i++) { + ObProxyGroupItem *group_item = dynamic_cast(group_by_exprs.at(i)); + ObProxyExpr *expr = NULL; + if (OB_ISNULL(group_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", "expr", group_by_exprs.at(i), K(ret)); + } else if (OB_ISNULL(expr = group_item->get_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group item expr is NULL", K(ret)); + } else if (OB_FAIL(handle_derived(expr, true))) { + LOG_WARN("fail to handle derived", K(ret)); + } + } + + return ret; +} + +int ObShardingSelectLogPlan::analyze_order_by_clause() +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &order_by_exprs = select_stmt->order_by_exprs_; + + for (int64_t i = 0; OB_SUCC(ret) && i < order_by_exprs.count(); i++) { + ObProxyOrderItem *order_item = dynamic_cast(order_by_exprs.at(i)); + ObProxyExpr *expr = NULL; + if (OB_ISNULL(order_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", "expr", order_by_exprs.at(i), K(ret)); + } else if (OB_ISNULL(expr = order_item->get_expr())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("order item expr is NULL", K(ret)); + } else if (OB_FAIL(handle_derived(expr))) { + LOG_WARN("fail to handle derived", K(ret)); + } + } + return ret; } int ObShardingSelectLogPlan::analyze_select_clause() { int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); ObIArray &select_expr_array = select_stmt->select_exprs_; for (int64_t i = 0; OB_SUCC(ret) && i < select_expr_array.count(); i++) { - ObProxyExpr *expr = select_expr_array.at(i); - if (expr->has_agg()) { - ObSEArray agg_expr_array; - if (OB_FAIL(get_agg_related_expr(expr, agg_expr_array))) { - LOG_WARN("get agg related expr failed", K(ret), K(expr)); + ObProxyExpr *select_expr = select_expr_array.at(i); + if (OB_FAIL(handle_select_derived(select_expr))) { + LOG_WARN("fail to handle select derived", K(ret)); + } + } + + return ret; +} + +int ObShardingSelectLogPlan::append_derived_order_by(bool &is_same_group_and_order) +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &group_by_exprs = select_stmt->group_by_exprs_; + ObIArray &order_by_exprs = select_stmt->order_by_exprs_; + + if (!group_by_exprs.empty() && order_by_exprs.empty()) { + is_same_group_and_order = true; + for (int64_t i = 0; OB_SUCC(ret) && i < group_by_exprs.count(); i++) { + ObProxyGroupItem* group_expr = group_by_exprs.at(i); + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator_->alloc(sizeof(ObProxyOrderItem)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc order expr buf", K(ret)); } else { - for (int64_t j = 0; OB_SUCC(ret) && j < agg_expr_array.count(); j++) { - ObProxyExpr *agg_expr = agg_expr_array.at(j); - if (OB_PROXY_EXPR_TYPE_FUNC_AVG == agg_expr->get_expr_type()) { - if (OB_FAIL(handle_avg_expr(static_cast(agg_expr)))) { - LOG_WARN("handle avg expr failed", K(ret)); - } - } else { - if (OB_FAIL(output_expr_array_.push_back(agg_expr))) { - LOG_WARN("output expr push back failed", K(ret)); - } else { - agg_expr->set_index(output_expr_array_.count() - 1); - } + ObProxyExpr *expr = group_expr->get_expr(); + ObSqlString sql_string; + char *buf = NULL; + ObProxyOrderItem *order_expr = new (ptr) ObProxyOrderItem(); + order_expr->set_expr_type(OB_PROXY_EXPR_TYPE_FUNC_ORDER); + order_expr->set_expr(expr); + if (OB_FAIL(order_expr->to_sql_string(sql_string))) { + LOG_WARN("fail to get sql string", K(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(sql_string.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc sql string buf", K(ret)); + } else { + MEMCPY(buf, sql_string.ptr(), sql_string.length()); + ObString derived_order(sql_string.length(), buf); + + if (OB_FAIL(order_by_exprs.push_back(order_expr))) { + LOG_WARN("fail to push back order expr", K(ret)); + } else if (OB_FAIL(derived_orders_.push_back(derived_order))) { + LOG_WARN("fail to add derived order by", K(ret)); } } } } } - for (int64_t i = select_expr_array.count() - 1; OB_SUCC(ret) && i >=0; i--) { - ObProxyExpr *expr = select_expr_array.at(i); - if (OB_FAIL(output_expr_array_.push_back(expr))) { - LOG_WARN("output expr push failed", K(ret)); - } else if (!expr->has_agg() && !expr->is_star_expr()) { - expr->set_index(output_expr_array_.count() - 1); + return ret; +} + +int ObShardingSelectLogPlan::rewrite_sql(ObSqlString &new_sql) +{ + int ret = OB_SUCCESS; + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObString sql = client_request_.get_sql(); + + if (derived_columns_.empty() && derived_orders_.empty() + && (select_stmt->limit_size_ == -1 || select_stmt->limit_size_ == 0)) { + new_sql.append(sql); + } else { + const char *sql_ptr = sql.ptr(); + int64_t sql_len = sql.length(); + + if (derived_columns_.empty() && derived_orders_.empty()) { + int64_t limit_position = select_stmt->limit_token_off_; + new_sql.append(sql_ptr, limit_position); + new_sql.append("LIMIT "); + if (select_stmt->limit_size_ == 0) { + new_sql.append_fmt("%d", select_stmt->limit_size_); + } else { + new_sql.append_fmt("%d", select_stmt->limit_offset_ + select_stmt->limit_size_); + } + } else { + int64_t from_position = select_stmt->get_from_token_off(); + int64_t limit_position = select_stmt->limit_token_off_; + new_sql.append(sql_ptr, from_position); + + if (!derived_columns_.empty()) { + new_sql.append(", "); + int64_t derived_column_count = derived_columns_.count(); + for (int64_t i = 0; i < derived_column_count; i++) { + ObString derived_column = derived_columns_.at(i); + if (i == derived_column_count - 1) { + new_sql.append(derived_column); + new_sql.append(" "); + } else { + new_sql.append(derived_column); + new_sql.append(", "); + } + } + } + + if (limit_position > 0) { + new_sql.append(sql_ptr + from_position, limit_position - from_position); + } else { + new_sql.append(sql_ptr + from_position, sql_len - from_position); + } + + if (!derived_orders_.empty()) { + // Bottom group by id; scenes with semicolons + while (!new_sql.empty() && ';' == new_sql.ptr()[new_sql.length() - 1]) { + new_sql.set_length(new_sql.length() - 1); + } + + new_sql.append(" ORDER BY "); + int64_t derived_order_by_count = derived_orders_.count(); + for (int64_t i = 0; i < derived_order_by_count; i++) { + ObString derived_order_by = derived_orders_.at(i); + if (i == derived_order_by_count - 1) { + new_sql.append(derived_order_by); + new_sql.append(" "); + } else { + new_sql.append(derived_order_by); + new_sql.append(","); + } + } + } + + if (limit_position > 0) { + new_sql.append("LIMIT "); + if (select_stmt->limit_size_ == 0) { + new_sql.append_fmt("%d", select_stmt->limit_size_); + } else { + new_sql.append_fmt("%d", select_stmt->limit_offset_ + select_stmt->limit_size_); + } + } } + } + + return ret; +} + +template +int ObShardingSelectLogPlan::add_agg_operator(bool need_set_limit) +{ + int ret = OB_SUCCESS; - if (expr->is_star_expr() && i != 0) { - ret = OB_ERR_PARSER_SYNTAX; - LOG_WARN("* is not fisrt expr in select", K(ret)); + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &group_by_exprs = select_stmt->group_by_exprs_; + + T *agg_operator = NULL; + ObProxyAggInput *agg_input = NULL; + if (OB_FAIL(create_operator_and_input(allocator_, agg_operator, agg_input))) { + LOG_WARN("create operator and input for agg failed", K(ret)); + } else if (OB_FAIL(agg_input->set_group_by_exprs(group_by_exprs))) { + LOG_WARN("fail to set group exprs", K(ret)); + } else if (OB_FAIL(agg_input->set_agg_exprs(agg_exprs_))) { + LOG_WARN("fail to set agg exprs", K(ret)); + } else if (OB_FAIL(agg_operator->set_child(0, plan_root_))) { + LOG_WARN("set child failed", K(ret)); + } else { + if (need_set_limit) { + agg_input->set_limit_offset(select_stmt->limit_offset_); + agg_input->set_limit_size(select_stmt->limit_size_); + is_set_limit_ = true; } + plan_root_ = agg_operator; } - for (int64_t i = 0, j = output_expr_array_.count() - 1; OB_SUCC(ret) && i < j; i++, j--) { - ObProxyExpr *tmp_expr = output_expr_array_.at(i); - output_expr_array_.at(i) = output_expr_array_.at(j); - output_expr_array_.at(j) = tmp_expr; + return ret; +} + +int ObShardingSelectLogPlan::add_stream_agg_operator(bool need_set_limit) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(add_agg_operator(need_set_limit))) { + LOG_WARN("fail to add stream agg", K(need_set_limit), K(ret)); } return ret; } -int ObShardingSelectLogPlan::add_projection_operator() { +int ObShardingSelectLogPlan::add_mem_merge_agg_operator(bool need_set_limit) +{ int ret = OB_SUCCESS; - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - ObProxyProOp *op = NULL; - ObProxyProInput *input = NULL; + if (OB_FAIL(add_agg_operator(need_set_limit))) { + LOG_WARN("fail to add mem merge agg", K(ret)); + } - if (OB_FAIL(create_operator_and_input(allocator_, op, input))) { - LOG_WARN("create projection failed", K(ret)); + return ret; +} + +template +int ObShardingSelectLogPlan::add_sort_operator(bool need_set_limit) +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &order_by_exprs = select_stmt->order_by_exprs_; + + T *sort_operator = NULL; + ObProxySortInput *sort_input = NULL; + if (OB_FAIL(create_operator_and_input(allocator_, sort_operator, sort_input))) { + LOG_WARN("create operator and input for agg failed", K(ret)); + } else if (OB_FAIL(sort_input->set_order_by_exprs(order_by_exprs))) { + LOG_WARN("fail to set order exprs", K(ret)); + } else if (OB_FAIL(sort_operator->set_child(0, plan_root_))) { + LOG_WARN("set child failed", K(ret)); } else { - input->set_select_exprs(select_stmt->select_exprs_); - if (OB_FAIL(op->set_child(0, plan_root_))) { - LOG_WARN("set child failed", K(ret)); - } else { - plan_root_ = op; + if (need_set_limit) { + sort_input->set_limit_offset(select_stmt->limit_offset_); + sort_input->set_limit_size(select_stmt->limit_size_); + is_set_limit_ = true; } + plan_root_ = sort_operator; } return ret; } -int ObShardingSelectLogPlan::get_agg_related_expr(ObProxyExpr* expr, ObIArray &array) +int ObShardingSelectLogPlan::add_stream_sort_operator(bool need_set_limit) { int ret = OB_SUCCESS; - if (NULL != expr && OB_SUCC(ret)) { - if (expr->is_alias()) { - ObProxyExprShardingConst *const_expr = static_cast(expr); - if (OB_FAIL(get_agg_related_expr(const_expr->expr_, array))) { - LOG_WARN("get agg related expr failed", K(ret)); - } - } else if (!expr->has_agg() || expr->is_agg()) { - if(OB_FAIL(array.push_back(expr))) { - LOG_WARN("push back expr failed", K(ret)); - } - } else if (!expr->is_func_expr()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", K(ret), K(*expr)); - } else { - ObProxyFuncExpr *func_expr = static_cast(expr); - int64_t count = func_expr->get_param_array().count(); - if (OB_PROXY_EXPR_TYPE_SHARDING_ALIAS == func_expr->get_expr_type()) { - count--; + + if (OB_FAIL(add_sort_operator(need_set_limit))) { + LOG_WARN("fail to add mem merge agg", K(ret)); + } + + return ret; +} + +int ObShardingSelectLogPlan::add_mem_merge_sort_operator(bool need_set_limit) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(add_sort_operator(need_set_limit))) { + LOG_WARN("fail to add mem merge agg", K(ret)); + } + + return ret; +} + +int ObShardingSelectLogPlan::add_agg_and_sort_operator(bool is_same_group_and_order) +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &group_by_exprs = select_stmt->group_by_exprs_; + ObIArray &order_by_exprs = select_stmt->order_by_exprs_; + + if (!group_by_exprs.empty() || !agg_exprs_.empty()) { + if (!is_same_group_and_order) { + if (OB_FAIL(compare_group_and_order(is_same_group_and_order))) { + LOG_WARN("fail to compare group and order", K(is_same_group_and_order), K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - ObProxyExpr* param_expr = func_expr->get_param_array().at(i); - if (OB_FAIL(get_agg_related_expr(param_expr, array))) { - LOG_WARN("get agg related expr failed", K(ret)); + } + + if (OB_SUCC(ret)) { + if (is_same_group_and_order) { + if (OB_FAIL(add_stream_sort_operator(false))) { + LOG_WARN("fail to add stream sort operator", K(ret)); + } else if (OB_FAIL(add_stream_agg_operator(true))) { + LOG_WARN("fail to add stream agg operator", K(ret)); + } + } else { + if (OB_FAIL(add_mem_merge_agg_operator(false))) { + LOG_WARN("fail to add mem merge agg operator", K(ret)); + } else if (OB_FAIL(add_mem_merge_sort_operator(true))) { + LOG_WARN("fail to add mem merge sort operator", K(ret)); } } } + } else if (!order_by_exprs.empty()) { + if (OB_FAIL(add_stream_sort_operator(true))) { + LOG_WARN("fail to add stream sort operator", K(ret)); + } } + return ret; } -int ObShardingSelectLogPlan::traverse_plan_tree(ObProxyOperator *op) +int ObShardingSelectLogPlan::add_table_scan_operator(ObIArray &shard_connector_array, + ObIArray &shard_prop_array, + ObIArray > &table_name_map_array, + ObSqlString &new_sql) { int ret = OB_SUCCESS; - if (NULL != op) { - ObProxySelectStmt *select_stmt = static_cast(parse_result_.get_proxy_stmt()); - ObProxyOpInput* input = op->get_input(); - input->set_op_limit(select_stmt->limit_start_, select_stmt->limit_offset_); - input->set_select_exprs(output_expr_array_); - input->set_added_row_count(output_expr_array_.count() - select_stmt->select_exprs_.count()); - if (OB_FAIL(traverse_plan_tree(op->get_child(0)))) { - LOG_WARN("traverse plan tree failed", K(ret)); + const uint32_t PARSE_EXTRA_CHAR_NUM = 2; + + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObProxyTableScanOp *table_scan_op = NULL; + ObProxyTableScanInput *table_scan_input = NULL; + char *buf = NULL; + if (OB_FAIL(create_operator_and_input(allocator_, table_scan_op, table_scan_input))) { + LOG_WARN("create operator and input failed", K(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(new_sql.length() + PARSE_EXTRA_CHAR_NUM)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc column string buf", K(ret)); + } else { + MEMCPY(buf, new_sql.ptr(), new_sql.length()); + MEMSET(buf + new_sql.length(), '\0', PARSE_EXTRA_CHAR_NUM); + ObString request_sql(new_sql.length() + PARSE_EXTRA_CHAR_NUM, buf); + if (OB_FAIL(table_scan_input->set_db_key_names(shard_connector_array))) { + LOG_WARN("fail to set db key name", K(ret)); + } else if (OB_FAIL(table_scan_input->set_shard_props(shard_prop_array))) { + LOG_WARN("fail to set db key name", K(ret)); + } else if (OB_FAIL(table_scan_input->set_table_name_maps(table_name_map_array))) { + LOG_WARN("fail to set table name", K(ret)); + } else if (OB_FAIL(table_scan_input->set_calc_exprs(calc_exprs_))) { + LOG_WARN("fail to set calc expr", K(ret)); + } else if (OB_FAIL(table_scan_input->set_agg_exprs(agg_exprs_))) { + LOG_WARN("fail to set agg expr", K(ret)); + } else if (OB_FAIL(table_scan_input->set_group_exprs(select_stmt->group_by_exprs_))) { + LOG_WARN("fail to set group exprs", K(ret)); + } else if (OB_FAIL(table_scan_input->set_order_exprs(select_stmt->order_by_exprs_))) { + LOG_WARN("fail to set order exprs", K(ret)); + } else { + table_scan_input->set_request_sql(request_sql); } + + plan_root_ = table_scan_op; } + return ret; } -void ObShardingSelectLogPlan::print_plan_info() +int ObShardingSelectLogPlan::add_projection_operator() { - LOG_DEBUG("plan output info", K(output_expr_array_.count())); - for (int64_t i = 0; i < output_expr_array_.count(); i++) { - ObProxyExpr::print_proxy_expr(output_expr_array_.at(i)); + int ret = OB_SUCCESS; + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + + ObProxyProOp *op = NULL; + ObProxyProInput *input = NULL; + + if (OB_FAIL(create_operator_and_input(allocator_, op, input))) { + LOG_WARN("create projection failed", K(ret)); + } else { + input->set_calc_exprs(calc_exprs_); + input->set_derived_column_count(derived_column_count_); + if (!is_set_limit_) { + input->set_limit_offset(select_stmt->limit_offset_); + input->set_limit_size(select_stmt->limit_size_); + is_set_limit_ = true; + } + if (OB_FAIL(op->set_child(0, plan_root_))) { + LOG_WARN("set child failed", K(ret)); + } else { + plan_root_ = op; + } } - ObProxyOperator::print_execute_plan_info(plan_root_); + + return ret; } -int ObShardingSelectLogPlan::set_shard_connector_array(const ObIArray & array) +int ObShardingSelectLogPlan::compare_group_and_order(bool &is_same_group_and_order) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); i++) { - dbconfig::ObShardConnector *connector = array.at(i); - if (OB_FAIL(shard_connector_array_.push_back(connector))) { - LOG_WARN("shard connector push back failed", K(ret)); + ObProxySelectStmt *select_stmt = static_cast(client_request_.get_parse_result().get_proxy_stmt()); + ObIArray &group_by_exprs = select_stmt->group_by_exprs_; + ObIArray &order_by_exprs = select_stmt->order_by_exprs_; + + if (!group_by_exprs.empty() && !order_by_exprs.empty() + && group_by_exprs.count() == order_by_exprs.count()) { + is_same_group_and_order = true; + for (int64_t i = 0; is_same_group_and_order && i < group_by_exprs.count(); i++) { + ObProxyExpr *group_expr = group_by_exprs.at(i)->get_expr(); + ObProxyExpr *order_expr = order_by_exprs.at(i)->get_expr(); + if (OB_ISNULL(group_expr) || OB_ISNULL(order_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group child or order child is NULL", KP(group_expr), KP(order_expr), K(ret)); + } else { + ObProxyExprType group_expr_type = group_expr->get_expr_type(); + ObProxyExprType order_expr_type = order_expr->get_expr_type(); + if (group_expr_type != order_expr_type) { + is_same_group_and_order = false; + } else if (OB_PROXY_EXPR_TYPE_COLUMN == group_expr_type) { + ObProxyExprColumn* group_expr_column = dynamic_cast(group_expr); + ObProxyExprColumn* order_expr_column = dynamic_cast(order_expr); + if (OB_ISNULL(group_expr_column) || OB_ISNULL(order_expr_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to dynamic cast", KP(group_expr), KP(order_expr), K(ret)); + } else { + ObString &group_column_name = group_expr_column->get_column_name(); + ObString &group_alias_name = group_expr_column->get_alias_name(); + ObString &order_column_name = order_expr_column->get_column_name(); + ObString &order_alias_name = group_expr_column->get_alias_name(); + if (0 != group_column_name.case_compare(order_column_name) + || 0 != group_alias_name.case_compare(order_alias_name)) { + is_same_group_and_order = false; + } + } + } else if (OB_PROXY_EXPR_TYPE_SHARDING_CONST == group_expr_type + || group_expr->is_func_expr()) { + ObString &group_expr_name = group_expr->get_expr_name(); + ObString &order_expr_name = order_expr->get_expr_name(); + if (0 != group_expr_name.case_compare(order_expr_name)) { + is_same_group_and_order = false; + } + } + } } + } else { + is_same_group_and_order = false; } + return ret; } +void ObShardingSelectLogPlan::print_plan_info() +{ + ObProxyOperator::print_execute_plan_info(plan_root_); +} + } // end optimizer } // end obproxy } // end oceanbase diff --git a/src/obproxy/optimizer/ob_sharding_select_log_plan.h b/src/obproxy/optimizer/ob_sharding_select_log_plan.h index c38ee9191d56e27d96ce2e9be383a8d38bf6c65d..1d2d37cbb8ce23b163568b55804f07e4fe978f33 100644 --- a/src/obproxy/optimizer/ob_sharding_select_log_plan.h +++ b/src/obproxy/optimizer/ob_sharding_select_log_plan.h @@ -28,37 +28,69 @@ namespace optimizer class ObShardingSelectLogPlan { public: - ObShardingSelectLogPlan(obutils::ObSqlParseResult &parse_result, - common::ObIAllocator *allocator, - common::ObIArray &physical_table_name_array); + ObShardingSelectLogPlan(proxy::ObProxyMysqlRequest &client_request, + common::ObIAllocator *allocator); ~ObShardingSelectLogPlan(); - int generate_plan(); - int analyze_from_clause(); - int analyze_where_clause(); + int generate_plan(common::ObIArray &shard_connector_array, + common::ObIArray &shard_prop_array, + common::ObIArray > &table_name_map_array); + int analyze_select_clause(); int analyze_group_by_clause(); int analyze_order_by_clause(); - int analyze_select_clause(); + int rewrite_sql(common::ObSqlString &new_sql); + + int add_table_scan_operator(common::ObIArray &shard_connector_array, + common::ObIArray &shard_prop_array, + common::ObIArray > &table_name_map_array, + ObSqlString &new_sql); + int add_mem_merge_agg_operator(bool is_set_limit); + int add_stream_agg_operator(bool is_set_limit); + int add_mem_merge_sort_operator(bool is_set_limit); + int add_stream_sort_operator(bool is_set_limit); + int add_agg_and_sort_operator(bool is_same_group_and_order); int add_projection_operator(); - int traverse_plan_tree(engine::ObProxyOperator *op); + int append_derived_order_by(bool &is_same_group_and_order); + int is_need_derived(opsql::ObProxyExpr *expr, opsql::ObProxyExpr *&exist_expr); + int add_derived_column(opsql::ObProxyExpr *expr); void print_plan_info(); - int set_shard_connector_array(const ObIArray & array); common::ObIAllocator* get_allocator() const { return allocator_; } engine::ObProxyOperator* get_plan_root() const { return plan_root_; } private: - int get_agg_related_expr(opsql::ObProxyExpr* expr, common::ObIArray &array); + template + int add_agg_operator(bool is_set_limit); + template + int add_sort_operator(bool need_set_limit); int handle_avg_expr(opsql::ObProxyExprAvg *expr); + int handle_agg_expr(opsql::ObProxyExpr *expr, bool need_add_calc = false); + int compare_group_and_order(bool &is_same_group_and_order); + int handle_select_derived(opsql::ObProxyExpr *expr); + int do_handle_select_derived(opsql::ObProxyExpr *expr, bool &bret, bool is_root = false); + int handle_derived(opsql::ObProxyExpr *expr, bool column_first = false); + int do_need_derived(common::ObIArray &expr_array, opsql::ObProxyExpr *expr, + bool column_first, bool &bret); + int do_other_need_derived(opsql::ObProxyExpr *tmp_expr, opsql::ObProxyExpr *expr, bool &bret); + int do_other_need_derived_for_avg(opsql::ObProxyExpr *tmp_expr, opsql::ObProxyExpr *expr); + int do_column_need_derived(common::ObIArray &expr_array, opsql::ObProxyExpr *expr, bool &bret); + int do_column_need_derived_with_alias(common::ObIArray &expr_array, opsql::ObProxyExpr *expr, bool &bret); + int do_column_need_derived_with_star(common::ObIArray &expr_array, opsql::ObProxyExpr *expr, bool &bret); + template + int do_handle_avg_expr(opsql::ObProxyExprAvg *agg_expr, T *&expr, + const char* op, ObProxyExprType expr_type); private: - obutils::ObSqlParseResult &parse_result_; + proxy::ObProxyMysqlRequest &client_request_; common::ObIAllocator *allocator_; engine::ObProxyOperator *plan_root_; - common::ObSEArray shard_connector_array_; - common::ObIArray &physical_table_name_array_; - engine::ObProxyTableScanOp *table_scan_; - common::ObSEArray output_expr_array_; + int64_t derived_column_count_; + bool is_set_limit_; + common::ObSEArray agg_exprs_; + common::ObSEArray calc_exprs_; + common::ObSEArray derived_exprs_; + common::ObSEArray derived_columns_; + common::ObSEArray derived_orders_; }; } // end optimizer diff --git a/src/obproxy/packet/ob_mysql_packet_reader.cpp b/src/obproxy/packet/ob_mysql_packet_reader.cpp index ba658669940cecf266ecc852fdc9c3975e9d3335..f5c95effe244f91f2df646e4d8e5d63f26305741 100644 --- a/src/obproxy/packet/ob_mysql_packet_reader.cpp +++ b/src/obproxy/packet/ob_mysql_packet_reader.cpp @@ -111,35 +111,19 @@ inline int ObMysqlPacketReader::get_buf(ObIOBufferReader &buf_reader, const int6 return ret; } -inline int ObMysqlPacketReader::get_content_len(ObIOBufferReader &buf_reader, - const int64_t offset, int64_t &content_len) +inline int ObMysqlPacketReader::get_content_len_and_seq(ObIOBufferReader &buf_reader, + const int64_t offset, int64_t &content_len, uint8_t &seq) { int ret = OB_SUCCESS; char *pbuf = NULL; - if (OB_FAIL(get_buf(buf_reader, OB_MYSQL_CONTENT_LENGTH_ENCODE_SIZE, offset, pbuf))) { - LOG_WARN("fail to get content length buf", K(ret)); + if (OB_FAIL(get_buf(buf_reader, OB_MYSQL_NET_HEADER_LENGTH, offset, pbuf))) { + LOG_WARN("fail to get header buf", K(ret)); } else if (OB_ISNULL(pbuf)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("pbuf is null, which is unexpected", K(pbuf), K(ret)); } else { content_len = static_cast(ob_uint3korr(pbuf)); - } - return ret; -} - -inline int ObMysqlPacketReader::get_seq(ObIOBufferReader &buf_reader, - const int64_t offset, uint8_t &seq) -{ - int ret = OB_SUCCESS; - char *pbuf = NULL; - if (OB_FAIL(get_buf(buf_reader, OB_MYSQL_SEQ_ENCODE_SIZE, - offset + OB_MYSQL_CONTENT_LENGTH_ENCODE_SIZE, pbuf))) { - LOG_WARN("fail to get seq buf", K(ret)); - } else if (OB_ISNULL(pbuf)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("pbuf is null, which is unexpected", K(pbuf), K(ret)); - } else { - seq = static_cast(ob_uint1korr(pbuf)); + seq = static_cast(ob_uint1korr(pbuf + 3)); } return ret; } @@ -154,10 +138,8 @@ int ObMysqlPacketReader::get_ok_packet(ObIOBufferReader &buf_reader, char *content_buf = NULL; int64_t content_len = 0; uint8_t seq = 0; - if (OB_FAIL(get_content_len(buf_reader, offset, content_len))) { - LOG_WARN("fail to get content length", K(content_len), K(offset), K(ret)); - } else if (OB_FAIL(get_seq(buf_reader, offset, seq))) { - LOG_WARN("fail to get seq", K(seq), K(offset), K(ret)); + if (OB_FAIL(get_content_len_and_seq(buf_reader, offset, content_len, seq))) { + LOG_WARN("fail to get content length and seq", K(ret), K(offset), K(content_len), K(seq)); } else if (OB_FAIL(get_buf(buf_reader, content_len, offset + OB_MYSQL_NET_HEADER_LENGTH, content_buf))) { LOG_WARN("fail to get content buf", K(content_len), K(content_buf), K(offset), K(ret)); } else { @@ -182,10 +164,8 @@ int ObMysqlPacketReader::get_ok_packet_server_status(ObIOBufferReader &buf_reade int64_t content_len = 0; uint8_t seq = 0; int64_t offset = 0; - if (OB_FAIL(get_content_len(buf_reader, offset, content_len))) { - LOG_WARN("fail to get content length", K(content_len), K(offset), K(ret)); - } else if (OB_FAIL(get_seq(buf_reader, offset, seq))) { - LOG_WARN("fail to get seq", K(seq), K(offset), K(ret)); + if (OB_FAIL(get_content_len_and_seq(buf_reader, offset, content_len, seq))) { + LOG_WARN("fail to get content length and seq", K(content_len), K(seq), K(offset), K(ret)); } else if (OB_FAIL(get_buf(buf_reader, content_len, offset + OB_MYSQL_NET_HEADER_LENGTH, content_buf))) { LOG_WARN("fail to get content buf", K(content_len), K(content_buf), K(offset), K(ret)); } else { @@ -217,10 +197,8 @@ int ObMysqlPacketReader::get_packet(ObIOBufferReader &buf_reader, int64_t content_len = 0; uint8_t seq = 0; int64_t offset = 0; - if (OB_FAIL(get_content_len(buf_reader, offset, content_len))) { - LOG_WARN("fail to get content length", K(content_len), K(offset), K(ret)); - } else if (OB_FAIL(get_seq(buf_reader, offset, seq))) { - LOG_WARN("fail to get seq", K(seq), K(offset), K(ret)); + if (OB_FAIL(get_content_len_and_seq(buf_reader, offset, content_len, seq))) { + LOG_WARN("fail to get content length and seq", K(content_len), K(seq), K(offset), K(ret)); } else if (OB_FAIL(get_buf(buf_reader, content_len, offset + OB_MYSQL_NET_HEADER_LENGTH, content_buf))) { LOG_WARN("fail to get content buf", K(content_len), K(content_buf), K(offset), K(ret)); } else { diff --git a/src/obproxy/packet/ob_mysql_packet_reader.h b/src/obproxy/packet/ob_mysql_packet_reader.h index 1a7684ed7709e341a491eb613fe3e44f8204cdd2..75dadab073f330365a94266a1d1a235c0df695ec 100644 --- a/src/obproxy/packet/ob_mysql_packet_reader.h +++ b/src/obproxy/packet/ob_mysql_packet_reader.h @@ -42,8 +42,7 @@ public: void reset(); // === GET function will not comsume the packet - int get_content_len(event::ObIOBufferReader &buf_reader, const int64_t offset, int64_t &content_len); - int get_seq(event::ObIOBufferReader &buf_reader, const int64_t offset, uint8_t &seq); + int get_content_len_and_seq(event::ObIOBufferReader &buf_reader, const int64_t offset, int64_t &content_len, uint8_t &seq); // DESC: get the next ok packet from MIOBuffer // NOTE: 1. get function will NOT consume buffer as packet may hold some string int mio_buf diff --git a/src/obproxy/packet/ob_mysql_packet_util.cpp b/src/obproxy/packet/ob_mysql_packet_util.cpp index 4ee992a4c5856c96aa8e5e0b206932755505b32b..e8086c9f35daf55b1bbda6bfcb5bb017a9b9e598 100644 --- a/src/obproxy/packet/ob_mysql_packet_util.cpp +++ b/src/obproxy/packet/ob_mysql_packet_util.cpp @@ -71,11 +71,12 @@ int ObMysqlPacketUtil::encode_header(ObMIOBuffer &write_buf, int ObMysqlPacketUtil::encode_row_packet(ObMIOBuffer &write_buf, uint8_t &seq, - const ObNewRow &row) + const ObNewRow &row, + ObIArray *fields) { int ret = OB_SUCCESS; - OMPKRow row_packet(ObSMRow(TEXT, row)); + OMPKRow row_packet(ObSMRow(TEXT, row, NULL, fields)); row_packet.set_seq(seq++); if (OB_FAIL(ObMysqlPacketWriter::write_row_packet(write_buf, row_packet))) { diff --git a/src/obproxy/packet/ob_mysql_packet_util.h b/src/obproxy/packet/ob_mysql_packet_util.h index 10fb403ddb183899d89db5c5ee62adfadd96f8d1..6ad9b1cf571589b6e9dd8ae97382a2d18f12df00 100644 --- a/src/obproxy/packet/ob_mysql_packet_util.h +++ b/src/obproxy/packet/ob_mysql_packet_util.h @@ -15,6 +15,7 @@ #include "lib/container/ob_array.h" #include "rpc/obmysql/ob_mysql_field.h" +#include "common/ob_field.h" #include "rpc/obmysql/ob_mysql_packet.h" namespace oceanbase @@ -43,7 +44,8 @@ public: common::ObIArray &fields, uint16_t status_flag = 0); static int encode_row_packet(event::ObMIOBuffer &write_buf, uint8_t &seq, - const common::ObNewRow &row); + const common::ObNewRow &row, + common::ObIArray *fields = NULL); static int encode_eof_packet(event::ObMIOBuffer &write_buf, uint8_t &seq, uint16_t status_flag = 0); static int encode_err_packet(event::ObMIOBuffer &write_buf, uint8_t &seq, int errcode); diff --git a/src/obproxy/prometheus/Makemodule.am b/src/obproxy/prometheus/Makemodule.am index 89b09946529300e2b36d94d7087c6f77322a2ef7..e2d0f0ca8281d653c632e53309e38b3877d37a4b 100644 --- a/src/obproxy/prometheus/Makemodule.am +++ b/src/obproxy/prometheus/Makemodule.am @@ -20,4 +20,6 @@ obproxy/prometheus/ob_sql_prometheus.h\ obproxy/prometheus/ob_net_prometheus.cpp\ obproxy/prometheus/ob_net_prometheus.h\ obproxy/prometheus/ob_route_prometheus.cpp\ -obproxy/prometheus/ob_route_prometheus.h +obproxy/prometheus/ob_route_prometheus.h\ +obproxy/prometheus/ob_thread_prometheus.cpp\ +obproxy/prometheus/ob_thread_prometheus.h diff --git a/src/obproxy/prometheus/ob_prometheus_exporter.cpp b/src/obproxy/prometheus/ob_prometheus_exporter.cpp index 830f0501bf8e7a0a009f810a20b2d6d0cf3d6a26..d329dfeb4b2dbdfcbb23ff1e50a2dba67b1d5700 100644 --- a/src/obproxy/prometheus/ob_prometheus_exporter.cpp +++ b/src/obproxy/prometheus/ob_prometheus_exporter.cpp @@ -31,14 +31,34 @@ int ObProxyPrometheusExporter::init(int32_t listen_port) } else if (!(registry_ = std::make_shared())) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { + listen_port_ = listen_port; + if (OB_FAIL(create_exposer())) { + } else { + is_inited_ = true; + } + } + + return ret; +} + +void ObProxyPrometheusExporter::destroy_exposer() +{ + if (NULL != exposer_) { + delete exposer_; + exposer_ = NULL; + } +} + +int ObProxyPrometheusExporter::create_exposer() +{ + int ret = OB_SUCCESS; + + if (OB_LIKELY(NULL == exposer_)) { try { - if (!(exposer_ = new Exposer(std::to_string(listen_port)))) { + if (OB_UNLIKELY(NULL == (exposer_ = new Exposer(std::to_string(listen_port_))))) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { - listen_port_ = listen_port; - exposer_->RegisterCollectable(registry_); - is_inited_ = true; } } catch (...) { ret = OB_INVALID_ARGUMENT; @@ -124,6 +144,16 @@ int init_prometheus(int32_t listen_port) return get_obproxy_prometheus_exporter().init(listen_port); } +void destory_prometheus_exposer() +{ + get_obproxy_prometheus_exporter().destroy_exposer(); +} + +int create_prometheus_exposer() +{ + return get_obproxy_prometheus_exporter().create_exposer(); +} + } // end of namespace prometheus } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/prometheus/ob_prometheus_exporter.h b/src/obproxy/prometheus/ob_prometheus_exporter.h index b4af47f0b9592d3367ffd455670ba7bf88105991..fa345b8c8be10b3b16db7be43b0f3a892e1c7e05 100644 --- a/src/obproxy/prometheus/ob_prometheus_exporter.h +++ b/src/obproxy/prometheus/ob_prometheus_exporter.h @@ -34,6 +34,8 @@ public: ~ObProxyPrometheusExporter() {}; int init(int32_t listen_port); + int create_exposer(); + void destroy_exposer(); int get_or_create_counter_family(const std::string& name, const std::string& help, const std::map& labels, diff --git a/src/obproxy/prometheus/ob_prometheus_info.h b/src/obproxy/prometheus/ob_prometheus_info.h index 64fad4d5b27e5157542887c1be19dce8483a5b54..bd10d56a9cf6d4bf904bcc60702822714ddea08f 100644 --- a/src/obproxy/prometheus/ob_prometheus_info.h +++ b/src/obproxy/prometheus/ob_prometheus_info.h @@ -49,6 +49,7 @@ enum ObPrometheusMetrics PROMETHEUS_REQUEST_TOTAL_TIME, PROMETHEUS_CURRENT_SESSION, + PROMETHEUS_USED_CONNECTIONS, PROMETHEUS_ENTRY_LOOKUP_COUNT, PROMETHEUS_REQUEST_BYTE, diff --git a/src/obproxy/prometheus/ob_prometheus_processor.cpp b/src/obproxy/prometheus/ob_prometheus_processor.cpp index 6fb21fc5d30d29d108452382be1e7f125497da39..a3e3b82aa23e28e3b24ef3ae9edfd048a80523ca 100644 --- a/src/obproxy/prometheus/ob_prometheus_processor.cpp +++ b/src/obproxy/prometheus/ob_prometheus_processor.cpp @@ -18,9 +18,12 @@ #include "obutils/ob_proxy_table_processor_utils.h" #include "obutils/ob_proxy_config.h" #include "utils/ob_proxy_monitor_utils.h" +#include "prometheus/ob_thread_prometheus.h" +#include "iocore/net/ob_net_def.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::obutils; +using namespace oceanbase::obproxy::event; namespace oceanbase { @@ -29,9 +32,35 @@ namespace obproxy namespace prometheus { int init_prometheus(int32_t listen_port); +void destory_prometheus_exposer(); +int create_prometheus_exposer(); ObPrometheusProcessor g_ob_prometheus_processor; +int ObPrometheusProcessor::start_prometheus() +{ + int ret = OB_SUCCESS; + // Initialize thread-level prometheus statistics + int64_t net_thread_count = g_event_processor.thread_count_for_type_[ET_NET]; + ObEThread **ethreads = g_event_processor.event_thread_[ET_NET]; + for (int64_t i = 0; OB_SUCC(ret) && i < net_thread_count; i++) { + if (OB_ISNULL(ethreads[i]->thread_prometheus_ = new(std::nothrow) ObThreadPrometheus())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObThreadPrometheus", K(i), K(ret)); + } else if (OB_FAIL(ethreads[i]->thread_prometheus_->init(ethreads[i]))) { + LOG_WARN("fail to init thread prometheus", K(i), K(ret)); + } + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(start_prometheus_task())) { + LOG_WARN("start prometheus failed", K(ret)); + } + + return ret; +} + int ObPrometheusProcessor::start_prometheus_task() { int ret = OB_SUCCESS; @@ -424,6 +453,16 @@ int ObPrometheusProcessor::get_or_create_family(const ObString &name, const ObSt return ret; } +void ObPrometheusProcessor::destroy_exposer() +{ + destory_prometheus_exposer(); +} + +int ObPrometheusProcessor::create_exposer() +{ + return create_prometheus_exposer(); +} + int ObPrometheusProcessor::init() { int ret = OB_SUCCESS; @@ -448,7 +487,13 @@ int ObPrometheusProcessor::init() offset += strlen(package); MEMCPY(version_ + offset, "_", 1); offset ++; - MEMCPY(version_ + offset, version, strlen(version) - 34); + + static const size_t minimumVersionLength = 17; + size_t version_len = strlen(version); + if (version_len > minimumVersionLength) { + version_len = minimumVersionLength; + } + MEMCPY(version_ + offset, version, version_len); ObProxyPrometheusUtils::build_label(default_constant_labels_, "ip", proxy_ip_, false); ObProxyPrometheusUtils::build_label(default_constant_labels_, "namespace", "ODP", false); diff --git a/src/obproxy/prometheus/ob_prometheus_processor.h b/src/obproxy/prometheus/ob_prometheus_processor.h index f32fd04033cfe348fb59bdfac187a79c5c49f1fd..c41e33d9a219d5a781434cee6bb90cba6e198b97 100644 --- a/src/obproxy/prometheus/ob_prometheus_processor.h +++ b/src/obproxy/prometheus/ob_prometheus_processor.h @@ -34,8 +34,11 @@ public: ~ObPrometheusProcessor(); int init(); + int create_exposer(); + void destroy_exposer(); bool is_inited() { return is_inited_; } + int start_prometheus(); int start_prometheus_task(); int handle_counter(const char *name_ptr, const char *help_ptr, diff --git a/src/obproxy/prometheus/ob_prometheus_utils.h b/src/obproxy/prometheus/ob_prometheus_utils.h index 688055d7abe047f42dbd1d3918a0cb40bc40bf2c..22def2aa6252c8963f8cfbac63ba071749fd4640 100644 --- a/src/obproxy/prometheus/ob_prometheus_utils.h +++ b/src/obproxy/prometheus/ob_prometheus_utils.h @@ -32,6 +32,8 @@ namespace prometheus #define COST_TOTAL_HELP "user cost total" #define CURRENT_SESSION "odp_current_session" #define CURRENT_SESSION_HELP "The num of current session" +#define USED_CONNECTIONS "odp_used_connections" +#define USED_CONNECTIONS_HELP "The num of used connections" #define REQUEST_BYTE "odp_request_byte" #define REQUEST_BYTE_HELP "The num of request byte" @@ -61,6 +63,7 @@ namespace prometheus #define LABEL_SUCC "success" #define LABEL_FALSE "false" #define LABEL_TRUE "true" +#define LABEL_VIP "vip" class ObProxyPrometheusUtils { diff --git a/src/obproxy/prometheus/ob_sql_prometheus.cpp b/src/obproxy/prometheus/ob_sql_prometheus.cpp index e10b574b02b49b9a2671833d74e84145b0ec3369..922fda3d87231f1470f14418210e46371cc5f985 100644 --- a/src/obproxy/prometheus/ob_sql_prometheus.cpp +++ b/src/obproxy/prometheus/ob_sql_prometheus.cpp @@ -38,11 +38,11 @@ int ObSQLPrometheus::handle_prometheus(const ObString &logic_tenant_name, va_list args; va_start(args, metric); - + const ObString vip_addr_name; if (OB_FAIL(handle_prometheus(logic_tenant_name, logic_database_name, cluster_name, - tenant_name, database_name, stmt_type, metric, args))) { + tenant_name, vip_addr_name, database_name, stmt_type, metric, args))) { LOG_WARN("fail to handle_prometheus with ObClientSessionInfo", K(logic_tenant_name), K(logic_database_name), - K(cluster_name), K(tenant_name), K(database_name), K(metric), K(ret)); + K(cluster_name), K(tenant_name), K(vip_addr_name), K(database_name), K(metric), K(ret)); } va_end(args); @@ -60,6 +60,7 @@ int ObSQLPrometheus::handle_prometheus(const ObClientSessionInfo &cs_info, ObString cluster_name; ObString tenant_name; ObString database_name; + ObString vip_addr_name; cs_info.get_logic_tenant_name(logic_tenant_name); if (OB_UNLIKELY(logic_tenant_name.empty())) { @@ -74,15 +75,18 @@ int ObSQLPrometheus::handle_prometheus(const ObClientSessionInfo &cs_info, cs_info.get_cluster_name(cluster_name); cs_info.get_tenant_name(tenant_name); cs_info.get_database_name(database_name); + if (get_global_proxy_config().need_convert_vip_to_tname) { + cs_info.get_vip_addr_name(vip_addr_name); + } } va_list args; va_start(args, metric); if (OB_FAIL(handle_prometheus(logic_tenant_name, logic_database_name, cluster_name, - tenant_name, database_name, OBPROXY_T_MAX, metric, args))) { + tenant_name, vip_addr_name, database_name, OBPROXY_T_MAX, metric, args))) { LOG_WARN("fail to handle_prometheus with ObClientSessionInfo", K(logic_tenant_name), K(logic_database_name), - K(cluster_name), K(tenant_name), K(database_name), K(metric), K(ret)); + K(cluster_name), K(tenant_name), K(vip_addr_name), K(database_name), K(metric), K(ret)); } va_end(args); @@ -94,6 +98,7 @@ int ObSQLPrometheus::handle_prometheus(const ObString &logic_tenant_name, const ObString &logic_database_name, const ObString &cluster_name, const ObString &tenant_name, + const ObString &vip_addr_name, const ObString &database_name, const ObProxyBasicStmtType stmt_type, const ObPrometheusMetrics metric, @@ -169,6 +174,18 @@ int ObSQLPrometheus::handle_prometheus(const ObString &logic_tenant_name, } break; } + case PROMETHEUS_USED_CONNECTIONS: + { + int32_t value = va_arg(args, int32_t); + + ObProxyPrometheusUtils::build_label(label_vector, LABEL_VIP, vip_addr_name, false); + + if (OB_FAIL(g_ob_prometheus_processor.handle_gauge(USED_CONNECTIONS, USED_CONNECTIONS_HELP, + label_vector, value, false))) { + LOG_WARN("fail to handle counter with USED_CONNECTIONS", K(ret)); + } + break; + } default: break; } diff --git a/src/obproxy/prometheus/ob_sql_prometheus.h b/src/obproxy/prometheus/ob_sql_prometheus.h index ec19fbce29135b0d4798f54f2c2a6fbb8f39399f..65500b254322f64f3adf737c7ee7216cc4760ffc 100644 --- a/src/obproxy/prometheus/ob_sql_prometheus.h +++ b/src/obproxy/prometheus/ob_sql_prometheus.h @@ -42,6 +42,7 @@ private: const common::ObString &logic_database_name, const common::ObString &cluster_name, const common::ObString &tenant_name, + const common::ObString &vip_addr_name, const common::ObString &database_name, const ObProxyBasicStmtType stmt_type, const ObPrometheusMetrics metric, diff --git a/src/obproxy/prometheus/ob_thread_prometheus.cpp b/src/obproxy/prometheus/ob_thread_prometheus.cpp new file mode 100644 index 0000000000000000000000000000000000000000..687942be6ef8c1aa4322f65a008f18dd21e49570 --- /dev/null +++ b/src/obproxy/prometheus/ob_thread_prometheus.cpp @@ -0,0 +1,277 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define USING_LOG_PREFIX PROXY + +#include "prometheus/ob_thread_prometheus.h" +#include "obutils/ob_proxy_config.h" +#include "utils/ob_proxy_hot_upgrader.h" +#include "iocore/net/ob_net_def.h" +#include "opsql/parser/ob_proxy_parse_result.h" +#include "prometheus/ob_prometheus_info.h" +#include "prometheus/ob_sql_prometheus.h" + +using namespace oceanbase::common; +using namespace oceanbase::common::hash; +using namespace oceanbase::obproxy::net; +using namespace oceanbase::obproxy::event; +using namespace oceanbase::obproxy::obutils; + +namespace oceanbase +{ +namespace obproxy +{ +namespace prometheus +{ + +int64_t SQLMonitorInfo::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(request_count), K_(request_total_time), K_(server_process_request_time), + K_(prepare_send_request_to_server_time), K_(cluster_name), K_(tenant_name)); + J_OBJ_END(); + return pos; +} + +int ObSQLMonitorInfoCont::init(int64_t report_interval_us, ObEThread *thread, ObThreadPrometheus *thread_prometheus) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(report_interval_us <= 0 || NULL == thread || NULL == thread_prometheus)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(report_interval_us), K(ret)); + } else { + report_interval_us_ = report_interval_us; + thread_ = thread; + thread_prometheus_ = thread_prometheus; + is_inited_ = true; + } + + return ret; +} + +int ObSQLMonitorInfoCont::set_report_interval(const int64_t interval) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(interval <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid interval value", K(interval), K(ret)); + } else { + report_interval_us_ = interval; + } + + return ret; +} + +int ObSQLMonitorInfoCont::schedule_report_prometheus_info() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K_(is_inited), K(ret)); + } else if (get_global_hot_upgrade_info().is_graceful_exit_timeout(get_hrtime())) { + LOG_WARN("proxy need exit now"); + } else if (OB_UNLIKELY(!self_ethread().is_event_thread_type(ET_NET))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("sql monitor info cont must be scheduled in net thread", K(ret)); + } else if (OB_ISNULL(thread_->schedule_in(this, HRTIME_USECONDS(report_interval_us_), EVENT_NONE))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to schedule report prometheus info", K(ret)); + } + + return ret; +} + +int ObSQLMonitorInfoCont::main_handler(int event, void *data) +{ + UNUSED(event); + UNUSED(data); + int ret = OB_SUCCESS; + + for (int64_t i = 0; i < thread_prometheus_->monitor_info_used_; i++) { + SQLMonitorInfo &info = thread_prometheus_->monitor_info_array_[i]; + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_REQUEST_COUNT, false, false); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_REQUEST_TOTAL_TIME, hrtime_to_usec(info.request_total_time_)); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_SERVER_PROCESS_REQUEST_TIME, hrtime_to_usec(info.server_process_request_time_)); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_PREPARE_SEND_REQUEST_TIME, hrtime_to_usec(info.prepare_send_request_to_server_time_)); + } + thread_prometheus_->monitor_info_used_ = 0; + memset(thread_prometheus_->monitor_info_array_, 0, sizeof(thread_prometheus_->monitor_info_array_)); + + MonitorInfoHashMap::iterator iter = thread_prometheus_->monitor_info_hash_map_.begin(); + for(; iter != thread_prometheus_->monitor_info_hash_map_.end(); ++iter) { + SQLMonitorInfo &info = iter->second; + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_REQUEST_COUNT, false, false); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_REQUEST_TOTAL_TIME, hrtime_to_usec(info.request_total_time_)); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_SERVER_PROCESS_REQUEST_TIME, hrtime_to_usec(info.server_process_request_time_)); + SQL_PROMETHEUS_STAT("", "", info.cluster_name_, info.tenant_name_, + "", OBPROXY_T_INVALID, PROMETHEUS_PREPARE_SEND_REQUEST_TIME, hrtime_to_usec(info.prepare_send_request_to_server_time_)); + } + thread_prometheus_->monitor_info_hash_map_.reuse(); + + if (OB_FAIL(schedule_report_prometheus_info())) { + LOG_WARN("schedule report prometheus info failed", K(ret)); + } + return ret; +} + +void ObSQLMonitorInfoCont::kill_this() +{ + if (is_inited_) { + LOG_INFO("ObSQLMonitorInfoCont will kill self"); + report_interval_us_ = 0; + thread_ = NULL; + } + + op_free(this); +} + +int ObThreadPrometheus::init(ObEThread *thread) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(NULL == thread)) { + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(monitor_info_hash_map_.create(16, ObModIds::OB_PROMETHEUS_RELATED))) { + LOG_WARN("monitor info hash map create failed", K(ret)); + } else if (OB_ISNULL(sql_monitor_info_cont_ = op_alloc(ObSQLMonitorInfoCont))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc ObSQLMonitorInfoCont", K(ret)); + } else if (OB_FAIL(sql_monitor_info_cont_->init( + get_global_proxy_config().prometheus_sync_interval / 5, thread, this))) { + LOG_WARN("set report interval failed", K(ret)); + } else if (OB_FAIL(sql_monitor_info_cont_->schedule_report_prometheus_info())) { + LOG_WARN("schedule report prometheus info failed", K(ret)); + } else { + thread_ = thread; + } + + if (OB_FAIL(ret)) { + if (NULL != sql_monitor_info_cont_) { + sql_monitor_info_cont_->kill_this(); + sql_monitor_info_cont_ = NULL; + } + thread_ = NULL; + } + + return ret; +} + +int ObThreadPrometheus::set_sql_monitor_info(const ObString &tenant_name, + const ObString &cluster_name, + const SQLMonitorInfo &info) +{ + int ret = OB_SUCCESS; + if (set_sql_monitor_info_using_array(tenant_name, cluster_name, info)) { + LOG_DEBUG("set sql monitor info using array", K(tenant_name), K(cluster_name), K(ret)); + } else if (OB_FAIL(set_sql_monitor_info_using_hashmap(tenant_name, cluster_name, info))) { + LOG_WARN("set sql monitor info using hashmap failed", K(tenant_name), K(cluster_name), K(info), K(ret)); + } + + return ret; +} + +int ObThreadPrometheus::set_sql_monitor_info_using_hashmap(const ObString &tenant_name, + const ObString &cluster_name, + const SQLMonitorInfo &info) +{ + int ret = OB_SUCCESS; + ObFixedLengthString key_string; + SQLMonitorInfo monitor_info; + if (OB_UNLIKELY(tenant_name.empty() || cluster_name.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(tenant_name), K(cluster_name), K(ret)); + } else if (OB_FAIL(paste_tenant_and_cluster_name(tenant_name, cluster_name, key_string))) { + LOG_WARN("paste tenant and cluster name failed", K(tenant_name), K(cluster_name), K(ret)); + } else if (OB_FAIL(monitor_info_hash_map_.get_refactored(key_string, monitor_info))) { + if (OB_HASH_NOT_EXIST == ret) { + memset(&monitor_info, 0, sizeof(monitor_info)); + MEMCPY(monitor_info.cluster_name_str_, cluster_name.ptr(), cluster_name.length()); + MEMCPY(monitor_info.tenant_name_str_, tenant_name.ptr(), tenant_name.length()); + ret = OB_SUCCESS; + } + } + + if (OB_SUCC(ret)) { + monitor_info.request_count_ += info.request_count_; + monitor_info.request_total_time_ += info.request_count_; + monitor_info.server_process_request_time_ += info.server_process_request_time_; + monitor_info.prepare_send_request_to_server_time_ += info.prepare_send_request_to_server_time_; + if (OB_FAIL(monitor_info_hash_map_.set_refactored(key_string, monitor_info, 1))) { + LOG_WARN("monitor info hash map set failed", K(ret)); + } + } + + return ret; +} + +bool ObThreadPrometheus::set_sql_monitor_info_using_array(const ObString &tenant_name, + const ObString &cluster_name, + const SQLMonitorInfo &info) +{ + bool bret = false; + int64_t index = -1; + for (int64_t i = 0; !bret && i < monitor_info_used_; i++) { + if (monitor_info_array_[i].tenant_name_ == tenant_name && monitor_info_array_[i].cluster_name_ == cluster_name) { + bret = true; + index = i; + } + } + + if (!bret && monitor_info_used_ < SQL_MONITOR_INFO_ARRAY_SIZE) { + memset(&monitor_info_array_[monitor_info_used_], 0, sizeof(monitor_info_array_[monitor_info_used_])); + MEMCPY(monitor_info_array_[monitor_info_used_].cluster_name_str_, cluster_name.ptr(), cluster_name.length()); + MEMCPY(monitor_info_array_[monitor_info_used_].tenant_name_str_, tenant_name.ptr(), tenant_name.length()); + monitor_info_array_[monitor_info_used_].tenant_name_.assign_ptr(monitor_info_array_[monitor_info_used_].tenant_name_str_, tenant_name.length()); + monitor_info_array_[monitor_info_used_].cluster_name_.assign_ptr(monitor_info_array_[monitor_info_used_].cluster_name_str_, cluster_name.length()); + index = monitor_info_used_; + monitor_info_used_++; + bret = true; + } + + if (bret) { + monitor_info_array_[index].request_count_ += info.request_count_; + monitor_info_array_[index].request_total_time_ += info.request_count_; + monitor_info_array_[index].server_process_request_time_ += info.server_process_request_time_; + monitor_info_array_[index].prepare_send_request_to_server_time_ += info.prepare_send_request_to_server_time_; + } + + return bret; +} + +} // end of prometheus +} // end of obproxy +} // end of oceanbase diff --git a/src/obproxy/prometheus/ob_thread_prometheus.h b/src/obproxy/prometheus/ob_thread_prometheus.h new file mode 100644 index 0000000000000000000000000000000000000000..8bc27f4faede1bc6287fabcaa905289d05db85f5 --- /dev/null +++ b/src/obproxy/prometheus/ob_thread_prometheus.h @@ -0,0 +1,124 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_THREAD_PROMETHEUS_PROCESSOR_H_ +#define OB_THREAD_PROMETHEUS_PROCESSOR_H_ + +#include "lib/hash/ob_hashmap.h" +#include "lib/string/ob_string.h" +#include "obutils/ob_async_common_task.h" +#include "iocore/eventsystem/ob_continuation.h" +#include "iocore/eventsystem/ob_ethread.h" + +#define SQL_MONITOR_INFO_ARRAY_SIZE 4 + +namespace oceanbase +{ +namespace obproxy +{ +namespace prometheus +{ + +struct SQLMonitorInfo { + int64_t request_count_; + int64_t request_total_time_; + int64_t server_process_request_time_; + int64_t prepare_send_request_to_server_time_; + common::ObString cluster_name_; + common::ObString tenant_name_; + + char cluster_name_str_[OB_PROXY_MAX_CLUSTER_NAME_LENGTH]; + char tenant_name_str_[oceanbase::common::OB_MAX_TENANT_NAME_LENGTH]; + + int64_t to_string(char *buf, const int64_t buf_len) const; +}; + +typedef common::hash::ObHashMap, SQLMonitorInfo> MonitorInfoHashMap; + +class ObSQLMonitorInfoCont : public event::ObContinuation +{ +public: + ObSQLMonitorInfoCont() : ObContinuation(NULL), is_inited_(false), report_interval_us_(0), + thread_(NULL), thread_prometheus_(NULL) + { + SET_HANDLER(&ObSQLMonitorInfoCont::main_handler); + } + ~ObSQLMonitorInfoCont() {} + int init(int64_t report_interval_us, event::ObEThread *thread, ObThreadPrometheus *thread_prometheus); + int set_report_interval(const int64_t interval); + int schedule_report_prometheus_info(); + void kill_this(); + +private: + int main_handler(int event, void *data); + +private: + bool is_inited_; + int64_t report_interval_us_; + event::ObEThread *thread_; + ObThreadPrometheus *thread_prometheus_; + + DISALLOW_COPY_AND_ASSIGN(ObSQLMonitorInfoCont); +}; + +class ObThreadPrometheus +{ +public: + int init(event::ObEThread *thread); + ObThreadPrometheus() : monitor_info_used_(0), monitor_info_hash_map_(), sql_monitor_info_cont_(NULL), thread_(NULL) {} + ~ObThreadPrometheus() {} + int set_sql_monitor_info(const common::ObString &tenant_name, const common::ObString &cluster_name, const SQLMonitorInfo &info); + +private: + bool set_sql_monitor_info_using_array(const common::ObString &tenant_name, const common::ObString &cluster_name, const SQLMonitorInfo &info); + int set_sql_monitor_info_using_hashmap(const common::ObString &tenant_name, const common::ObString &cluster_name, const SQLMonitorInfo &info); + +public: + // For monitoring information, it is stored in the form of array + hashmap, + // mainly because the performance of get and set operations of hashmap has a great impact. + // About 3% loss, use hashmap when the data is full + // If the CPU tenant isolation is implemented later, + // the tenant corresponding to a thread is fixed, and only one position in the array is used. + SQLMonitorInfo monitor_info_array_[SQL_MONITOR_INFO_ARRAY_SIZE]; + int64_t monitor_info_used_; + MonitorInfoHashMap monitor_info_hash_map_; +private: + ObSQLMonitorInfoCont *sql_monitor_info_cont_; + event::ObEThread *thread_; + +private: + DISALLOW_COPY_AND_ASSIGN(ObThreadPrometheus); +}; + +} // end of prometheus +} // end of obproxy +} // end of oceanbase + +#endif diff --git a/src/obproxy/proxy/api/ob_api.cpp b/src/obproxy/proxy/api/ob_api.cpp index 22f7ffb896c12e45738aa5c05a775a20e8418093..d1aee20f62c8c13ce33de541659409b7595f9329 100644 --- a/src/obproxy/proxy/api/ob_api.cpp +++ b/src/obproxy/proxy/api/ob_api.cpp @@ -22,6 +22,7 @@ #include "proxy/plugins/ob_mysql_request_compress_transform_plugin.h" #include "proxy/plugins/ob_mysql_response_cursor_transform_plugin.h" #include "proxy/plugins/ob_mysql_response_new_ps_transform_plugin.h" +#include "proxy/plugins/ob_mysql_request_execute_transform_plugin.h" /**************************************************************** * IMPORTANT - READ ME @@ -451,6 +452,7 @@ int api_init() //init_null_transform(); //init_trim_okpacket_transform(); init_mysql_request_prepare_transform(); + init_mysql_request_execute_transform(); init_mysql_request_compress_transform(); init_mysql_resposne_compress_transform(); init_mysql_response_prepare_transform(); diff --git a/src/obproxy/proxy/api/ob_intercept_plugin.cpp b/src/obproxy/proxy/api/ob_intercept_plugin.cpp index e772217e49ac634cbab645c4686e908ec92b4a8b..c7137010d66b2a6103644d1559c981e6be680217 100644 --- a/src/obproxy/proxy/api/ob_intercept_plugin.cpp +++ b/src/obproxy/proxy/api/ob_intercept_plugin.cpp @@ -193,6 +193,7 @@ int ObInterceptPlugin::handle_event_internal(ObEventType event, void *edata) // else fall through into the next shut down cases WARN_API("Error while reading request!"); + __attribute__ ((fallthrough)); case OB_EVENT_VCONN_READ_COMPLETE: // fall through intentional case OB_EVENT_VCONN_WRITE_COMPLETE: diff --git a/src/obproxy/proxy/api/ob_mysql_sm_api.cpp b/src/obproxy/proxy/api/ob_mysql_sm_api.cpp index 6b4b67e6e48bd4f351f0fd856998546666c8e17e..92feba16f34bbb0107fcace911561c1326a84716 100644 --- a/src/obproxy/proxy/api/ob_mysql_sm_api.cpp +++ b/src/obproxy/proxy/api/ob_mysql_sm_api.cpp @@ -942,10 +942,11 @@ int ObMysqlSMApi::setup_server_transfer_to_transform() const uint8_t req_seq = sm_->get_request_seq(); const obmysql::ObMySQLCmd cmd = sm_->get_request_cmd(); ObMysqlCompressAnalyzer *compress_analyzer = &sm_->get_compress_analyzer(); + const ObMysqlProtocolMode mysql_mode = sm_->client_session_->get_session_info().is_oracle_mode() ? OCEANBASE_ORACLE_PROTOCOL_MODE : OCEANBASE_MYSQL_PROTOCOL_MODE; compress_analyzer->reset(); const bool enable_extra_ok_packet_for_stats = sm_->is_extra_ok_packet_for_stats_enabled(); if (OB_FAIL(compress_analyzer->init(req_seq, ObMysqlCompressAnalyzer::SIMPLE_MODE, - cmd, enable_extra_ok_packet_for_stats, req_seq, sm_->get_server_session()->get_server_request_id(), + cmd, mysql_mode, enable_extra_ok_packet_for_stats, req_seq, sm_->get_server_session()->get_server_request_id(), sm_->get_server_session()->get_server_sessid()))) { LOG_WARN("fail to init compress analyzer", K(req_seq), K(cmd), K(enable_extra_ok_packet_for_stats), K(ret)); diff --git a/src/obproxy/proxy/client/ob_client_vc.cpp b/src/obproxy/proxy/client/ob_client_vc.cpp index 4eb8fa7926519def5bd6bf78f94adea95ed0e1bc..69c1f205f76b6c147bd4265ad29284656b712339 100644 --- a/src/obproxy/proxy/client/ob_client_vc.cpp +++ b/src/obproxy/proxy/client/ob_client_vc.cpp @@ -16,6 +16,7 @@ #include "proxy/client/ob_mysql_client_pool.h" #include "proxy/mysqllib/ob_mysql_request_builder.h" #include "proxy/mysql/ob_mysql_client_session.h" +#include "obutils/ob_resource_pool_processor.h" using namespace oceanbase::common; using namespace oceanbase::obmysql; @@ -969,6 +970,53 @@ int ObMysqlClient::setup_read_normal_resp() return ret; } +int ObMysqlClient::do_new_connection_with_shard_conn(ObMysqlClientSession *client_session) +{ + int ret = OB_SUCCESS; + + dbconfig::ObShardConnector *shard_conn = pool_->acquire_shard_conn(); // inc ref + dbconfig::ObShardProp *shard_prop = pool_->acquire_shard_prop(); // inc ref + LOG_DEBUG("new connection", KP(shard_conn), KP(shard_prop)); + if (OB_ISNULL(shard_conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("shard conn should not be null here", K(ret)); + client_session->destroy(); + } else if (OB_FAIL(client_session->new_connection(client_vc_, request_buf_, request_reader_, + shard_conn, shard_prop))) { + LOG_WARN("fail to new_connection", K(ret)); + } + + if (NULL != shard_conn) { + shard_conn->dec_ref(); + } + + if (NULL != shard_prop) { + shard_prop->dec_ref(); + } + + return ret; +} + +int ObMysqlClient::do_new_connection_with_cr(ObMysqlClientSession *client_session) +{ + int ret = OB_SUCCESS; + + ObClusterResource *cr = pool_->acquire_cluster_resource(); // inc ref + LOG_DEBUG("new connection", K(cr), KPC(cr)); + if (OB_ISNULL(cr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cluster resource should not be null here", K(ret)); + client_session->destroy(); + } else if (OB_FAIL(client_session->new_connection(client_vc_, request_buf_, request_reader_, cr))) { + LOG_WARN("fail to new_connection", K(ret)); + } + + if (NULL != cr) { + cr->dec_ref(); + } + return ret; +} + int ObMysqlClient::setup_read_handshake() { int ret = OB_SUCCESS; @@ -994,21 +1042,19 @@ int ObMysqlClient::setup_read_handshake() client_vc_->clear_resp_received(); next_action_ = CLIENT_ACTION_READ_HANDSHAKE; - ObSharedRefCount *param = pool_->acquire_connection_param(); // inc ref - LOG_DEBUG("new connection", K(param)); - if (OB_ISNULL(param)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("connection param should not be null here", K(ret)); - client_session->destroy(); - } else if (OB_FAIL(client_session->new_connection(client_vc_, request_buf_, request_reader_, - pool_->is_cluster_param(), param))) { - LOG_WARN("fail to new_connection", K(ret)); + if (pool_->is_cluster_param()) { + if (OB_FAIL(do_new_connection_with_cr(client_session))) { + LOG_WARN("fail to new connection with cr", K(ret)); + } } else { - LOG_DEBUG("create new proxy client_session", K(client_session->get_proxy_sessid()), - K(client_session->get_cs_id())); + if (OB_FAIL(do_new_connection_with_shard_conn(client_session))) { + LOG_WARN("fail to new connection with shard conn", K(ret)); + } } - if (NULL != param) { - param->dec_ref(); + + if (OB_SUCC(ret)) { + LOG_DEBUG("create new proxy client_session", K(client_session->get_proxy_sessid()), + K(client_session->get_cs_id())); } } diff --git a/src/obproxy/proxy/client/ob_client_vc.h b/src/obproxy/proxy/client/ob_client_vc.h index 766f2e8e57645c4da3ea7e7068ce0da3a0f6b393..4e0d981c308f387d7d528f01a618567c6f138d73 100644 --- a/src/obproxy/proxy/client/ob_client_vc.h +++ b/src/obproxy/proxy/client/ob_client_vc.h @@ -177,6 +177,8 @@ private: int notify_transfer_completed(); + int do_new_connection_with_cr(ObMysqlClientSession *client_session); + int do_new_connection_with_shard_conn(ObMysqlClientSession *client_session); int setup_read_handshake(); int setup_read_login_resp(); int setup_read_autocommit_resp(); diff --git a/src/obproxy/proxy/client/ob_mysql_client_pool.cpp b/src/obproxy/proxy/client/ob_mysql_client_pool.cpp index 3d48858d94de2a216fc2628a81192e3ca8234b75..99ba5d7bff84ea414171426a29251af8ff80fc21 100644 --- a/src/obproxy/proxy/client/ob_mysql_client_pool.cpp +++ b/src/obproxy/proxy/client/ob_mysql_client_pool.cpp @@ -211,6 +211,11 @@ void ObMysqlClientPool::free() shard_conn_ = NULL; } + if (OB_LIKELY(NULL != shard_prop_)) { + shard_prop_->dec_ref(); + shard_prop_ = NULL; + } + CLIENT_POOL_INCREMENT_DYN_STAT(FREE_CLUSTER_CLIENT_POOL_COUNT); op_free(this); @@ -274,6 +279,20 @@ void ObMysqlClientPool::set_cluster_resource(ObClusterResource *cr) } } +void ObMysqlClientPool::set_shard_prop(ObShardProp *shard_prop) +{ + if (NULL != shard_prop_) { + LOG_DEBUG("shard_prop will dec ref", K_(shard_prop), KPC_(shard_prop)); + shard_prop_->dec_ref(); + shard_prop_ = NULL; + } + if (NULL != shard_prop) { + shard_prop->inc_ref(); + shard_prop_ = shard_prop; + shard_prop = NULL; + } +} + void ObMysqlClientPool::set_shard_conn(ObShardConnector *shard_conn) { if (NULL != shard_conn_) { @@ -289,19 +308,28 @@ void ObMysqlClientPool::set_shard_conn(ObShardConnector *shard_conn) } } -ObSharedRefCount *ObMysqlClientPool::acquire_connection_param() +ObClusterResource *ObMysqlClientPool::acquire_cluster_resource() { - if (is_cluster_param_) { - if (OB_NOT_NULL(cluster_resource_)) { - cluster_resource_->inc_ref(); - } - return cluster_resource_; - } else { - if (OB_NOT_NULL(shard_conn_)) { - shard_conn_->inc_ref(); - } - return shard_conn_; + if (NULL != cluster_resource_) { + cluster_resource_->inc_ref(); + } + return cluster_resource_; +} + +ObShardConnector *ObMysqlClientPool::acquire_shard_conn() +{ + if (NULL != shard_conn_) { + shard_conn_->inc_ref(); + } + return shard_conn_; +} + +ObShardProp *ObMysqlClientPool::acquire_shard_prop() +{ + if (NULL != shard_prop_) { + shard_prop_->inc_ref(); } + return shard_prop_; } } // end of namespace proxy diff --git a/src/obproxy/proxy/client/ob_mysql_client_pool.h b/src/obproxy/proxy/client/ob_mysql_client_pool.h index 19e611dfba8d042b3548f8a1b6f4548c70eaa021..3e5e9b3366506610bdc7d69d8a7fb7f8b532c890 100644 --- a/src/obproxy/proxy/client/ob_mysql_client_pool.h +++ b/src/obproxy/proxy/client/ob_mysql_client_pool.h @@ -44,7 +44,7 @@ class ObMysqlClientPool : public common::ObSharedRefCount public: ObMysqlClientPool() : is_inited_(false), stop_(false), mc_count_(0), cluster_resource_(NULL), - shard_conn_(NULL), free_mc_list_(), is_cluster_param_(true) {} + shard_conn_(NULL), shard_prop_(NULL), free_mc_list_(), is_cluster_param_(true) {} virtual ~ObMysqlClientPool() {} virtual void free(); @@ -60,8 +60,11 @@ public: void release_mysql_client(ObMysqlClient *mysql_client); void set_cluster_resource(obutils::ObClusterResource *cr); + obutils::ObClusterResource *acquire_cluster_resource(); void set_shard_conn(dbconfig::ObShardConnector *shard_conn); - common::ObSharedRefCount *acquire_connection_param(); + dbconfig::ObShardConnector *acquire_shard_conn(); + void set_shard_prop(dbconfig::ObShardProp *shard_prop); + dbconfig::ObShardProp *acquire_shard_prop(); int64_t to_string(char *buf, const int64_t buf_len) const; static int64_t get_mysql_client_pool_count(const bool is_meta_mysql_client); @@ -76,6 +79,7 @@ private: int64_t mc_count_; // mysql client count obutils::ObClusterResource *cluster_resource_; dbconfig::ObShardConnector *shard_conn_; + dbconfig::ObShardProp *shard_prop_; common::ObAtomicList free_mc_list_; bool is_cluster_param_; diff --git a/src/obproxy/proxy/client/ob_mysql_proxy.cpp b/src/obproxy/proxy/client/ob_mysql_proxy.cpp index 7d8b46b0b763e4bd104288276bb04dbb148ce18d..a3a40e97ea7315b7f98969c8de47b94569dd396c 100644 --- a/src/obproxy/proxy/client/ob_mysql_proxy.cpp +++ b/src/obproxy/proxy/client/ob_mysql_proxy.cpp @@ -483,6 +483,7 @@ int ObMysqlProxy::init(const int64_t timeout_ms, } int ObMysqlProxy::rebuild_client_pool(ObShardConnector *shard_conn, + ObShardProp *shard_prop, const bool is_meta_mysql_client, const ObString &user_name, const ObString &password, @@ -503,6 +504,7 @@ int ObMysqlProxy::rebuild_client_pool(ObShardConnector *shard_conn, LOG_WARN("fail to alloc client pool", K(is_meta_mysql_client), K(user_name), K(database), K(ret)); } else { client_pool_->set_shard_conn(shard_conn); + client_pool_->set_shard_prop(shard_prop); } } diff --git a/src/obproxy/proxy/client/ob_mysql_proxy.h b/src/obproxy/proxy/client/ob_mysql_proxy.h index 19edd2d33fca84c0761c7d4e66bc02bce38bcdd7..811fcab89ab1431d8ace545915e03b5c48ec56d3 100644 --- a/src/obproxy/proxy/client/ob_mysql_proxy.h +++ b/src/obproxy/proxy/client/ob_mysql_proxy.h @@ -66,6 +66,7 @@ public: // otherwise, we will clear current client pool and create a new one int rebuild_client_pool(dbconfig::ObShardConnector *shard_conn, + dbconfig::ObShardProp *shard_prop, const bool is_meta_mysql_client, const common::ObString &user_name, const common::ObString &password, diff --git a/src/obproxy/proxy/client/ob_raw_mysql_client.cpp b/src/obproxy/proxy/client/ob_raw_mysql_client.cpp index dfb70007c752e53a6f5fbcbd990948db40dd17c8..8bdbb5896d45814b1d81a762e5163d2f2e47e9a9 100644 --- a/src/obproxy/proxy/client/ob_raw_mysql_client.cpp +++ b/src/obproxy/proxy/client/ob_raw_mysql_client.cpp @@ -104,12 +104,18 @@ int ObRawMysqlClientActor::sync_raw_execute(const char *sql, const int64_t timeo // Get the password again, it may be changed if (!is_avail() && OB_FAIL(connect(addr_, timeout_ms))) { - if (!is_avail() && info_->change_password() && OB_FAIL(connect(addr_, timeout_ms))) { - LOG_WARN("fail to connect using password1", "addr", addr_, K(ret)); + if (!is_avail() && info_->change_password()) { + if (OB_FAIL(connect(addr_, timeout_ms))) { + LOG_WARN("fail to connect using password1", "addr", addr_, K(ret)); + } } else { LOG_WARN("fail to connect using password", "addr", addr_, K(ret)); } - } else if (OB_FAIL(send_request(sql))) { + } + + if (OB_FAIL(ret)) { + // do nothing + } if (OB_FAIL(send_request(sql))) { LOG_WARN("fail to post request", K(sql), K(ret)); } else { resp = resp_; diff --git a/src/obproxy/proxy/mysql/ob_mysql_client_session.cpp b/src/obproxy/proxy/mysql/ob_mysql_client_session.cpp index 0673c55473ef17e2275f0c3be3887d86731f1fbf..fcea98f760c270c3572abd4aa7942e69f4bb0e93 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_client_session.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_client_session.cpp @@ -21,6 +21,7 @@ #include "prometheus/ob_sql_prometheus.h" #include "dbconfig/ob_proxy_pb_utils.h" #include "proxy/mysql/ob_mysql_global_session_manager.h" +#include "omt/ob_resource_unit_table_processor.h" using namespace oceanbase::common; using namespace oceanbase::common::hash; @@ -30,6 +31,7 @@ using namespace oceanbase::obproxy::net; using namespace oceanbase::obproxy::obutils; using namespace oceanbase::obproxy::prometheus; using namespace oceanbase::obproxy::dbconfig; +using namespace oceanbase::obproxy::omt; namespace oceanbase { @@ -69,12 +71,12 @@ ObMysqlClientSession::ObMysqlClientSession() cluster_resource_(NULL), dummy_entry_(NULL), is_need_update_dummy_entry_(false), dummy_ldc_(), dummy_entry_valid_time_ns_(0), server_state_version_(0), inner_request_param_(NULL), tcp_init_cwnd_set_(false), half_close_(false), - conn_decrease_(false), conn_prometheus_decrease_(false), + conn_decrease_(false), conn_prometheus_decrease_(false), vip_connection_decrease_(false), magic_(MYSQL_CS_MAGIC_DEAD), create_thread_(NULL), is_local_connection_(false), client_vc_(NULL), in_list_stat_(LIST_INIT), current_tid_(-1), cs_id_(0), proxy_sessid_(0), bound_ss_(NULL), cur_ss_(NULL), lii_ss_(NULL), last_bound_ss_(NULL), read_buffer_(NULL), buffer_reader_(NULL), mysql_sm_(NULL), read_state_(MCS_INIT), ka_vio_(NULL), - server_ka_vio_(NULL), trace_stats_(NULL), select_plan_(NULL), ps_cache_(), + server_ka_vio_(NULL), trace_stats_(NULL), select_plan_(NULL), ps_id_(0), cursor_id_(CURSOR_ID_START), text_ps_cache_(), using_ldg_(false) { SET_HANDLER(&ObMysqlClientSession::main_handler); @@ -128,14 +130,18 @@ void ObMysqlClientSession::destroy() // here need place before session_info_.destroy, because use some session_info's data if (conn_prometheus_decrease_) { - SESSION_PROMETHEUS_STAT(session_info_, PROMETHEUS_CURRENT_SESSION, true, -1); + SESSION_PROMETHEUS_STAT(session_info_, PROMETHEUS_CURRENT_SESSION, true, -1); + SESSION_PROMETHEUS_STAT(session_info_, PROMETHEUS_USED_CONNECTIONS, -1); conn_prometheus_decrease_ = false; } + if (vip_connection_decrease_) { + decrease_used_connections(); + vip_connection_decrease_ = false; + } test_server_addr_.reset(); session_info_.destroy(); // destroy ps_cache after destroy session, because client session info hold ps entry pointer - ps_cache_.destroy(); text_ps_cache_.destroy(); if (NULL != trace_stats_) { trace_stats_->destory(); @@ -160,6 +166,25 @@ void ObMysqlClientSession::destroy() op_reclaim_free(this); } +inline void ObMysqlClientSession::decrease_used_connections() +{ + ObString cluster_name; + ObString tenant_name; + ObString ip_name; + + if (is_need_convert_vip_to_tname() && is_vip_lookup_success()) { + cluster_name = get_vip_cluster_name(); + tenant_name = get_vip_tenant_name(); + session_info_.get_vip_addr_name(ip_name); + } else { + session_info_.get_cluster_name(cluster_name); + session_info_.get_tenant_name(tenant_name); + } + + get_global_resource_unit_table_processor().dec_conn( + cluster_name, tenant_name, ip_name); +} + int ObMysqlClientSession::ssn_hook_append(ObMysqlHookID id, ObContInternal *cont) { int ret = OB_SUCCESS; @@ -227,16 +252,23 @@ int ObMysqlClientSession::new_transaction(const bool is_new_conn/* = false*/) int ObMysqlClientSession::new_connection( ObNetVConnection *new_vc, ObMIOBuffer *iobuf, - ObIOBufferReader *reader, bool is_cluster_param, ObSharedRefCount *param) + ObIOBufferReader *reader, + ObShardConnector *shard_conn, + ObShardProp *shard_prop) { - if (NULL != param) { - if (is_cluster_param) { - param->inc_ref(); - cluster_resource_ = dynamic_cast(param); - } else { - session_info_.set_shard_connector(dynamic_cast(param)); - } - param = NULL; + session_info_.set_shard_connector(shard_conn); + session_info_.set_shard_prop(shard_prop); + return new_connection(new_vc, iobuf, reader); +} + +int ObMysqlClientSession::new_connection( + ObNetVConnection *new_vc, ObMIOBuffer *iobuf, + ObIOBufferReader *reader, ObClusterResource *cluster_resource) +{ + if (NULL != cluster_resource) { + cluster_resource->inc_ref(); + cluster_resource_ = cluster_resource; + cluster_resource = NULL; } return new_connection(new_vc, iobuf, reader); } @@ -339,6 +371,7 @@ int ObMysqlClientSession::new_connection( } else { session_info_.set_is_read_only_user(ct_info_.vip_tenant_.is_read_only()); session_info_.set_is_request_follower_user(ct_info_.vip_tenant_.is_request_follower()); + session_info_.set_vip_addr_name(ct_info_.vip_tenant_.vip_addr_.addr_); } } @@ -995,8 +1028,7 @@ void ObMysqlClientSession::handle_transaction_complete(ObIOBufferReader *r, bool if (OB_LIKELY(MCS_ACTIVE_READER == read_state_) && OB_LIKELY(NULL != mysql_sm_)) { PROXY_CS_LOG(DEBUG, "client session handle transaction complete", K_(cs_id), K(mysql_sm_->sm_id_), KPC_(cluster_resource)); - if (get_global_proxy_config().is_metadb_used() - && mysql_sm_->trans_state_.mysql_config_params_->enable_report_session_stats_) { + if (OB_UNLIKELY(get_global_proxy_config().is_metadb_used() && mysql_sm_->trans_state_.mysql_config_params_->enable_report_session_stats_)) { update_session_stats(); } @@ -1011,8 +1043,8 @@ void ObMysqlClientSession::handle_transaction_complete(ObIOBufferReader *r, bool K(*this)); // here only check in non-sharding mode. // in sharding mode, will switch cluster and set to NULL when use db - } else if (!session_info_.is_sharding_user() && session_info_.is_oceanbase_server()) { - if (!is_proxysys_tenant() && (OB_ISNULL(cluster_resource_) || OB_UNLIKELY(cluster_resource_->is_deleting()))) { + } else if (OB_LIKELY(!session_info_.is_sharding_user() && session_info_.is_oceanbase_server())) { + if ((OB_ISNULL(cluster_resource_) || OB_UNLIKELY(cluster_resource_->is_deleting())) && !is_proxysys_tenant()) { if (NULL != cluster_resource_) { PROXY_CS_LOG(INFO, "the cluster resource is deleting, client session will close", KPC_(cluster_resource), K_(cluster_resource), K_(is_proxy_mysql_client)); @@ -1021,12 +1053,13 @@ void ObMysqlClientSession::handle_transaction_complete(ObIOBufferReader *r, bool } } if (OB_LIKELY(!close_cs)) { - MYSQL_DECREMENT_DYN_STAT(CURRENT_CLIENT_TRANSACTIONS); - - if (OB_UNLIKELY(NULL != trace_stats_) && trace_stats_->is_trace_stats_used()) { - PROXY_CS_LOG(DEBUG, "current trace stats", KPC_(trace_stats), K_(cs_id)); - //mark_trace_stats_need_reuse() will not remove history stats, just denote trace_stats need reset next time - trace_stats_->mark_trace_stats_need_reuse(); + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { + MYSQL_DECREMENT_DYN_STAT(CURRENT_CLIENT_TRANSACTIONS); + if (OB_UNLIKELY(NULL != trace_stats_) && trace_stats_->is_trace_stats_used()) { + PROXY_CS_LOG(DEBUG, "current trace stats", KPC_(trace_stats), K_(cs_id)); + //mark_trace_stats_need_reuse() will not remove history stats, just denote trace_stats need reset next time + trace_stats_->mark_trace_stats_need_reuse(); + } } // Clean up the write VIO in case of inactivity timeout @@ -1137,7 +1170,7 @@ int ObMysqlClientSession::acquire_svr_session_no_pool(const sockaddr &addr, ObMy PROXY_CS_LOG(DEBUG, "[acquire server session] try to acquire session in session pool", K_(cs_id)); ObShardConnector *shard_conn = session_info_.get_shard_connector(); // if shard_conn not null, need use shard_conn - if (OB_NOT_NULL(shard_conn)) { + if (OB_UNLIKELY(NULL != shard_conn)) { if (OB_FAIL(session_manager_new_.acquire_server_session(shard_conn->shard_name_.config_string_, addr, session_info_.get_full_username(), svr_session))) { PROXY_CS_LOG(DEBUG, "[acquire server session] fail to acquire server session from " @@ -1199,7 +1232,7 @@ int ObMysqlClientSession::acquire_svr_session(const sockaddr &addr, const bool n // 2. try other session in common pool if (!is_proxy_mysql_client_ && NULL == svr_session) { - if (is_session_pool_client()) { + if (OB_UNLIKELY(is_session_pool_client())) { ret = acquire_svr_session_in_session_pool(addr, svr_session); } else { ret = acquire_svr_session_no_pool(addr, svr_session); @@ -1428,21 +1461,14 @@ bool ObMysqlClientSession::is_need_convert_vip_to_tname() return (get_global_proxy_config().need_convert_vip_to_tname && !this->is_proxy_mysql_client_ ); } -bool ObMysqlClientSession::is_need_use_proxy_tenant_name() -{ - return strlen(get_global_proxy_config().proxy_tenant_name.str()) > 0 - && strlen(get_global_proxy_config().rootservice_cluster_name.str()) > 0 - && !this->is_proxy_mysql_client_; -} - inline bool ObMysqlClientSession::need_close() const { bool bret = false; const ObHotUpgraderInfo &info = get_global_hot_upgrade_info(); - if (OB_LIKELY(!info.need_conn_accept_) + if (OB_UNLIKELY(info.graceful_exit_start_time_ > 0) + && OB_LIKELY(!info.need_conn_accept_) && !is_proxy_mysql_client_ && info.active_client_vc_count_ > 0 - && OB_LIKELY(info.graceful_exit_start_time_ > 0) && OB_LIKELY(info.graceful_exit_end_time_ > info.graceful_exit_start_time_)) { int64_t current_active_count = 0; NET_READ_GLOBAL_DYN_SUM(NET_GLOBAL_CLIENT_CONNECTIONS_CURRENTLY_OPEN, current_active_count); @@ -1521,9 +1547,20 @@ int ObMysqlClientSession::check_update_ldc() { int ret = OB_SUCCESS; common::ModulePageAllocator *allocator = NULL; - if (OB_ISNULL(cluster_resource_)) { + ObClusterResource *cluster_resource = NULL; + bool need_dec_cr = false; + if (get_global_resource_pool_processor().get_default_cluster_resource() == cluster_resource_) { + const ObTableEntryName &name = mysql_sm_->trans_state_.pll_info_.te_name_; + uint64_t cluster_id = get_cluster_id(); + cluster_resource = get_global_resource_pool_processor().acquire_cluster_resource(name.cluster_name_, cluster_id); + need_dec_cr = true; + } else { + cluster_resource = cluster_resource_; + } + + if (OB_ISNULL(cluster_resource)) { ret = OB_INVALID_ARGUMENT; - PROXY_CS_LOG(WARN, "cluster_resource_ is not avail", K(ret)); + PROXY_CS_LOG(WARN, "cluster_resource is not avail", K(ret)); } else if (OB_ISNULL(dummy_entry_) || OB_UNLIKELY(!dummy_entry_->is_tenant_servers_valid())) { ret = OB_INVALID_ARGUMENT; PROXY_CS_LOG(WARN, "dummy_entry_ is not avail", KPC(dummy_entry_), K(ret)); @@ -1533,7 +1570,7 @@ int ObMysqlClientSession::check_update_ldc() } else if (OB_FAIL(ObLDCLocation::get_thread_allocator(allocator))) { PROXY_CS_LOG(WARN, "fail to get_thread_allocator", K(ret)); } else { - bool is_base_servers_added = cluster_resource_->is_base_servers_added(); + bool is_base_servers_added = cluster_resource->is_base_servers_added(); ObString new_idc_name = get_current_idc_name(); //we need update ldc when the follow happened: //1. servers_state_version has changed @@ -1543,28 +1580,28 @@ int ObMysqlClientSession::check_update_ldc() //3. idc_name has changed //or //4. base_servers has not added - if (cluster_resource_->server_state_version_ != server_state_version_ + if (cluster_resource->server_state_version_ != server_state_version_ || dummy_ldc_.is_empty() || 0 != new_idc_name.case_compare(dummy_ldc_.get_idc_name()) || !is_base_servers_added) { PROXY_CS_LOG(DEBUG, "need update dummy_ldc", "old_idc_name", dummy_ldc_.get_idc_name(), K(new_idc_name), - "cluster_name", cluster_resource_->get_cluster_name(), + "cluster_name", cluster_resource->get_cluster_name(), "old_ss_version", server_state_version_, - "new_ss_version", cluster_resource_->server_state_version_, + "new_ss_version", cluster_resource->server_state_version_, "dummy_ldc_is_empty", dummy_ldc_.is_empty(), K(is_base_servers_added)); ObSEArray simple_servers_info( ObServerStateRefreshCont::DEFAULT_SERVER_COUNT, *allocator); bool need_ignore = false; - if (!is_base_servers_added && 0 == cluster_resource_->server_state_version_) { + if (!is_base_servers_added && 0 == cluster_resource->server_state_version_) { PROXY_CS_LOG(INFO, "base servers has not added, treat all tenant server as ok", "tenant_server", *(dummy_entry_->get_tenant_servers()), K(ret)); } else { - const uint64_t new_ss_version = cluster_resource_->server_state_version_; - common::ObIArray &server_state_info = cluster_resource_->get_server_state_info(new_ss_version); - common::DRWLock &server_state_lock = cluster_resource_->get_server_state_lock(new_ss_version); + const uint64_t new_ss_version = cluster_resource->server_state_version_; + common::ObIArray &server_state_info = cluster_resource->get_server_state_info(new_ss_version); + common::DRWLock &server_state_lock = cluster_resource->get_server_state_lock(new_ss_version); int err_no = 0; if (0 != (err_no = server_state_lock.try_rdlock())) { if (dummy_ldc_.is_empty()) { @@ -1580,7 +1617,7 @@ int ObMysqlClientSession::check_update_ldc() "old_ss_version", server_state_version_, "new_ss_version", new_ss_version, "dummy_ldc_is_empty", dummy_ldc_.is_empty(), - K(cluster_resource_->is_base_servers_added()), + K(cluster_resource->is_base_servers_added()), K(is_base_servers_added), K(need_ignore), K(dummy_ldc_)); } else { if (OB_FAIL(simple_servers_info.assign(server_state_info))) { @@ -1593,8 +1630,8 @@ int ObMysqlClientSession::check_update_ldc() } if (OB_SUCC(ret) && !need_ignore) { if (OB_FAIL(dummy_ldc_.assign(dummy_entry_->get_tenant_servers(), simple_servers_info, - new_idc_name, is_base_servers_added, cluster_resource_->get_cluster_name(), - cluster_resource_->get_cluster_id()))) { + new_idc_name, is_base_servers_added, cluster_resource->get_cluster_name(), + cluster_resource->get_cluster_id()))) { if (OB_EMPTY_RESULT == ret) { if (dummy_entry_->is_entry_from_rslist()) { set_need_delete_cluster(); @@ -1611,21 +1648,26 @@ int ObMysqlClientSession::check_update_ldc() } } } - if (cluster_resource_->is_base_servers_added()) { - dummy_ldc_.set_safe_snapshot_manager(&cluster_resource_->safe_snapshot_mgr_); + if (cluster_resource->is_base_servers_added()) { + dummy_ldc_.set_safe_snapshot_manager(&cluster_resource->safe_snapshot_mgr_); } } else { PROXY_CS_LOG(DEBUG, "no need update dummy_ldc", "old_idc_name", dummy_ldc_.get_idc_name(), K(new_idc_name), "old_ss_version", server_state_version_, - "new_ss_version", cluster_resource_->server_state_version_, + "new_ss_version", cluster_resource->server_state_version_, "dummy_ldc_is_empty", dummy_ldc_.is_empty(), K(is_base_servers_added), K(dummy_ldc_)); } allocator = NULL; } + + if (OB_UNLIKELY(need_dec_cr && NULL != cluster_resource)) { + cluster_resource->dec_ref(); + } + return ret; } diff --git a/src/obproxy/proxy/mysql/ob_mysql_client_session.h b/src/obproxy/proxy/mysql/ob_mysql_client_session.h index 526454d13291634c47395ac9499cb0f8e2fe325e..06013b423634ac2bef08f9c079362619c089bccf 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_client_session.h +++ b/src/obproxy/proxy/mysql/ob_mysql_client_session.h @@ -26,6 +26,7 @@ #include "rpc/obmysql/packet/ompk_handshake.h" #include "optimizer/ob_sharding_select_log_plan.h" #include "optimizer/ob_proxy_optimizer_processor.h" +#include "iocore/net/ob_unix_net_vconnection.h" namespace oceanbase { @@ -63,8 +64,12 @@ public: event::ObIOBufferReader *reader); int new_connection(net::ObNetVConnection *new_vc, event::ObMIOBuffer *iobuf, - event::ObIOBufferReader *reader, bool is_cluster_param, - common::ObSharedRefCount *param); + event::ObIOBufferReader *reader, obutils::ObClusterResource *cluster_resource); + + int new_connection(net::ObNetVConnection *new_vc, event::ObMIOBuffer *iobuf, + event::ObIOBufferReader *reader, dbconfig::ObShardConnector *shard_conn, + dbconfig::ObShardProp *shard_prop); + // Implement ObVConnection interface. virtual event::ObVIO *do_io_read(event::ObContinuation *c, const int64_t nbytes = INT64_MAX, @@ -98,14 +103,14 @@ public: int acquire_svr_session_no_pool(const sockaddr &addr, ObMysqlServerSession *&svr_session); int64_t get_svr_session_count() const; - ObMysqlServerSession *get_server_session() const { return bound_ss_; } - ObMysqlServerSession *get_cur_server_session() const { return cur_ss_; } - ObMysqlServerSession *get_lii_server_session() const { return lii_ss_; } - ObMysqlServerSession *get_last_bound_server_session() const { return last_bound_ss_; } - void set_server_session(ObMysqlServerSession *ssession) { bound_ss_ = ssession; } - void set_cur_server_session(ObMysqlServerSession *ssession) { cur_ss_ = ssession; } - void set_lii_server_session(ObMysqlServerSession *ssession) { lii_ss_ = ssession; } - void set_last_bound_server_session(ObMysqlServerSession *ssession) { last_bound_ss_ = ssession; } + inline ObMysqlServerSession *get_server_session() const { return bound_ss_; } + inline ObMysqlServerSession *get_cur_server_session() const { return cur_ss_; } + inline ObMysqlServerSession *get_lii_server_session() const { return lii_ss_; } + inline ObMysqlServerSession *get_last_bound_server_session() const { return last_bound_ss_; } + inline void set_server_session(ObMysqlServerSession *ssession) { bound_ss_ = ssession; } + inline void set_cur_server_session(ObMysqlServerSession *ssession) { cur_ss_ = ssession; } + inline void set_lii_server_session(ObMysqlServerSession *ssession) { lii_ss_ = ssession; } + inline void set_last_bound_server_session(ObMysqlServerSession *ssession) { last_bound_ss_ = ssession; } bool is_hold_conn_id(const uint32_t conn_id); // Functions for manipulating api hooks @@ -127,8 +132,6 @@ public: //proxy inner mysql_client do need convert vip to tenant bool is_need_convert_vip_to_tname(); - // for cloud user, proxy start with proxy_tenant and cluster name - bool is_need_use_proxy_tenant_name(); int64_t get_current_tid() const { return current_tid_; } @@ -225,7 +228,7 @@ public: void set_session_pool_client(bool is_session_pool_client) { session_info_.is_session_pool_client_ = is_session_pool_client; } - bool is_session_pool_client() { return session_info_.is_session_pool_client_; } + inline bool is_session_pool_client() { return session_info_.is_session_pool_client_; } void set_server_addr(proxy::ObCommonAddr addr) {common_addr_ = addr;} void set_first_dml_sql_got() { is_first_dml_sql_got_ = true; } bool is_first_dml_sql_got() const { return is_first_dml_sql_got_; } @@ -252,6 +255,7 @@ public: ObProxyLoginUserType get_user_identity() const { return session_info_.get_user_identity(); } void set_user_identity(const ObProxyLoginUserType identity) { session_info_.set_user_identity(identity); } void set_conn_prometheus_decrease(bool conn_prometheus_decrease) { conn_prometheus_decrease_ = conn_prometheus_decrease; } + void set_vip_connection_decrease(bool vip_connection_decrease) { vip_connection_decrease_ = vip_connection_decrease; } optimizer::ObShardingSelectLogPlan* get_sharding_select_log_plan() const { return select_plan_; } void set_sharding_select_log_plan(optimizer::ObShardingSelectLogPlan *plan) { if (NULL != select_plan_) { @@ -262,7 +266,7 @@ public: } select_plan_ = NULL; } - + select_plan_ = plan; } @@ -270,8 +274,6 @@ public: void set_can_direct_ok(bool val) { can_direct_ok_ = val; } // ps cache - ObPsEntry *get_ps_entry(const common::ObString &sql); - int add_ps_entry(ObPsEntry *entry) { return ps_cache_.set_ps_entry(entry); } ObTextPsEntry *get_text_ps_entry(const common::ObString &sql); int add_text_ps_entry(ObTextPsEntry *entry) { return text_ps_cache_.set_text_ps_entry(entry); } int delete_text_ps_entry(ObTextPsEntry *entry) { return text_ps_cache_.delete_text_ps_entry(entry); } @@ -308,6 +310,7 @@ private: int fetch_tenant_by_vip(); int get_vip_addr(); + inline void decrease_used_connections(); void update_session_stats(); bool need_close() const; @@ -390,6 +393,7 @@ private: bool half_close_; bool conn_decrease_; bool conn_prometheus_decrease_; + bool vip_connection_decrease_; int magic_; event::ObEThread *create_thread_; @@ -431,7 +435,6 @@ private: ObSessionStats session_stats_; ObTraceStats *trace_stats_; optimizer::ObShardingSelectLogPlan *select_plan_; - ObBasePsEntryCache ps_cache_; uint32_t ps_id_; uint32_t cursor_id_; ObBasePsEntryCache text_ps_cache_; @@ -461,18 +464,6 @@ inline uint32_t ObMysqlClientSession::get_next_ps_stmt_id() return ret; } -inline ObPsEntry *ObMysqlClientSession::get_ps_entry(const common::ObString &sql) -{ - int ret = OB_SUCCESS; - ObPsEntry *entry = NULL; - if (OB_FAIL(ps_cache_.get_ps_entry(sql, entry))) { - if (OB_HASH_NOT_EXIST != ret) { - _PROXY_LOG(WARN, "fail to get ps entry with sql, ret=%d", ret); - } - } - return entry; -} - ObTextPsEntry *ObMysqlClientSession::get_text_ps_entry(const common::ObString &sql) { int ret = OB_SUCCESS; diff --git a/src/obproxy/proxy/mysql/ob_mysql_global_session_utils.cpp b/src/obproxy/proxy/mysql/ob_mysql_global_session_utils.cpp index e9affa6dd27aed4bb103a3a0ec6570f624f5c528..1f19c1aeca222edae8474d20e7eda73efbe9e2a7 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_global_session_utils.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_global_session_utils.cpp @@ -624,4 +624,4 @@ GlobalSingleConnectorMap& get_global_single_connector_map() } } // end of namespace proxy } // end of namespace obproxy -} // end of namespace oceanbase +} // end of namespace oceanbase \ No newline at end of file diff --git a/src/obproxy/proxy/mysql/ob_mysql_proxy_server_main.cpp b/src/obproxy/proxy/mysql/ob_mysql_proxy_server_main.cpp index 0af7b0aedd032689993416d33a80962a5604ac2d..43fb615da3a3f696923947b629115650c057c84c 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_proxy_server_main.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_proxy_server_main.cpp @@ -23,7 +23,6 @@ #include "iocore/eventsystem/ob_grpc_task.h" #include "iocore/eventsystem/ob_shard_watch_task.h" #include "obutils/ob_congestion_manager.h" -#include "iocore/net/ob_ssl_processor.h" #include "obutils/ob_proxy_config.h" #include "dbconfig/ob_proxy_db_config_processor.h" @@ -136,8 +135,6 @@ int ObMysqlProxyServerMain::start_mysql_proxy_server(const ObMysqlConfigParams & int ret = OB_SUCCESS; if (OB_FAIL(start_processor_threads(config_params))) { LOG_ERROR("fail to start processor threads", K(ret)); - } else if (OB_FAIL(g_ssl_processor.init())) { - LOG_ERROR("fail to init ssl processor", K(ret)); } else if (get_global_proxy_config().enable_sharding && !get_global_proxy_config().use_local_dbconfig && !get_global_db_config_processor().is_config_inited() @@ -186,7 +183,8 @@ int ObMysqlProxyServerMain::start_processor_threads(const ObMysqlConfigParams &c && OB_FAIL(g_grpc_task_processor.start(grpc_threads, stack_size))) { // if use local config, no need start grpc threads LOG_ERROR("fail to start grpc task processor", K(stack_size), K(ret)); - } else if (OB_FAIL(g_shard_watch_task_processor.start(grpc_watch_threads, stack_size))) { + } else if (get_global_proxy_config().enable_sharding + && OB_FAIL(g_shard_watch_task_processor.start(grpc_watch_threads, stack_size))) { LOG_ERROR("fail to start grpc parent task processor", K(stack_size), K(ret)); } else if (OB_FAIL(init_cs_map_for_thread())) { LOG_ERROR("fail to init cs_map for thread", K(ret)); @@ -200,6 +198,8 @@ int ObMysqlProxyServerMain::start_processor_threads(const ObMysqlConfigParams &c LOG_ERROR("fail to init routine_map for thread", K(ret)); } else if (OB_FAIL(init_sql_table_map_for_thread())) { LOG_ERROR("fail to init sql_table_map for thread", K(ret)); + } else if (OB_FAIL(init_ps_entry_cache_for_thread())) { + LOG_ERROR("fail to init ps entry cache for thread", K(ret)); } else if (OB_FAIL(init_random_seed_for_thread())) { LOG_ERROR("fail to init random seed for thread", K(ret)); } else {} diff --git a/src/obproxy/proxy/mysql/ob_mysql_server_session.h b/src/obproxy/proxy/mysql/ob_mysql_server_session.h index 749d8ddf2f5e7142b28e6e7a14835e4f162f06fe..ad97ccb3ffa28308b25dbacade43df234541d98d 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_server_session.h +++ b/src/obproxy/proxy/mysql/ob_mysql_server_session.h @@ -131,11 +131,11 @@ public: int release(); net::ObNetVConnection *get_netvc() const { return server_vc_; } - void set_client_session(ObMysqlClientSession &client_session) { client_session_ = &client_session; } - void clear_client_session() { client_session_ = NULL; } - ObMysqlClientSession *get_client_session() { return client_session_; } - ObServerSessionInfo &get_session_info() { return session_info_; } - const ObServerSessionInfo &get_session_info() const { return session_info_; } + inline void set_client_session(ObMysqlClientSession &client_session) { client_session_ = &client_session; } + inline void clear_client_session() { client_session_ = NULL; } + inline ObMysqlClientSession *get_client_session() { return client_session_; } + inline ObServerSessionInfo &get_session_info() { return session_info_; } + inline const ObServerSessionInfo &get_session_info() const { return session_info_; } const char *get_state_str() const; // set inactivity to the value timeout(in nanoseconds) diff --git a/src/obproxy/proxy/mysql/ob_mysql_session_manager.cpp b/src/obproxy/proxy/mysql/ob_mysql_session_manager.cpp index f397c124452c4bd71da059da0980d2f4cb94ae76..44f4bb276014a776edf52ea57a949aff1b0c4d49 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_session_manager.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_session_manager.cpp @@ -253,7 +253,7 @@ int ObMysqlSessionManager::acquire_server_session(const sockaddr &addr, const Ob { int ret = OB_SESSION_NOT_FOUND; server_session = NULL; - if (NULL != (server_session = session_pool_.acquire_session(addr, auth_user))) { + if (OB_LIKELY(NULL != (server_session = session_pool_.acquire_session(addr, auth_user)))) { ret = OB_SUCCESS; LOG_DEBUG("[acquire session] pool search successful"); } diff --git a/src/obproxy/proxy/mysql/ob_mysql_sm.cpp b/src/obproxy/proxy/mysql/ob_mysql_sm.cpp index 199b61795c01f71a7b3a9f58f307d8850a77f2ea..d17df9fd245303f42a5e9619f621e9afc2b86c94 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_sm.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_sm.cpp @@ -58,12 +58,17 @@ #include "qos/ob_proxy_qos_stat_processor.h" #include "obutils/ob_proxy_config_processor.h" #include "proxy/mysqllib/ob_mysql_packet_rewriter.h" -#include "iocore/net/ob_ssl_processor.h" #include "packet/ob_mysql_packet_writer.h" #include "dbconfig/ob_proxy_db_config_info.h" #include "optimizer/ob_proxy_optimizer_processor.h" #include "dbconfig/ob_proxy_pb_utils.h" #include "proxy/mysql/ob_mysql_global_session_manager.h" +#include "omt/ob_white_list_table_processor.h" +#include "omt/ob_resource_unit_table_processor.h" +#include "omt/ob_ssl_config_table_processor.h" +#include "iocore/net/ob_inet.h" +#include "prometheus/ob_thread_prometheus.h" +#include "common/obsm_utils.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -77,6 +82,7 @@ using namespace oceanbase::obproxy::qos; using namespace oceanbase::obproxy::dbconfig; using namespace oceanbase::obproxy::engine; using namespace oceanbase::obproxy::optimizer; +using namespace oceanbase::obproxy::omt; namespace oceanbase { @@ -86,8 +92,11 @@ namespace proxy { using namespace obmysql; -#define MYSQL_INCREMENT_TRANS_STAT(X) ObMysqlTransact::update_stat(trans_state_, X, 1); -#define MYSQL_SUM_TRANS_STAT(X, S) ObMysqlTransact::update_stat(trans_state_, X, S); +#define MYSQL_SUM_TRANS_STAT(X, S) { \ + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { \ + ObMysqlTransact::update_stat(trans_state_, X, S); \ + } \ +} // We have a debugging list that can use to find stuck // state machines @@ -146,7 +155,11 @@ void ObMysqlSM::instantiate_func(ObMysqlSM &prototype, ObMysqlSM &new_instance) } } -#define MYSQL_INCREMENT_TRANS_STAT(X) ObMysqlTransact::update_stat(trans_state_, X, 1); +#define MYSQL_INCREMENT_TRANS_STAT(X) { \ + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { \ + ObMysqlTransact::update_stat(trans_state_, X, 1); \ + } \ +} #define __REMEMBER(x) #x #define _REMEMBER(x) __REMEMBER(x) @@ -156,11 +169,13 @@ void ObMysqlSM::instantiate_func(ObMysqlSM &prototype, ObMysqlSM &new_instance) } #define STATE_ENTER(state_name, e) { \ - REMEMBER (e, reentrancy_count_); \ - int64_t stack_start = event::self_ethread().stack_start_; \ - _PROXY_SM_LOG(DEBUG, "sm_id=%u, stack_size=%ld, next_action=%s, event=%s", \ - sm_id_, stack_start - reinterpret_cast(&stack_start), \ - #state_name, ObMysqlDebugNames::get_event_name(e)); } + if (OB_UNLIKELY(!get_global_performance_params().enable_performance_mode_)) { \ + REMEMBER (e, reentrancy_count_); \ + int64_t stack_start = event::self_ethread().stack_start_; \ + _PROXY_SM_LOG(DEBUG, "sm_id=%u, stack_size=%ld, next_action=%s, event=%s", \ + sm_id_, stack_start - reinterpret_cast(&stack_start), \ + #state_name, ObMysqlDebugNames::get_event_name(e)); } \ + } #define MYSQL_SM_SET_DEFAULT_HANDLER(h) { \ REMEMBER(-1,reentrancy_count_); \ @@ -176,7 +191,8 @@ ObMysqlSM::ObMysqlSM() default_handler_(NULL), pending_action_(NULL), reentrancy_count_(0), terminate_sm_(false), kill_this_async_done_(false), handling_ssl_request_(false), need_renew_cluster_resource_(false), is_in_trans_(true), - retry_acquire_server_session_count_(0), start_acquire_server_session_time_(0) + retry_acquire_server_session_count_(0), start_acquire_server_session_time_(0), + skip_plugin_(false) { static bool scatter_inited = false; @@ -327,8 +343,8 @@ inline int ObMysqlSM::kill_this_async_hook(int event, void *data) inline void ObMysqlSM::refresh_cluster_resource() { if (OB_LIKELY(NULL != client_session_)) { - if (sm_cluster_resource_ != client_session_->cluster_resource_ - && OB_LIKELY(NULL != client_session_->cluster_resource_)) { + if (OB_UNLIKELY(sm_cluster_resource_ != client_session_->cluster_resource_ + && NULL != client_session_->cluster_resource_)) { client_session_->cluster_resource_->inc_ref(); if (NULL != sm_cluster_resource_) { sm_cluster_resource_->dec_ref(); @@ -475,26 +491,36 @@ int ObMysqlSM::state_client_request_read(int event, void *data) set_client_net_read_timeout(); if (trans_state_.is_trans_first_request_) { - if (NULL != client_session_) { + if (OB_LIKELY(NULL != client_session_)) { client_session_->is_waiting_trans_first_request_ = false; } trans_state_.refresh_mysql_config(); refresh_cluster_resource(); - if (0 == milestones_.trans_start_) { - milestones_.trans_start_ = get_based_hrtime(); - MYSQL_INCREMENT_DYN_STAT(TOTAL_TRANSACTION_COUNT); - MYSQL_INCREMENT_DYN_STAT(CURRENT_CLIENT_TRANSACTIONS); + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { + if (0 == milestones_.trans_start_) { + MYSQL_INCREMENT_DYN_STAT(TOTAL_TRANSACTION_COUNT); + MYSQL_INCREMENT_DYN_STAT(CURRENT_CLIENT_TRANSACTIONS); + } + } + + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + if (0 == milestones_.trans_start_) { + milestones_.trans_start_ = get_based_hrtime(); + } } } - if (0 == milestones_.client_.client_begin_) { - milestones_.client_.client_begin_ = get_based_hrtime(); + + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + if (0 == milestones_.client_.client_begin_) { + milestones_.client_.client_begin_ = get_based_hrtime(); + } } - if (client_entry_->read_vio_ != reinterpret_cast(data) + if (OB_UNLIKELY(client_entry_->read_vio_ != reinterpret_cast(data) || (NULL != server_entry_) || (NULL != server_session_) - || (client_entry_->eos_)) { + || (client_entry_->eos_))) { ret = OB_INNER_STAT_ERROR; LOG_WARN("invalid internal state", K_(client_entry_->read_vio), K(data), K_(server_entry), K_(server_session), K_(client_entry_->eos)); @@ -558,7 +584,7 @@ int ObMysqlSM::state_client_request_read(int event, void *data) if (OB_SUCC(ret) && !client_session_->is_proxy_mysql_client_) { ObNetVConnection *vc = client_session_->get_netvc(); - if (OB_LIKELY(NULL != vc) && vc->options_.sockopt_flags_ != trans_state_.mysql_config_params_->client_sock_option_flag_out_) { + if (OB_UNLIKELY(NULL != vc && vc->options_.sockopt_flags_ != trans_state_.mysql_config_params_->client_sock_option_flag_out_)) { vc->options_.sockopt_flags_ = static_cast(trans_state_.mysql_config_params_->client_sock_option_flag_out_); if (vc->options_.sockopt_flags_ & ObNetVCOptions::SOCK_OPT_KEEP_ALIVE) { vc->options_.set_keepalive_param(static_cast(trans_state_.mysql_config_params_->client_tcp_keepidle_), @@ -588,7 +614,7 @@ int ObMysqlSM::state_client_request_read(int event, void *data) cmd_time_stats_.client_request_read_time_ += (milestones_.client_.client_read_end_ - milestones_.client_.client_begin_); } - switch (status) { + switch (__builtin_expect(status, ANALYZE_DONE)) { case ANALYZE_OBPARSE_ERROR: //must read all data of the request, otherwise will read the remain request data when recv next request trans_state_.mysql_errcode_ = OB_ERR_PARSE_SQL; @@ -602,18 +628,13 @@ int ObMysqlSM::state_client_request_read(int event, void *data) callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_SEND_RESPONSE); } break; - case ANALYZE_OBUNSUPPORT_ERROR: - //must read all data of the request, otherwise will read the remain request data when recv next request - trans_state_.mysql_errcode_ = OB_ERROR_UNSUPPORT_EXPR_TYPE; - trans_state_.mysql_errmsg_ = ob_strerror(OB_ERROR_UNSUPPORT_EXPR_TYPE); - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("fail to build err resp", K(ret)); - } else if (OB_FAIL(client_buffer_reader_->consume_all())) { - LOG_WARN("fail to consume all", K_(sm_id), K(ret)); - } else { - trans_state_.next_action_ = ObMysqlTransact::SM_ACTION_INTERNAL_NOOP; - callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_SEND_RESPONSE); - } + case ANALYZE_CAN_NOT_PASS_WHITE_LIST_ERROR: + ret = OB_ERR_CAN_NOT_PASS_WHITELIST; + LOG_WARN("error not pass white list", K(ret), K_(sm_id)); + set_client_abort(ObMysqlTransact::ABORTED, event); + + // Disable further I/O on the client + client_entry_->read_vio_->nbytes_ = client_entry_->read_vio_->ndone_; break; case ANALYZE_ERROR: ret = OB_ERR_UNEXPECTED; @@ -677,7 +698,6 @@ int ObMysqlSM::state_client_request_read(int event, void *data) } } break; - case ANALYZE_DONE: { ObServerRoutingMode mode = trans_state_.mysql_config_params_->server_routing_mode_; LOG_DEBUG("done parsing client request", @@ -691,7 +711,7 @@ int ObMysqlSM::state_client_request_read(int event, void *data) // We read the whole mysql request packet and then analyze it, so there is not // data from client after reading a whole mysql request packet. The packet // size is assured. Enable further IO to watch for client aborts - if (handling_ssl_request_) { + if (OB_UNLIKELY(handling_ssl_request_)) { client_entry_->read_vio_->nbytes_ = INT64_MAX; } client_entry_->read_vio_->reenable(); @@ -707,12 +727,15 @@ int ObMysqlSM::state_client_request_read(int event, void *data) } } - if (!handling_ssl_request_) { + if (OB_LIKELY(!handling_ssl_request_)) { bool need_direct_response_for_client = false; + bool need_wait_callback = false; ObClientSessionInfo &session_info = client_session_->get_session_info(); - if (session_info.is_sharding_user()) { - if (OB_FAIL(handle_shard_request(need_direct_response_for_client))) { + if (OB_UNLIKELY(session_info.is_sharding_user())) { + if (OB_FAIL(handle_shard_request(need_direct_response_for_client, need_wait_callback))) { LOG_ERROR("handle shard request failed", K(ret)); + } else if (need_wait_callback) { + // do nothing } else if (need_direct_response_for_client) { if (OB_FAIL(client_buffer_reader_->consume_all())) { LOG_WARN("fail to consume all", K_(sm_id), K(ret)); @@ -734,20 +757,20 @@ int ObMysqlSM::state_client_request_read(int event, void *data) } } } else { - if (get_global_proxy_config().enable_qos + if (OB_UNLIKELY(get_global_proxy_config().enable_qos && !client_session_->is_proxy_mysql_client_ - && OB_FAIL(handle_limit(need_direct_response_for_client))) { + && OB_FAIL(handle_limit(need_direct_response_for_client)))) { LOG_WARN("fail to handle limit", K(ret)); } if (OB_SUCC(ret) && !need_direct_response_for_client && !client_session_->is_proxy_mysql_client_ - && get_global_proxy_config().enable_ldg + && OB_UNLIKELY(get_global_proxy_config().enable_ldg) && OB_FAIL(handle_ldg(need_direct_response_for_client))) { LOG_WARN("fail to handle ldg", K(ret)); } - if (OB_SUCC(ret) && need_direct_response_for_client) { + if (OB_SUCC(ret) && OB_UNLIKELY(need_direct_response_for_client)) { if (OB_FAIL(client_buffer_reader_->consume_all())) { LOG_WARN("fail to consume all", K_(sm_id), K(ret)); } else { @@ -756,8 +779,8 @@ int ObMysqlSM::state_client_request_read(int event, void *data) } } - if (OB_SUCC(ret) && !need_direct_response_for_client) { - if (client_session_->get_session_info().is_oceanbase_server()) { + if (OB_SUCC(ret) && OB_LIKELY(!need_direct_response_for_client)) { + if (OB_LIKELY(client_session_->get_session_info().is_oceanbase_server())) { setup_get_cluster_resource(); } else { setup_set_cached_variables(); @@ -785,6 +808,119 @@ int ObMysqlSM::state_client_request_read(int event, void *data) return event_ret; } +int ObMysqlSM::setup_handle_shard_ddl(ObAction *action) +{ + int ret = OB_SUCCESS; + + LOG_DEBUG("setup handle shard ddl"); + MYSQL_SM_SET_DEFAULT_HANDLER(&ObMysqlSM::state_handle_shard_ddl); + + int64_t total_len = client_buffer_reader_->read_avail(); + if (total_len > trans_state_.trans_info_.client_request_.get_packet_meta().pkt_len_) { + total_len = trans_state_.trans_info_.client_request_.get_packet_meta().pkt_len_; + } + + // consume data in client buffer reader + if (OB_FAIL(client_buffer_reader_->consume(total_len))) { + LOG_WARN("fail to consume all", K_(sm_id), K(ret)); + } else if (OB_NOT_NULL(pending_action_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pending_action must be NULL here", K_(pending_action), K_(sm_id), K(ret)); + } else { + pending_action_ = action; + } + + return ret; +} + +int ObMysqlSM::state_handle_shard_ddl(int event, void *data) +{ + int ret = OB_SUCCESS; + + STATE_ENTER(ObMysqlSM::state_handle_shard_ddl, event); + + pending_action_ = NULL; + + switch (event) { + case ASYNC_PROCESS_DONE_EVENT: + if (OB_ISNULL(data)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("data is NULL", K_(sm_id), K(ret)); + } else if (OB_FAIL(process_shard_ddl_result(reinterpret_cast(data)))) { + LOG_WARN("fail to process executor result, will disconnect", K_(sm_id), K(ret)); + } else { + trans_state_.next_action_ = ObMysqlTransact::SM_ACTION_INTERNAL_NOOP; + callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_SEND_RESPONSE); + } + break; + case VC_EVENT_EOS: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_INACTIVITY_TIMEOUT: + LOG_WARN("handle shard ddl meet error, will disconnect", K_(sm_id), + "event", ObMysqlDebugNames::get_event_name(event), K(ret)); + ret = OB_CONNECT_ERROR; + break; + case VC_EVENT_ERROR: + LOG_WARN("handle shard ddl meet error, will disconnect", K_(sm_id), + "event", ObMysqlDebugNames::get_event_name(event), K(ret)); + ret = OB_ERR_UNEXPECTED; + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("Unexpected event", K_(sm_id), K(event), K(ret)); + break; + } + } + + if (OB_FAIL(ret)) { + // If there is a problem with the internal processing, clear the cache and directly disconnect the link + trans_state_.free_internal_buffer(); + trans_state_.inner_errcode_ = ret; + call_transact_and_set_next_state(ObMysqlTransact::handle_error_jump); + } + + return VC_EVENT_NONE; +} + +int ObMysqlSM::process_shard_ddl_result(ObShardDDLStatus *ddl_status) +{ + int ret = OB_SUCCESS; + + ObProxyMysqlRequest &client_request = trans_state_.trans_info_.client_request_; + uint8_t seq = static_cast(client_request.get_packet_meta().pkt_seq_ + 1); + ObMIOBuffer *buf = NULL; + + if (NULL != trans_state_.internal_buffer_) { + buf = trans_state_.internal_buffer_; + } else { + if (OB_FAIL(trans_state_.alloc_internal_buffer(MYSQL_BUFFER_SIZE))) { + LOG_WARN("fail to allocate internal buffer", K(ret)); + } else { + buf = trans_state_.internal_buffer_; + } + } + + if (OB_SUCC(ret)) { + if (ddl_status->is_success()) { + const ObMySQLCapabilityFlags &capability = client_session_->get_session_info().get_orig_capability_flags(); + if (OB_FAIL(ObMysqlPacketUtil::encode_ok_packet(*buf, seq, 0, capability))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to encode login response ok packet", + K_(sm_id), K(ret)); + } + } else { + int64_t error_code = 0; + if (OB_FAIL(get_int_value(ddl_status->get_error_code(), error_code))) { + LOG_WARN("fail to get int error code", "errcode", ddl_status->get_error_code(), K(ret)); + } else if (OB_FAIL(ObMysqlPacketUtil::encode_err_packet_buf(*buf, seq, static_cast(-error_code), ddl_status->get_error_message()))) { + LOG_WARN("fail to encode err pacekt buf", K(seq), "errmsg", ddl_status->get_error_message(), + "errcode", error_code, K(ret)); + } + } + } + + return ret; +} + int ObMysqlSM::setup_handle_execute_plan() { int ret = OB_SUCCESS; @@ -800,7 +936,13 @@ int ObMysqlSM::setup_handle_execute_plan() total_len = trans_state_.trans_info_.client_request_.get_packet_meta().pkt_len_; } - ObHRTime execute_timeout = client_session_->get_session_info().get_query_timeout(); + ObShardProp *shard_prop = client_session_->get_session_info().get_shard_prop(); + ObHRTime execute_timeout = 0; + if (OB_NOT_NULL(shard_prop)) { + execute_timeout = HRTIME_MSECONDS(shard_prop->get_socket_timeout()); + } else { + execute_timeout = client_session_->get_session_info().get_query_timeout(); + } // consume data in client buffer reader if (OB_FAIL(client_buffer_reader_->consume(total_len))) { @@ -815,7 +957,7 @@ int ObMysqlSM::setup_handle_execute_plan() ret = OB_ERR_UNEXPECTED; LOG_WARN("pending_action must be NULL here", K_(pending_action), K_(sm_id), K(ret)); } else if (OB_FAIL(operator_root->open(this, pending_action_, hrtime_to_msec(execute_timeout)))) { - LOG_WARN("fail to open operator", K_(sm_id), K(ret)); + LOG_WARN("fail to open operator", K_(sm_id), K(execute_timeout), K(ret)); } else if (OB_FAIL(operator_root->get_next_row())) { LOG_WARN("fail to get next row", K_(sm_id), K(ret)); } else if (OB_ISNULL(pending_action_)) { @@ -923,10 +1065,11 @@ int ObMysqlSM::build_executor_resp(ObMIOBuffer *write_buf, uint8_t &seq, ObProxy { int ret = OB_SUCCESS; int64_t column_count = result_resp->get_column_count(); + ObSEArray *fields = NULL; + ObSEArray ob_fields; // header , cols , first eof if (OB_SUCC(ret)) { - ObSEArray *fields = NULL; if (OB_FAIL(result_resp->get_fields(fields))) { LOG_WARN("faild to push field", K(fields), K(ret)); } else if (OB_ISNULL(fields)) { @@ -937,6 +1080,16 @@ int ObMysqlSM::build_executor_resp(ObMIOBuffer *write_buf, uint8_t &seq, ObProxy LOG_WARN("fields count should equal column_count", K(column_count), "fileds count", fields->count(), K(ret)); } else if (OB_FAIL(ObMysqlPacketUtil::encode_header(*write_buf, seq, *fields))) { LOG_WARN("faild to encode header", K(fields), K(seq), K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < column_count; i++) { + ObMySQLField &mysql_field = fields->at(i); + ObField ob_field; + if (OB_FAIL(ObSMUtils::to_ob_field(mysql_field, ob_field))) { + LOG_WARN("faild to ob field", K(mysql_field), K(ret)); + } else if (OB_FAIL(ob_fields.push_back(ob_field))) { + LOG_WARN("faild to push ob field", K(ob_field), K(ret)); + } + } } } @@ -946,6 +1099,7 @@ int ObMysqlSM::build_executor_resp(ObMIOBuffer *write_buf, uint8_t &seq, ObProxy int64_t buf_len = sizeof(ObObj) * column_count; if (OB_ISNULL(objs = static_cast(op_fixed_mem_alloc(buf_len)))) { + ret = OB_ERR_UNEXPECTED; LOG_WARN("faild to alloc obj array", K(column_count), K(ret)); } else { ObNewRow row; @@ -958,14 +1112,16 @@ int ObMysqlSM::build_executor_resp(ObMIOBuffer *write_buf, uint8_t &seq, ObProxy } else if (OB_UNLIKELY(row_array->count() != column_count)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("row_array count should equal column_count", K(column_count), "row_array count", row_array->count(), K(ret)); - } else { - for (int64_t i = 0; i < column_count; i++) { - objs[i] = *(row_array->at(i)); - } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < column_count; i++) { + objs[i] = *(row_array->at(i)); + } + if (OB_SUCC(ret)) { row.cells_ = objs; row.count_ = column_count; - if (OB_FAIL(ObMysqlPacketUtil::encode_row_packet(*write_buf, seq, row))) { + if (OB_FAIL(ObMysqlPacketUtil::encode_row_packet(*write_buf, seq, row, &ob_fields))) { LOG_WARN("faild to encode row", K(seq), K(row), K(ret)); } else { row.reset(); @@ -995,11 +1151,12 @@ int ObMysqlSM::build_executor_resp(ObMIOBuffer *write_buf, uint8_t &seq, ObProxy return ret; } -int ObMysqlSM::handle_shard_request(bool &need_response_for_stmt) +int ObMysqlSM::handle_shard_request(bool &need_response_for_stmt, bool &need_wait_callback) { int ret = OB_SUCCESS; need_response_for_stmt = false; + need_wait_callback = false; ObClientSessionInfo &session_info = client_session_->get_session_info(); ObProxyMysqlRequest &client_request = trans_state_.trans_info_.client_request_; ObSqlParseResult &parse_result = client_request.get_parse_result(); @@ -1017,7 +1174,7 @@ int ObMysqlSM::handle_shard_request(bool &need_response_for_stmt) } else { if (obmysql::OB_MYSQL_COM_FIELD_LIST == req_cmd) { ret = OB_NOT_SUPPORTED; - LOG_WARN("ddl stmt is unsupported for sharding table", K(ret)); + LOG_WARN("com_field_list cmd is unsupported for sharding table", K(ret)); } else if (obmysql::OB_MYSQL_COM_QUERY != req_cmd) { //do nothing } else if (parse_result.is_invalid_stmt() && parse_result.has_shard_comment()) { @@ -1040,12 +1197,12 @@ int ObMysqlSM::handle_shard_request(bool &need_response_for_stmt) //do nothing } else if (db_info->is_single_shard_db_table()) { // sindle db table - if (OB_FAIL(ObProxyShardUtils::handle_single_shard_request(*client_session_, - trans_state_, *client_buffer_reader_, *db_info))) { + if (OB_FAIL(ObProxyShardUtils::handle_single_shard_request(this, *client_session_, + trans_state_, *client_buffer_reader_, *db_info, need_wait_callback))) { LOG_WARN("fail to handle single shard request", K(ret)); } - } else if (OB_FAIL(ObProxyShardUtils::handle_shard_request(*client_session_, - trans_state_, *client_buffer_reader_, *db_info))) { + } else if (OB_FAIL(ObProxyShardUtils::handle_shard_request(this, *client_session_, + trans_state_, *client_buffer_reader_, *db_info, need_wait_callback))) { LOG_WARN("fail to handle shard request", K(ret)); } } @@ -1054,7 +1211,7 @@ int ObMysqlSM::handle_shard_request(bool &need_response_for_stmt) int tmp_ret = ObProxyShardUtils::build_error_packet(ret, need_response_for_stmt, trans_state_); ret = OB_SUCC(tmp_ret) ? tmp_ret : ret; } - if (OB_SUCC(ret) && !need_response_for_stmt) { + if (OB_SUCC(ret) && !need_response_for_stmt && !need_wait_callback) { if (OB_FAIL(save_user_login_info(session_info, hsr))) { LOG_WARN("fail to save user login info", K_(sm_id), K(ret)); } @@ -1252,9 +1409,9 @@ void ObMysqlSM::setup_get_cluster_resource() int ret = OB_SUCCESS; ObAction *cr_handler = NULL; - if (OB_MYSQL_COM_LOGIN == trans_state_.trans_info_.sql_cmd_ + if (OB_UNLIKELY(OB_MYSQL_COM_LOGIN == trans_state_.trans_info_.sql_cmd_ || need_renew_cluster_resource_ - || (client_session_->get_session_info().is_sharding_user() && NULL == client_session_->cluster_resource_)) { + || (client_session_->get_session_info().is_sharding_user() && NULL == client_session_->cluster_resource_))) { if (trans_state_.mysql_config_params_->is_mysql_routing_mode()) { // here must be in mysql routing mode trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_LOGIN; @@ -1310,7 +1467,7 @@ void ObMysqlSM::setup_get_cluster_resource() LOG_WARN("cluster does not exist, this connection will disconnect", K_(sm_id), K(is_clustername_from_default), K(cluster_name), K(ret)); // return err packet to client "Access denied for ..." - trans_state_.mysql_errcode_ = OB_PASSWORD_WRONG; + trans_state_.mysql_errcode_ = OB_CLUSTER_NOT_EXIST; int tmp_ret = OB_SUCCESS; if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = ObMysqlTransact::build_error_packet(trans_state_)))) { LOG_WARN("fail to build err packet", K(tmp_ret)); @@ -1370,8 +1527,8 @@ void ObMysqlSM::setup_get_cluster_resource() } } else { // mainly for performance testing - if (OB_MYSQL_COM_HANDSHAKE == trans_state_.trans_info_.sql_cmd_ - && trans_state_.mysql_config_params_->is_mysql_routing_mode()) { + if (OB_UNLIKELY(OB_MYSQL_COM_HANDSHAKE == trans_state_.trans_info_.sql_cmd_ + && trans_state_.mysql_config_params_->is_mysql_routing_mode())) { ObResourcePoolProcessor &rp_processor = get_global_resource_pool_processor(); ObString cluster_name; @@ -1514,7 +1671,7 @@ inline int ObMysqlSM::init_request_content(ObRequestAnalyzeCtx &ctx) { int ret = OB_SUCCESS; - if (trans_state_.is_auth_request_) { + if (OB_UNLIKELY(trans_state_.is_auth_request_)) { ObMysqlAuthRequest &orig_auth_req = client_session_->get_session_info().get_login_req(); orig_auth_req.reset(); if (client_session_->is_need_convert_vip_to_tname() && client_session_->is_vip_lookup_success()) { @@ -1573,50 +1730,74 @@ inline bool ObMysqlSM::check_connection_throttle() return throttle; } -inline int ObMysqlSM::encode_throttle_message() +bool ObMysqlSM::can_pass_white_list() { - int ret = OB_SUCCESS; - - if (OB_FAIL(client_buffer_reader_->consume_all())) { - LOG_WARN("client buffer reader fail to consume all", K(ret)); + bool can_pass = false; + bool need_check_white_list = true; + ObStringKV string_kv; + ObClientSessionInfo &session_info = client_session_->get_session_info(); + ObHSRResult &hsr = session_info.get_login_req().get_hsr_result(); + ObUnixNetVConnection* unix_vc = static_cast(client_session_->get_netvc()); + if (OB_UNLIKELY(NULL == unix_vc)) { + LOG_WARN("invalid unix_vc"); } else { - trans_state_.mysql_errcode_ = OB_ERR_TOO_MANY_SESSIONS; - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::encode_throttle_message] fail to encode error response", - K_(sm_id), K(ret), "errcode", trans_state_.mysql_errcode_); + for (int64_t i = 0; need_check_white_list && i < hsr.response_.get_connect_attrs().count(); ++i) { + string_kv = hsr.response_.get_connect_attrs().at(i); + if (0 == string_kv.key_.case_compare(OB_MYSQL_CLIENT_IP)) { + if (!string_kv.value_.empty()) { + char client_ip_buf[256]; + snprintf(client_ip_buf, ObHandshakeResponseParam::OB_MAX_IP_BUF_LEN, "%.*s", string_kv.value_.length(), string_kv.value_.ptr()); + need_check_white_list = false; + if (get_global_white_list_table_processor().can_ip_pass(hsr.cluster_name_, hsr.tenant_name_, hsr.user_name_, client_ip_buf)) { + can_pass = true; + } + } + break; + } + } + if (need_check_white_list) { + if (get_global_white_list_table_processor().can_ip_pass(hsr.cluster_name_, hsr.tenant_name_, hsr.user_name_, unix_vc->get_remote_ip(), true)) { + can_pass = true; + } } } - return ret; + return can_pass; } -inline int ObMysqlSM::encode_unsupport_ps_message() +// chect weahter connections reach throttle +inline bool ObMysqlSM::check_vt_connection_throttle() { - int ret = OB_SUCCESS; - - if (OB_FAIL(client_buffer_reader_->consume_all())) { - LOG_WARN("client buffer reader fail to consume all", K(ret)); + ObString cluster_name; + ObString tenant_name; + ObString ip_name; + ObClientSessionInfo &cs_info = client_session_->get_session_info(); + if (client_session_->is_need_convert_vip_to_tname() && + client_session_->is_vip_lookup_success()) { + // Process public cloud VIP information + cluster_name = client_session_->get_vip_cluster_name(); + tenant_name = client_session_->get_vip_tenant_name(); + cs_info.get_vip_addr_name(ip_name); } else { - trans_state_.mysql_errcode_ = OB_UNSUPPORTED_PS; - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::encode_unsupport_ps_message] fail to encode error response", - K_(sm_id), K(ret), "errcode", trans_state_.mysql_errcode_); - } + // Private cloud scenarios without VIP information: there is no concept of VIP, only clusters and tenants + cs_info.get_cluster_name(cluster_name); + cs_info.get_tenant_name(tenant_name); } - return ret; + return get_global_resource_unit_table_processor().check_and_inc_conn( + cluster_name, tenant_name, ip_name); } -inline int ObMysqlSM::encode_unsupport_change_user_message() +int ObMysqlSM::encode_error_message(int err_code) { int ret = OB_SUCCESS; if (OB_FAIL(client_buffer_reader_->consume_all())) { LOG_WARN("client buffer reader fail to consume all", K(ret)); } else { - trans_state_.mysql_errcode_ = OB_NOT_SUPPORTED; + trans_state_.mysql_errcode_ = err_code; if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::encode_unsupport_change_user_message] fail to encode error response", + LOG_WARN("[ObMysqlSM::encode_error_message] fail to encode error response", K_(sm_id), K(ret), "errcode", trans_state_.mysql_errcode_); } } @@ -1658,7 +1839,7 @@ inline int ObMysqlSM::check_user_identity(const ObString &user_name, const ObHotUpgraderInfo &hu_info = get_global_hot_upgrade_info(); ObHSRResult &hsr = client_session_->get_session_info().get_login_req().get_hsr_result(); const bool is_current_cloud_user = is_cloud_user(); - if (need_reject_user_login(user_name, tenant_name, hsr.has_full_username_, is_current_cloud_user)) { + if (need_reject_user_login(user_name, tenant_name, hsr.has_tenant_username_, hsr.has_cluster_username_, is_current_cloud_user)) { ret = OB_USER_NOT_EXIST; LOG_WARN("access denied for this user", K(hsr), K(is_current_cloud_user), K(ret)); } else if (tenant_name == OB_PROXYSYS_TENANT_NAME) { @@ -1751,7 +1932,10 @@ inline int ObMysqlSM::save_user_login_info(ObClientSessionInfo &session_info, Ob void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) { int ret = OB_SUCCESS; - milestones_.client_.analyze_request_begin_ = get_based_hrtime(); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.client_.analyze_request_begin_ = get_based_hrtime(); + } + if (0 == cmd_size_stats_.client_request_bytes_ && 0 == milestones_.client_.analyze_request_end_) { cmd_time_stats_.client_request_read_time_ += (milestones_.client_.analyze_request_begin_ - milestones_.client_.client_begin_); } else if (cmd_size_stats_.client_request_bytes_ > 0 && milestones_.client_.analyze_request_end_ > 0) { @@ -1776,101 +1960,34 @@ void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) ObMysqlRequestAnalyzer::analyze_request(ctx, orig_auth_req, client_request, req_cmd, status, session_info.is_oracle_mode()); - if (ANALYZE_DONE == status) { + if (OB_LIKELY(ANALYZE_DONE == status)) { // 3. if BEGIN or START TRANSACTION, try to hold it - if (OB_MYSQL_COM_QUERY == req_cmd) { + if (OB_LIKELY(OB_MYSQL_COM_QUERY == req_cmd)) { MYSQL_INCREMENT_DYN_STAT(TOTAL_QUERY_COUNT); - if (client_request.get_parse_result().need_hold_start_trans()) { + if (OB_UNLIKELY(client_request.get_parse_result().need_hold_start_trans())) { if (OB_FAIL(session_info.set_start_trans_sql(client_request.get_sql()))) { LOG_WARN("fail to set start transaction sqld", K_(sm_id), K(ret)); } } if (OB_SUCC(ret)) { - if (client_request.get_parse_result().is_text_ps_prepare_stmt()) { + if (OB_UNLIKELY(client_request.get_parse_result().is_text_ps_prepare_stmt())) { if (OB_FAIL(analyze_text_ps_prepare_request())) { LOG_WARN("analyze text ps prepare request failed", K(ret)); } - } else if (client_request.get_parse_result().is_text_ps_execute_stmt()) { + } else if (OB_UNLIKELY(client_request.get_parse_result().is_text_ps_execute_stmt())) { if (OB_FAIL(analyze_text_ps_execute_request())) { LOG_WARN("analyze text ps execute request failed", K(ret)); } } } // 4. if OB_MYSQL_COM_LOGIN, do some check - } else if (OB_MYSQL_COM_LOGIN == req_cmd) { - ObHSRResult &hsr = orig_auth_req.get_hsr_result(); - ObUnixNetVConnection* unix_vc = static_cast(client_session_->get_netvc()); - if (NULL == unix_vc) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("client entry vc is null", K(ret)); - } else if (hsr.response_.is_ssl_request() && !unix_vc->ssl_connected()) { - if (OB_FAIL(unix_vc->ssl_init(ObUnixNetVConnection::SSL_SERVER))) { - LOG_WARN("ssl start handshake failed", K(ret)); - } else { - ctx.reader_->consume_all(); - handling_ssl_request_ = true; - } - } else { - //save orig capability - ObMySQLCapabilityFlags capability(session_info.get_orig_capability_flags().capability_ & hsr.response_.get_capability_flags().capability_); - session_info.save_orig_capability_flags(capability); - - //login succ, we need desc cache miss stat as we has inc it when fetch_tenant_by_vip - if (client_session_->is_need_convert_vip_to_tname() - && !client_session_->is_vip_lookup_success()) { - client_session_->get_session_stats().stats_[VIP_TO_TENANT_CACHE_MISS] -= 1; - MYSQL_DECREMENT_DYN_STAT(VIP_TO_TENANT_CACHE_MISS); - } - - // check whether the session uses logic tenant - // invoke this func before save_user_login_info, so can rewrite real user into auth packet - // all inner connections are not sharding connection - if (!client_session_->is_proxy_mysql_client_ && OB_FAIL(ObProxyShardUtils::handle_shard_auth(*client_session_, hsr))) { - LOG_WARN("fail to check user sharding", K_(sm_id), K(ret)); - // add some message for login error - bool need_response_for_stmt = false; - ObProxyShardUtils::build_error_packet(ret, need_response_for_stmt, trans_state_); - // if check user failed, disconnect - trans_state_.current_.state_ = ObMysqlTransact::INTERNAL_ERROR; - //save user name, tenant name, cluster name - } else if (OB_FAIL(save_user_login_info(session_info, hsr))) { - LOG_WARN("fail to save user login info", K_(sm_id), K(ret)); - - //check user identity - } else if (client_session_->get_session_info().is_oceanbase_server() - && OB_FAIL(check_user_identity(hsr.user_name_, hsr.tenant_name_, hsr.cluster_name_))) { - LOG_WARN("fail to check user identity", K_(sm_id), K(ret)); - - } else { - if (client_session_->is_proxysys_tenant()) { - //proxysys user no need check everything - } else { - if (!client_session_->is_proxy_mysql_client_) { - SESSION_PROMETHEUS_STAT(client_session_->get_session_info(), PROMETHEUS_CURRENT_SESSION, true, 1); - client_session_->set_conn_prometheus_decrease(true); - } - if (check_connection_throttle()) { - // check client connection throttle count - if (OB_FAIL(encode_throttle_message())) { - LOG_WARN("fail to encode throttle message", K_(sm_id), K(ret)); - } - status = ANALYZE_ERROR; // disconnect - - } else { - // do nothing - } - - // the data which remains in client_buffer_reader_ is orig auth request, - // and will not send to observer; - // later we will rewrite_first_login_req, send first_auth_req to observer - // and consume the client_buffer_reader_; - - }//end if !proxysys - } + } else if (OB_UNLIKELY(OB_MYSQL_COM_LOGIN == req_cmd)) { + if (OB_FAIL(analyze_login_request(ctx, status))) { + LOG_WARN("fail to analyze login request", K(ret)); } } else if (OB_MYSQL_COM_CHANGE_USER == req_cmd) { if (session_info.is_sharding_user() || session_info.is_session_pool_client_) { - if (OB_FAIL(encode_unsupport_change_user_message())) { + if (OB_FAIL(encode_error_message(OB_NOT_SUPPORTED))) { LOG_WARN("fail to encode unsupport change user error message", K(ret)); } else { LOG_INFO("not support change user"); @@ -1879,7 +1996,7 @@ void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) } } else if (OB_MYSQL_COM_STMT_PREPARE == req_cmd) { if (client_request.get_parse_result().is_start_trans_stmt() || session_info.is_sharding_user()) { - if (OB_FAIL(encode_unsupport_ps_message())) { + if (OB_FAIL(encode_error_message(OB_UNSUPPORTED_PS))) { LOG_WARN("fail to encode unsupport ps error message", K(ret)); } else { LOG_INFO("begin statement is not supported in prepare stament", K(ret)); @@ -1890,12 +2007,13 @@ void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) LOG_WARN("fail to analyze ps prepare request", K(ret)); } } else if (OB_MYSQL_COM_STMT_EXECUTE == req_cmd - || OB_MYSQL_COM_STMT_SEND_PIECE_DATA == req_cmd) { + || OB_MYSQL_COM_STMT_SEND_PIECE_DATA == req_cmd + || OB_MYSQL_COM_STMT_SEND_LONG_DATA == req_cmd) { if (OB_FAIL(analyze_ps_execute_request())) { - LOG_WARN("fail to analyze ps execute request", K(ret), K(req_cmd)); + LOG_WARN("fail to analyze ps request", K(ret), K(req_cmd)); } } else if (OB_MYSQL_COM_STMT_FETCH == req_cmd || OB_MYSQL_COM_STMT_GET_PIECE_DATA == req_cmd) { - // Every time you execute COM_STMT_GET_PIECE_DATA, you also need to set the cursor id, + // Every time you execute OB_MYSQL_COM_STMT_GET_PIECE_DATA, you also need to set the cursor id, // because it is the same, so the analyze_fetch_request function is reused here if (OB_FAIL(analyze_fetch_request())) { LOG_WARN("fail to analyze fetch request", K(ret)); @@ -1913,9 +2031,11 @@ void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) } } else if (ANALYZE_CONT == status) { // large request means we have received enough packet(> request_buffer_len_) - if ((OB_MYSQL_COM_STMT_EXECUTE == req_cmd || OB_MYSQL_COM_STMT_SEND_PIECE_DATA == req_cmd) + if ((OB_MYSQL_COM_STMT_EXECUTE == req_cmd + || OB_MYSQL_COM_STMT_SEND_PIECE_DATA == req_cmd + || OB_MYSQL_COM_STMT_SEND_LONG_DATA == req_cmd) && client_request.is_large_request()) { - if (OB_FAIL(analyze_ps_execute_request())) { + if (OB_FAIL(analyze_ps_execute_request(client_request.is_large_request()))) { LOG_WARN("fail to analyze ps execute request", K(ret)); } } else if (OB_MYSQL_COM_STMT_PREPARE_EXECUTE == req_cmd && client_request.is_large_request()) { @@ -1931,12 +2051,118 @@ void ObMysqlSM::analyze_mysql_request(ObMysqlAnalyzeStatus &status) if (OB_FAIL(ret)) { status = ANALYZE_ERROR; } - milestones_.client_.analyze_request_end_ = get_based_hrtime(); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.client_.analyze_request_end_ = get_based_hrtime(); + } cmd_time_stats_.client_request_analyze_time_ += milestone_diff(milestones_.client_.analyze_request_begin_, milestones_.client_.analyze_request_end_); } +int ObMysqlSM::analyze_login_request(ObRequestAnalyzeCtx &ctx, ObMysqlAnalyzeStatus &status) +{ + int ret = OB_SUCCESS; + + ObClientSessionInfo &session_info = client_session_->get_session_info(); + ObMysqlAuthRequest &orig_auth_req = session_info.get_login_req(); + ObHSRResult &hsr = orig_auth_req.get_hsr_result(); + ObUnixNetVConnection* unix_vc = static_cast(client_session_->get_netvc()); + if (NULL == unix_vc) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("client entry vc is null", K(ret)); + } else if (hsr.response_.is_ssl_request() && !unix_vc->ssl_connected()) { + if (OB_FAIL(unix_vc->ssl_init(ObUnixNetVConnection::SSL_SERVER, + client_session_->get_vip_cluster_name(), + client_session_->get_vip_tenant_name()))) { + LOG_WARN("ssl start handshake failed", K(ret)); + } else { + ctx.reader_->consume_all(); + handling_ssl_request_ = true; + } + } else { + //save orig capability + ObMySQLCapabilityFlags capability(session_info.get_orig_capability_flags().capability_ & hsr.response_.get_capability_flags().capability_); + session_info.save_orig_capability_flags(capability); + + //login succ, we need desc cache miss stat as we has inc it when fetch_tenant_by_vip + if (client_session_->is_need_convert_vip_to_tname() + && !client_session_->is_vip_lookup_success()) { + client_session_->get_session_stats().stats_[VIP_TO_TENANT_CACHE_MISS] -= 1; + MYSQL_DECREMENT_DYN_STAT(VIP_TO_TENANT_CACHE_MISS); + } + + // check whether the session uses logic tenant + // invoke this func before save_user_login_info, so can rewrite real user into auth packet + // all inner connections are not sharding connection + if (!client_session_->is_proxy_mysql_client_ && OB_FAIL(ObProxyShardUtils::handle_shard_auth(*client_session_, hsr))) { + LOG_WARN("fail to check user sharding", K_(sm_id), K(ret)); + // add some message for login error + bool need_response_for_stmt = false; + ObProxyShardUtils::build_error_packet(ret, need_response_for_stmt, trans_state_); + // As long as the check user fails, the link will be broken + trans_state_.current_.state_ = ObMysqlTransact::INTERNAL_ERROR; + //save user name, tenant name, cluster name + } else if (OB_FAIL(save_user_login_info(session_info, hsr))) { + LOG_WARN("fail to save user login info", K_(sm_id), K(ret)); + //check user identity + } else if (client_session_->get_session_info().is_oceanbase_server() + && !session_info.is_sharding_user() + && OB_FAIL(check_user_identity(hsr.user_name_, hsr.tenant_name_, hsr.cluster_name_))) { + LOG_WARN("fail to check user identity", K_(sm_id), K(ret)); + if (OB_SUCCESS != encode_error_message(OB_PASSWORD_WRONG)) { + LOG_WARN("fail to encode throttle message", K_(sm_id), K(ret)); + } + trans_state_.current_.state_ = ObMysqlTransact::INTERNAL_ERROR; + } else { + if (client_session_->is_proxysys_tenant()) { + //proxysys user no need check everything + } else { + if (!client_session_->is_proxy_mysql_client_) { + SESSION_PROMETHEUS_STAT(client_session_->get_session_info(), PROMETHEUS_CURRENT_SESSION, true, 1); + SESSION_PROMETHEUS_STAT(client_session_->get_session_info(), PROMETHEUS_USED_CONNECTIONS, 1); + client_session_->set_conn_prometheus_decrease(true); + if (!can_pass_white_list()) { + status = ANALYZE_CAN_NOT_PASS_WHITE_LIST_ERROR; + } else if (check_connection_throttle()) { + // check client connection throttle count + if (OB_FAIL(encode_error_message(OB_ERR_TOO_MANY_SESSIONS))) { + LOG_WARN("fail to encode throttle message", K_(sm_id), K(ret)); + } + status = ANALYZE_ERROR; // disconnect + } else if (check_vt_connection_throttle()) { + // check cloud vip connection throttle count + if (OB_FAIL(encode_error_message(OB_ERR_CON_COUNT_ERROR))) { + LOG_WARN("fail to encode vip throttle message", K_(sm_id), K(ret)); + } + status = ANALYZE_ERROR; // disconnect + } else { + client_session_->set_vip_connection_decrease(true); + } + } + + + // the data which remains in client_buffer_reader_ is orig auth request, + // and will not send to observer; + // later we will rewrite_first_login_req, send first_auth_req to observer + // and consume the client_buffer_reader_; + + }//end if !proxysys + } + + if (!client_session_->is_proxy_mysql_client_) { + net::ObIpEndpoint client_addr; + if (client_session_->is_need_convert_vip_to_tname()) { + net::ops_ip_copy(client_addr, client_session_->get_netvc()->get_real_client_addr()); + } else { + net::ops_ip_copy(client_addr, client_session_->get_netvc()->get_remote_addr()); + } + LOG_INFO("client login audit", K(client_addr), K(hsr.cluster_name_), K(hsr.tenant_name_), K(hsr.user_name_), + "status", ret == OB_SUCCESS && status != ANALYZE_ERROR ? "success" : "failed"); + } + } + return ret; +} + int ObMysqlSM::do_analyze_ps_prepare_request(const ObString &ps_sql) { int ret = OB_SUCCESS; @@ -1947,14 +2173,17 @@ int ObMysqlSM::do_analyze_ps_prepare_request(const ObString &ps_sql) ObClientSessionInfo &session_info = client_session_->get_session_info(); ObProxyMysqlRequest &client_request = trans_state_.trans_info_.client_request_; - ObPsEntry *ps_entry = client_session_->get_ps_entry(ps_sql); + ObPsEntry *ps_entry = NULL; ObPsIdEntry *ps_id_entry = NULL; bool is_new_ps_entry = false; + ObBasePsEntryCache &ps_entry_cache = self_ethread().get_ps_entry_cache(); + ps_entry_cache.get_ps_entry(ps_sql, ps_entry); + if (NULL == ps_entry) { if (OB_FAIL(ObPsEntry::alloc_and_init_ps_entry(ps_sql, client_request.get_parse_result(), ps_entry))) { LOG_WARN("fail to alloc and init ps entry", K(ret)); - } else if (OB_FAIL(client_session_->add_ps_entry(ps_entry))) { + } else if (OB_FAIL(ps_entry_cache.set_ps_entry(ps_entry))) { LOG_WARN("fail to add ps entry to cache", K(ret)); if (OB_LIKELY(NULL != ps_entry)) { ps_entry->destroy(); @@ -1980,6 +2209,7 @@ int ObMysqlSM::do_analyze_ps_prepare_request(const ObString &ps_sql) // set current ps info session_info.set_ps_entry(ps_id_entry->ps_entry_); session_info.set_client_ps_id(ps_id_entry->ps_id_); + session_info.set_ps_id_entry(ps_id_entry); } } @@ -2021,7 +2251,193 @@ int ObMysqlSM::analyze_ps_prepare_request() return ret; } -int ObMysqlSM::analyze_ps_execute_request() +int ObMysqlSM::do_analyze_ps_execute_request_with_flag(ObPsIdEntry *ps_id_entry) +{ + int ret = OB_SUCCESS; + + int64_t param_num = ps_id_entry->get_param_count(); + ObIOBufferReader *param_type_reader = NULL; + char *param_type_buf = NULL; + int64_t param_type_pos = MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH + ((param_num + 7) /8) + 1; + int64_t param_type_len = 0; + int64_t param_offset = 0; + bool is_finish = false; + ObIArray ¶m_types = ps_id_entry->get_ps_sql_meta().get_param_types(); + param_types.reset(); + + if (OB_ISNULL(param_type_reader = client_buffer_reader_->clone())) { + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "fail to alloc param_type_reader ", KP_(client_buffer_reader), K(ret)); + } else if (OB_FAIL(param_type_reader->consume(param_type_pos))) { + PROXY_API_LOG(WARN, "failed to consume param_type_reader", K(param_type_pos), K(ret)); + } else if (OB_FAIL(ObMysqlRequestAnalyzer::parse_param_type_from_reader(param_offset, param_num, param_types, + param_type_reader, + param_type_len, is_finish))) { + LOG_WARN("fail to parse param type", K(param_offset), K(param_num), K(ret)); + } else if (OB_UNLIKELY(!is_finish)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param type is not finish, somethins is wrong", K(is_finish), K(param_type_len), K(ret)); + } else if (OB_ISNULL(param_type_buf = static_cast(op_fixed_mem_alloc(param_type_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc param type buf", K(param_type_len), K(ret)); + } else { + param_type_reader->copy(param_type_buf, param_type_len, 0); + if (OB_FAIL(ps_id_entry->get_ps_sql_meta().set_param_type(param_type_buf, param_type_len))) { + LOG_WARN("fail to set param type", K(param_type_len), K(ret)); + } else { + LOG_DEBUG("set param type success", KPC(ps_id_entry), K(param_types), K(param_type_len), K(ret)); + } + } + + if (NULL != param_type_reader) { + param_type_reader->dealloc(); + param_type_reader = NULL; + } + + if (NULL != param_type_buf) { + op_fixed_mem_free(param_type_buf, param_type_len); + } + return ret; +} + +int ObMysqlSM::do_analyze_ps_execute_request_without_flag(ObPsIdEntry *ps_id_entry) +{ + int ret = OB_SUCCESS; + + int64_t param_num = ps_id_entry->get_param_count(); + uint64_t read_avail = client_buffer_reader_->read_avail(); + const ObString& param_type = ps_id_entry->get_ps_sql_meta().get_param_type(); + int64_t param_type_pos = MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH + ((param_num + 7) /8) + 1; + // decode execute packet to old execute obj + if (OB_ISNULL(client_buffer_reader_) || OB_UNLIKELY(param_type.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("reader is null or param_type is emptry, which is unexpected", K(param_type), KPC(ps_id_entry), K(ret)); + } else { + int32_t pkt_length = static_cast(read_avail - 4 + param_type.length()); + + char header[MYSQL_PAYLOAD_LENGTH_LENGTH]; + int64_t pos = 0; + if (OB_FAIL(ObMySQLUtil::store_int3(header, MYSQL_PAYLOAD_LENGTH_LENGTH, pkt_length, pos))) { + LOG_WARN("fail to store pkg meta header", K(ret)); + } else { + int64_t new_param_bound_flag_pos = param_type_pos - 1; + int8_t new_param_bound_flag = 1; + client_buffer_reader_->replace(header, MYSQL_PAYLOAD_LENGTH_LENGTH, 0); + client_buffer_reader_->replace(reinterpret_cast(&new_param_bound_flag), 1, new_param_bound_flag_pos); + } + } + + if (OB_SUCC(ret)) { + int64_t written_len = 0; + ObMIOBuffer *writer = client_buffer_reader_->writer(); + ObMysqlAnalyzeResult result; + if (OB_ISNULL(writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null values ", KP(writer), K_(sm_id), K(ret)); + + // The previous content is sent out first + } else if(OB_FAIL(writer->write(client_buffer_reader_, param_type_pos, written_len))) { + LOG_WARN("fail to write execute header", KP_(client_buffer_reader), K(param_type_pos), K_(sm_id), K(ret)); + } else if (OB_UNLIKELY(written_len != param_type_pos)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to write to writer", "expected size", param_type_pos, + "actual size", written_len, K(ret)); + + // Output type content + } else if (OB_FAIL(writer->write(param_type.ptr(), param_type.length(), written_len))) { + LOG_WARN("fail to write param type", "length", param_type.length(), K_(sm_id), K(ret)); + } else if (OB_UNLIKELY(written_len != param_type.length())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to write to writer", "expected size", param_type.length(), + "actual size", written_len, K(ret)); + + // Output the remaining content of this time + } else if (OB_FAIL(do_analyze_ps_execute_request_with_remain_value(writer, read_avail, param_type_pos))) { + LOG_WARN("fail to analyze ps execute request with remain value", K(ret), K(read_avail), K(param_type_pos)); + + // consume the previous request packet + } else if (OB_FAIL(client_buffer_reader_->consume(read_avail))) { + LOG_WARN("fail to consume client_buffer_reader", K(read_avail), K(ret)); + // Check the integrity of the package + } else if (OB_FAIL(ObProxyParserUtils::analyze_one_packet(*client_buffer_reader_, result))) { + LOG_WARN("fail to analyze one packet", K(ret)); + } else if (OB_UNLIKELY(ANALYZE_DONE != result.status_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("analyze_one_packet do not return ANALYZE_DONE, which is unexpected", K(result.status_), K(ret)); + } else { + ObProxyMysqlRequest &client_request = trans_state_.trans_info_.client_request_; + client_request.add_request(client_buffer_reader_, trans_state_.mysql_config_params_->request_buffer_length_); + client_request.get_packet_meta().pkt_len_ = static_cast(client_buffer_reader_->read_avail()); + } + } + + return ret; +} + +int ObMysqlSM::do_analyze_ps_execute_request_with_remain_value(ObMIOBuffer *writer, + int64_t read_avail, + int64_t param_type_pos) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(read_avail == param_type_pos)) { + /* no error here, please check the format of OB_MYSQL_COM_STMT_EXECUTE, data could be sent by OB_MYSQL_COM_STMT_SEND_LONG_DATA */ + LOG_DEBUG("The value of each param from package OB_MYSQL_COM_STMT_EXECUTE is null, data maybe sent by OB_MYSQL_COM_STMT_SEND_LONG_DATA"); + } else if (OB_UNLIKELY(read_avail < param_type_pos)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid param", K(ret), K(read_avail), K(param_type_pos)); + } else { + int64_t written_len = 0; + if (OB_FAIL(writer->write(client_buffer_reader_, read_avail - param_type_pos, written_len, param_type_pos))) { + LOG_WARN("fail to write param value", KP_(client_buffer_reader), + "length", read_avail - param_type_pos, K_(sm_id), K(ret)); + } else if (OB_UNLIKELY(written_len != (read_avail - param_type_pos))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to write to writer", "expected size", read_avail - param_type_pos, + "actual size", written_len, K(ret)); + } else { + /* do nothing here */ + } + } + + return ret; +} + +int ObMysqlSM::do_analyze_ps_execute_request(ObPsIdEntry *ps_id_entry, bool is_large_request) +{ + int ret = OB_SUCCESS; + + // large request will handled by ObMysqlRequestExecuteTransformPlugin + int64_t param_num = ps_id_entry->get_param_count(); + if (!is_large_request && param_num > 0) { + int64_t local_read_avail = client_buffer_reader_->read_avail(); + int8_t new_param_bound_flag = 0; + int64_t new_param_bound_flag_pos = MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH + ((param_num + 7) /8); + + // Because the data has not been consumed, local_read_avail here is the data from the very beginning + if (OB_UNLIKELY(local_read_avail <= new_param_bound_flag_pos)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new_param_bound_flag is not 1 or 0, which is unexpected", K(ret)); + } else if (FALSE_IT(client_buffer_reader_->copy(reinterpret_cast(&new_param_bound_flag), 1, new_param_bound_flag_pos))) { + // must not be here + } else if (1 == new_param_bound_flag) { + if (OB_FAIL(do_analyze_ps_execute_request_with_flag(ps_id_entry))) { + LOG_WARN("fail to analyze execute request with flag", KPC(ps_id_entry), K(ret)); + } + } else if (0 == new_param_bound_flag) { + if (OB_FAIL(do_analyze_ps_execute_request_without_flag(ps_id_entry))) { + LOG_WARN("fail to analyze execute request without flag", KPC(ps_id_entry), K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new_param_bound_flag is not 1 or 0, which is unexpected", K(ret)); + } + } + + return ret; +} + +int ObMysqlSM::analyze_ps_execute_request(bool is_large_request) { int ret = OB_SUCCESS; @@ -2035,16 +2451,23 @@ int ObMysqlSM::analyze_ps_execute_request() const char *pos = data.ptr() + MYSQL_NET_META_LENGTH; uint32_t ps_id = 0; ObMySQLUtil::get_uint4(pos, ps_id); - ObPsEntry *entry = NULL; - if (OB_ISNULL(entry = session_info.get_ps_entry(ps_id)) || !entry->is_valid()) { + ObPsIdEntry *ps_id_entry = NULL; + if (OB_ISNULL(ps_id_entry = session_info.get_ps_id_entry(ps_id)) || !ps_id_entry->is_valid()) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("ps entry does not exist", K(ps_id), KPC(entry), K(ret)); + LOG_WARN("ps id entry does not exist", K(ps_id), KPC(ps_id_entry), K(ret)); } else { - session_info.set_ps_entry(entry); + session_info.set_ps_entry(ps_id_entry->get_ps_entry()); session_info.set_client_ps_id(ps_id); - client_request.set_ps_parse_result(&entry->get_base_ps_parse_result()); + session_info.set_ps_id_entry(ps_id_entry); + client_request.set_ps_parse_result(&ps_id_entry->get_ps_entry()->get_base_ps_parse_result()); // no need to analyze execute param value here, // will do analyze when needed before calculate partition id + + if (OB_MYSQL_COM_STMT_EXECUTE == trans_state_.trans_info_.sql_cmd_) { + if (OB_FAIL(do_analyze_ps_execute_request(ps_id_entry, is_large_request))) { + LOG_WARN("fail to do analyze ps execute request", KPC(ps_id_entry), K(is_large_request), K(ret)); + } + } } } return ret; @@ -2232,6 +2655,7 @@ int ObMysqlSM::state_watch_for_client_abort(int event, void *data) case VC_EVENT_EOS: static_cast(client_entry_->vc_)->get_netvc()->do_io_shutdown(IO_SHUTDOWN_READ); client_entry_->eos_ = true; + __attribute__ ((fallthrough)); case VC_EVENT_ERROR: case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: @@ -2352,6 +2776,7 @@ int ObMysqlSM::state_api_callout(int event, void *data) void ObMysqlSM::handle_api_return() { int ret = OB_SUCCESS; + skip_plugin_ = false; switch (trans_state_.api_next_action_) { case ObMysqlTransact::SM_ACTION_API_SM_START: if (OB_FAIL(setup_client_request_read())) { @@ -2577,7 +3002,7 @@ int ObMysqlSM::state_server_response_read(int event, void *data) ret = OB_INNER_STAT_ERROR; LOG_ERROR("invalid internal state, server entry is NULL or data is NULL", K_(server_entry), K(data), K_(sm_id), K(ret)); - } else if (server_entry_->read_vio_ != reinterpret_cast(data)) { + } else if (OB_UNLIKELY(server_entry_->read_vio_ != reinterpret_cast(data))) { ret = OB_INNER_STAT_ERROR; LOG_ERROR("invalid internal state, server entry read vio is not the same as data", K_(server_entry_->read_vio), K(data), K_(sm_id), K(ret)); @@ -2616,11 +3041,13 @@ int ObMysqlSM::state_server_response_read(int event, void *data) } } - // set server_read_begin_ no matter if error or timeout happen - if (0 == milestones_.server_.server_read_begin_) { - milestones_.server_.server_read_begin_ = get_based_hrtime(); - cmd_time_stats_.server_process_request_time_ = - milestone_diff(milestones_.server_.server_write_end_, milestones_.server_.server_read_begin_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + // set server_read_begin_ no matter if error or timeout happen + if (0 == milestones_.server_.server_read_begin_) { + milestones_.server_.server_read_begin_ = get_based_hrtime(); + cmd_time_stats_.server_process_request_time_ = + milestone_diff(milestones_.server_.server_write_end_, milestones_.server_.server_read_begin_); + } } if (OB_SUCC(ret)) { @@ -2635,8 +3062,8 @@ int ObMysqlSM::state_server_response_read(int event, void *data) } bool need_receive_completed = false; - if ((OB_LIKELY(NULL != client_session_) && client_session_->is_proxy_mysql_client_) - || ObMysqlTransact::SERVER_SEND_REQUEST != trans_state_.current_.send_action_) { + if (OB_UNLIKELY((NULL != client_session_ && client_session_->is_proxy_mysql_client_) + || ObMysqlTransact::SERVER_SEND_REQUEST != trans_state_.current_.send_action_)) { need_receive_completed = true; } @@ -2646,7 +3073,7 @@ int ObMysqlSM::state_server_response_read(int event, void *data) state = ANALYZE_ERROR; } - if (ANALYZE_CONT != state) { + if (OB_UNLIKELY(ANALYZE_CONT != state)) { // Disable further I/O // Read the first packet, or command complete or transaction complete, // or error happen; @@ -2654,12 +3081,14 @@ int ObMysqlSM::state_server_response_read(int event, void *data) // be rest request body that we are tunneling, and we can't issue // another IO later for the rest request body with a different buffer server_entry_->read_vio_->nbytes_ = server_entry_->read_vio_->ndone_; - if (0 == milestones_.server_.server_read_end_) { - milestones_.server_.server_read_end_ = get_based_hrtime(); - cmd_time_stats_.server_response_read_time_ += ( - milestone_diff(milestones_.server_.server_read_begin_, milestones_.server_.server_read_end_) - - cmd_time_stats_.plugin_decompress_response_time_ - - cmd_time_stats_.server_response_analyze_time_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + if (0 == milestones_.server_.server_read_end_) { + milestones_.server_.server_read_end_ = get_based_hrtime(); + cmd_time_stats_.server_response_read_time_ += ( + milestone_diff(milestones_.server_.server_read_begin_, milestones_.server_.server_read_end_) + - cmd_time_stats_.plugin_decompress_response_time_ + - cmd_time_stats_.server_response_analyze_time_); + } } } @@ -2708,6 +3137,11 @@ int ObMysqlSM::state_server_response_read(int event, void *data) trans_state_.current_.state_ = ObMysqlTransact::CONNECTION_ALIVE; trans_state_.transact_return_point = ObMysqlTransact::handle_response; + if (OB_MYSQL_COM_QUERY == trans_state_.trans_info_.sql_cmd_ + && (trans_state_.trans_info_.server_response_.get_analyze_result().is_decompressed() + || PROTOCOL_NORMAL == use_compression_protocol())) { + skip_plugin_ = true; + } callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_READ_RESPONSE); break; } @@ -2798,20 +3232,19 @@ bool ObMysqlSM::is_cloud_user() const { bool bret = false; if (OB_LIKELY(NULL != client_session_)) { - bret = client_session_->is_need_convert_vip_to_tname() - || client_session_->is_need_use_proxy_tenant_name(); + bret = client_session_->is_need_convert_vip_to_tname(); } return bret; } bool ObMysqlSM::need_reject_user_login(const ObString &user, const ObString &tenant, - const bool has_full_username, const bool is_cloud_user) const + const bool has_tenant_username, const bool has_cluster_username, + const bool is_cloud_user) const { // blew case need reject user login: // 1. user proxyro@sys user - // 2. user LBACSYS - // 3. user ORAAUDITOR - // 4. cloud user with tenant or cluster + // 2. Cloud users with tenant or cluster + // 3. Non-cloud users do not bring cluster bool bret = false; if (OB_UNLIKELY(NULL != client_session_) && !client_session_->is_proxy_mysql_client_ @@ -2820,8 +3253,11 @@ bool ObMysqlSM::need_reject_user_login(const ObString &user, const ObString &ten && tenant.case_compare(OB_SYS_TENANT_NAME) == 0 && user.case_compare(ObProxyTableInfo::READ_ONLY_USERNAME_USER) == 0) || (is_cloud_user && !get_global_proxy_config().enable_cloud_full_username - && has_full_username); + && (has_tenant_username || has_cluster_username)) + || (!is_cloud_user && get_global_proxy_config().enable_full_username + && (!has_cluster_username || !has_tenant_username)); } + return bret; } @@ -2882,7 +3318,7 @@ inline int ObMysqlSM::handle_first_response_packet(ObMysqlAnalyzeStatus &state, LOG_DEBUG("handle_first_response_packet", K(need_receive_completed)); ObHRTime analyze_response_begin = get_based_hrtime(); - if (client_session_->get_session_info().is_oceanbase_server()) { + if (OB_LIKELY(client_session_->get_session_info().is_oceanbase_server())) { if (OB_FAIL(handle_oceanbase_first_response_packet(state, need_receive_completed, first_pkt_len))) { LOG_WARN("fail to handle oceanbase first response packet", K(ret)); } @@ -2903,7 +3339,7 @@ inline int ObMysqlSM::handle_oceanbase_first_response_packet(ObMysqlAnalyzeStatu ObProxyProtocol ob_proxy_protocol = use_compression_protocol(); - if (PROTOCOL_CHECKSUM == ob_proxy_protocol || PROTOCOL_OB20 == ob_proxy_protocol) { // compressed protocol + if (OB_UNLIKELY(PROTOCOL_CHECKSUM == ob_proxy_protocol || PROTOCOL_OB20 == ob_proxy_protocol)) { // compressed protocol if (OB_FAIL(handle_first_compress_response_packet(state, need_receive_completed, first_pkt_len))) { LOG_WARN("fail to handle_first_compress_response_packet", K(need_receive_completed), K(ret)); } @@ -2927,6 +3363,7 @@ inline int ObMysqlSM::handle_first_compress_response_packet(ObMysqlAnalyzeStatus compress_analyzer->reset(); const uint8_t req_seq = get_request_seq(); const ObMySQLCmd cmd = get_request_cmd(); + const ObMysqlProtocolMode mysql_mode = client_session_->get_session_info().is_oracle_mode() ? OCEANBASE_ORACLE_PROTOCOL_MODE : OCEANBASE_MYSQL_PROTOCOL_MODE; const bool enable_extra_ok_packet_for_stats = is_extra_ok_packet_for_stats_enabled(); if (server_buffer_reader_->is_read_avail_more_than(MYSQL_COMPRESSED_HEALDER_LENGTH)) { @@ -2937,7 +3374,7 @@ inline int ObMysqlSM::handle_first_compress_response_packet(ObMysqlAnalyzeStatus mode = ObMysqlCompressAnalyzer::DECOMPRESS_MODE; } - if (OB_FAIL(compress_analyzer->init(req_seq, mode, cmd, enable_extra_ok_packet_for_stats, + if (OB_FAIL(compress_analyzer->init(req_seq, mode, cmd, mysql_mode, enable_extra_ok_packet_for_stats, req_seq, server_session_->get_server_request_id(), server_session_->get_server_sessid()))) { LOG_WARN("fail to init compress_analyzer", K_(sm_id), K(req_seq), K(cmd), K(ret)); @@ -2980,18 +3417,19 @@ inline int ObMysqlSM::handle_first_normal_response_packet(ObMysqlAnalyzeStatus & is_autocommit_0 = true; } - if (server_buffer_reader_->is_read_avail_more_than(MYSQL_NET_META_LENGTH)) { - if (!client_session_->get_session_info().is_oceanbase_server() - || trans_state_.mysql_config_params_->is_mysql_routing_mode()) { + if (OB_LIKELY(server_buffer_reader_->is_read_avail_more_than(MYSQL_NET_META_LENGTH))) { + if (OB_UNLIKELY(!client_session_->get_session_info().is_oceanbase_server() + || trans_state_.mysql_config_params_->is_mysql_routing_mode())) { LOG_DEBUG("handle_first_normal_response_packet", K(trans_state_.current_.state_), K(is_in_trans_), K(is_autocommit_0)); analyzer_.set_server_cmd(trans_state_.trans_info_.sql_cmd_, STANDARD_MYSQL_PROTOCOL_MODE, false, ObMysqlTransact::is_in_trans(trans_state_)|| is_in_trans_|| is_autocommit_0); } else { const bool enable_extra_ok_packet_for_stats = is_extra_ok_packet_for_stats_enabled(); + const ObMysqlProtocolMode mysql_mode = client_session_->get_session_info().is_oracle_mode() ? OCEANBASE_ORACLE_PROTOCOL_MODE : OCEANBASE_MYSQL_PROTOCOL_MODE; analyzer_.set_server_cmd(trans_state_.trans_info_.sql_cmd_, - OCEANBASE_MYSQL_PROTOCOL_MODE, enable_extra_ok_packet_for_stats, - ObMysqlTransact::is_in_trans(trans_state_) || is_in_trans_ || is_autocommit_0); + mysql_mode, enable_extra_ok_packet_for_stats, + ObMysqlTransact::is_in_trans(trans_state_) || is_in_trans_ || is_autocommit_0); LOG_DEBUG("handle_first_normal_response_packet", K(trans_state_.current_.state_), K(is_in_trans_), K(is_autocommit_0), K(enable_extra_ok_packet_for_stats)); } @@ -3019,8 +3457,8 @@ int ObMysqlSM::state_server_request_send(int event, void *data) ret = OB_INNER_STAT_ERROR; LOG_ERROR("invalid internal state, server entry is NULL or data is NULL", K_(server_entry), K(data), K_(sm_id), K(ret)); - } else if (server_entry_->read_vio_ != reinterpret_cast(data) - && server_entry_->write_vio_ != reinterpret_cast(data)) { + } else if (OB_UNLIKELY(server_entry_->read_vio_ != reinterpret_cast(data) + && server_entry_->write_vio_ != reinterpret_cast(data))) { ret = OB_INNER_STAT_ERROR; LOG_ERROR("invalid internal state, server entry read vio isn't the same as data," "and server entry write vio isn't the same as data", @@ -3033,23 +3471,27 @@ int ObMysqlSM::state_server_request_send(int event, void *data) break; case VC_EVENT_WRITE_COMPLETE: { - milestones_.server_.server_write_end_ = get_based_hrtime(); - cmd_time_stats_.server_request_write_time_ += (milestones_.server_.server_write_end_ - milestones_.server_.server_write_begin_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.server_.server_write_end_ = get_based_hrtime(); + cmd_time_stats_.server_request_write_time_ += (milestones_.server_.server_write_end_ - milestones_.server_.server_write_begin_); + } // We are done sending the request, deallocate our // buffer and then decide what to do next - if (NULL != server_entry_->write_buffer_) { + if (OB_UNLIKELY(NULL != server_entry_->write_buffer_)) { free_miobuffer(server_entry_->write_buffer_); server_entry_->write_buffer_ = NULL; } ObMySQLCmd request_cmd = trans_state_.trans_info_.client_request_.get_packet_meta().cmd_; // before send quit cmd to observer, maybe we need send session vars first. // after send quit cmd to observer, this connection will disconnect soon. - if ((OB_MYSQL_COM_QUIT == request_cmd) && (ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { + if (OB_UNLIKELY(OB_MYSQL_COM_QUIT == request_cmd + && ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { // when receive 'quit' cmd, obproxy should disconnect after sending it to observer LOG_DEBUG("[setup_server_response_read] send quit to observer completed," " this connection will disconnect soon"); ret = OB_CONNECT_ERROR; - } else if ((OB_MYSQL_COM_STMT_CLOSE == request_cmd) && (ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { + } else if (OB_UNLIKELY(OB_MYSQL_COM_STMT_CLOSE == request_cmd + && ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { // remove ps_id_pair and cursor_id_pair ObClientSessionInfo &cs_info = client_session_->get_session_info(); ObServerSessionInfo &ss_info = server_session_->get_session_info(); @@ -3065,9 +3507,11 @@ int ObMysqlSM::state_server_request_send(int event, void *data) } call_transact_and_set_next_state(ObMysqlTransact::handle_request); - } else if (ObMysqlTransact::SERVER_SEND_SSL_REQUEST == trans_state_.current_.send_action_) { + } else if (OB_UNLIKELY(ObMysqlTransact::SERVER_SEND_SSL_REQUEST == trans_state_.current_.send_action_)) { ObUnixNetVConnection *vc = static_cast(server_session_->get_netvc()); - if (OB_FAIL(vc->ssl_init(ObUnixNetVConnection::SSL_CLIENT))) { + if (OB_FAIL(vc->ssl_init(ObUnixNetVConnection::SSL_CLIENT, + client_session_->get_vip_cluster_name(), + client_session_->get_vip_tenant_name()))) { LOG_WARN("client ssl init failed", K(ret)); } else if (trans_state_.is_auth_request_) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_LOGIN; @@ -3082,8 +3526,15 @@ int ObMysqlSM::state_server_request_send(int event, void *data) LOG_WARN("setup server request send failed", K(ret)); } } + } else if (OB_UNLIKELY(OB_MYSQL_COM_STMT_SEND_LONG_DATA == request_cmd + && ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { + if (OB_FAIL(handle_server_request_send_long_data())) { + LOG_WARN("fail to handle state server request send long data", K(ret)); + } else { + LOG_DEBUG("succ to handle state server request send long data"); + } } else { - if (!tunnel_.is_tunnel_active()) { + if (OB_LIKELY(!tunnel_.is_tunnel_active())) { // It's time to start reading the response if (OB_FAIL(setup_server_response_read())) { LOG_WARN("failed to setup_server_response_read", K_(sm_id), K(ret)); @@ -3135,6 +3586,49 @@ int ObMysqlSM::state_server_request_send(int event, void *data) return VC_EVENT_NONE; } +/* + * OB_MYSQL_COM_STMT_SEND_LONG_DATA has no rsp from server, trans to handle request + * use piece_info_map to record the server addr info + * only record after first send_long_data, remove after execute/close + */ +int ObMysqlSM::handle_server_request_send_long_data() +{ + int ret = OB_SUCCESS; + + ObPieceInfo *info = NULL; + ObClientSessionInfo &cs_info = client_session_->get_session_info(); + if (OB_FAIL(cs_info.get_piece_info(info))) { + if (OB_HASH_NOT_EXIST == ret) { + if (OB_ISNULL(info = op_alloc(ObPieceInfo))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc mem for piece info", K(ret)); + } else { + info->set_ps_id(cs_info.get_client_ps_id()); + info->set_addr(server_session_->get_netvc()->get_remote_addr()); + if (OB_FAIL(cs_info.add_piece_info(info))) { + LOG_WARN("fail to add piece info", K(ret)); + op_free(info); + info = NULL; + } + } + } + } + + if (OB_SUCC(ret)) { + if (ObMysqlTransact::is_in_trans(trans_state_)) { + trans_state_.current_.state_ = ObMysqlTransact::CMD_COMPLETE; + } else { + trans_state_.current_.state_ = ObMysqlTransact::TRANSACTION_COMPLETE; + } + + LOG_DEBUG("send_long_data send finish, trans to handle request"); + release_server_session(); + callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_CMD_COMPLETE); + } + + return ret; +} + int ObMysqlSM::process_partition_location(ObMysqlRouteResult &result) { int ret = OB_SUCCESS; @@ -3159,7 +3653,7 @@ int ObMysqlSM::process_partition_location(ObMysqlRouteResult &result) K_(sm_id), K(type), K(ret)); // if failed, encode err packet and send to client, then disconnect int ret_tmp = OB_SUCCESS; - trans_state_.mysql_errcode_ = OB_PASSWORD_WRONG; + trans_state_.mysql_errcode_ = OB_GET_LOCATION_TIME_OUT; if (OB_UNLIKELY(OB_SUCCESS != (ret_tmp = ObMysqlTransact::build_error_packet(trans_state_)))) { LOG_WARN("fail to build err packet", K(ret_tmp)); } @@ -3288,7 +3782,7 @@ int ObMysqlSM::process_server_addr_lookup(const ObProxyKillQueryInfo *query_info LOG_WARN("unexpected null values ", K(writer), K_(sm_id), K(ret)); } else if (OB_FAIL(ObServerAddrLookupHandler::build_kill_query_sql(query_info->real_conn_id_, sql))) { LOG_WARN("fail to build_kill_query_sql", K(*query_info), K_(sm_id), K(ret)); - // no need compress here, if server session support comrpess, it will compress later + // no need compress here, if server session support compress, it will compress later } else if (OB_FAIL(ObMysqlRequestBuilder::build_mysql_request(*writer, OB_MYSQL_COM_QUERY, sql.string(), use_compress, is_checksum_on))) { LOG_WARN("fail to build_mysql_request", K(*query_info), K_(sm_id), K(ret)); @@ -3356,10 +3850,11 @@ int ObMysqlSM::state_partition_location_lookup(int event, void *data) STATE_ENTER(ObMysqlSM::state_partition_location_lookup, event); pending_action_ = NULL; - milestones_.pl_lookup_end_ = get_based_hrtime(); - cmd_time_stats_.pl_lookup_time_ += milestone_diff(milestones_.pl_lookup_begin_, milestones_.pl_lookup_end_); - - milestones_.pl_process_begin_ = milestones_.pl_lookup_end_; + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.pl_lookup_end_ = get_based_hrtime(); + cmd_time_stats_.pl_lookup_time_ += milestone_diff(milestones_.pl_lookup_begin_, milestones_.pl_lookup_end_); + milestones_.pl_process_begin_ = milestones_.pl_lookup_end_; + } if (OB_UNLIKELY(TABLE_ENTRY_EVENT_LOOKUP_DONE != event) || OB_ISNULL(data)) { ret = OB_ERR_UNEXPECTED; @@ -3379,7 +3874,7 @@ int ObMysqlSM::state_partition_location_lookup(int event, void *data) } if (OB_SUCC(ret)) { - if (trans_state_.pll_info_.is_cached_dummy_force_renew()) { + if (OB_UNLIKELY(trans_state_.pll_info_.is_cached_dummy_force_renew())) { trans_state_.pll_info_.lookup_success_ = false; milestones_.pl_process_end_ = get_based_hrtime(); cmd_time_stats_.pl_process_time_ += milestone_diff(milestones_.pl_process_begin_, milestones_.pl_process_end_); @@ -3604,7 +4099,7 @@ int ObMysqlSM::tunnel_handler_response_transfered(int event, void *data) // There are two places to release server sssion normally: // 1. During the transaction, tunnel_handler_server // 2. End of transaction, setup_cmd_complete - // For the COM_STMT_FETCH in the transaction, if you need to switch to another Server: + // For the OB_MYSQL_COM_STMT_FETCH in the transaction, if you need to switch to another Server: // 1. In tunnel_handler_server, it is considered that the transaction is over. Because in_trans = false; // 2. Since the transaction status is modified here, it is considered to be in transaction in the setup_cmd_complete // So neither of the above will be released, so here we have to release it once @@ -4136,12 +4631,12 @@ int ObMysqlSM::trim_ok_packet(ObIOBufferReader &reader) // reset route_addr client_session->get_session_info().set_obproxy_route_addr(0); - if (is_causal_order_read_enabled()) { - // handle safe snapshot version - if (WEAK == trans_state_.get_trans_consistency_level(client_session->get_session_info()) - && trans_state_.trans_info_.client_request_.get_parse_result().is_select_stmt()) { - update_safe_read_snapshot(); - } + if (OB_UNLIKELY(is_causal_order_read_enabled())) { + // handle safe snapshot version + if (WEAK == trans_state_.get_trans_consistency_level(client_session->get_session_info()) + && trans_state_.trans_info_.client_request_.get_parse_result().is_select_stmt()) { + update_safe_read_snapshot(); + } } // handle other variables @@ -4279,6 +4774,7 @@ int ObMysqlSM::tunnel_handler_request_transfer_client(int event, ObMysqlTunnelPr // My reading of spec says that user clients can not terminate // request transfer with a half close so this is an error client_entry_->eos_ = true; + __attribute__ ((fallthrough)); case VC_EVENT_ERROR: case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ACTIVE_TIMEOUT: @@ -4494,11 +4990,12 @@ int ObMysqlSM::tunnel_handler_plugin_client(int event, ObMysqlTunnelConsumer &c) void ObMysqlSM::do_congestion_control_lookup() { - milestones_.congestion_control_begin_ = get_based_hrtime(); - milestones_.congestion_control_end_ = 0; + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.congestion_control_begin_ = get_based_hrtime(); + milestones_.congestion_control_end_ = 0; + } bool enable_congestion = trans_state_.mysql_config_params_->enable_congestion_; - bool is_cur_congestion_avail = sm_cluster_resource_->is_congestion_avail(); // we will lookup congestion, only the follow all happened // 1. enable_congestion from config @@ -4507,20 +5004,20 @@ void ObMysqlSM::do_congestion_control_lookup() // 4. not mysql route mode // // Attention! when force_retry_congested_, also need do congestion lookup - if (enable_congestion - && is_cur_congestion_avail + if (OB_UNLIKELY(enable_congestion + && sm_cluster_resource_->is_congestion_avail() && trans_state_.need_pl_lookup_ - && !trans_state_.mysql_config_params_->is_mysql_routing_mode()) { + && !trans_state_.mysql_config_params_->is_mysql_routing_mode())) { trans_state_.need_congestion_lookup_ = true; } else { LOG_DEBUG("no need do congestion lookup", K_(sm_id), K(enable_congestion), - K(is_cur_congestion_avail), "need_pl_lookup", trans_state_.need_pl_lookup_, + K(sm_cluster_resource_->is_congestion_avail()), "need_pl_lookup", trans_state_.need_pl_lookup_, "force_retry_congestion", trans_state_.force_retry_congested_, "route_mode", trans_state_.mysql_config_params_->server_routing_mode_); trans_state_.need_congestion_lookup_ = false; } - if (!trans_state_.need_congestion_lookup_) { // no need congestion lookup + if (OB_LIKELY(!trans_state_.need_congestion_lookup_)) { // no need congestion lookup // call ObMysqlTransact::handle_congestion_control_lookup() to handle fail / success call_transact_and_set_next_state(NULL); } else { @@ -4568,10 +5065,12 @@ void ObMysqlSM::do_congestion_control_lookup() void ObMysqlSM::do_partition_location_lookup() { - milestones_.pl_lookup_begin_ = get_based_hrtime(); - milestones_.pl_lookup_end_ = 0; + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.pl_lookup_begin_ = get_based_hrtime(); + milestones_.pl_lookup_end_ = 0; + } - if (trans_state_.api_server_addr_set_) { + if (OB_UNLIKELY(trans_state_.api_server_addr_set_)) { // If the API has set the server address before the partition // location lookup then we can skip the lookup LOG_DEBUG("[ObMysqlSM::do_partition_location_lookup] Skipping partition location " @@ -4599,44 +5098,84 @@ void ObMysqlSM::do_partition_location_lookup() } else { const ObTableEntryName &name = trans_state_.pll_info_.te_name_; - ObRouteParam param; - param.cont_ = this; - param.force_renew_ = trans_state_.pll_info_.is_force_renew(); - param.is_need_force_flush_ = trans_state_.pll_info_.is_need_force_flush(); - param.use_lower_case_name_ = client_session_->get_session_info().need_use_lower_case_names(); - param.mysql_proxy_ = &sm_cluster_resource_->mysql_proxy_; - param.cr_version_ = sm_cluster_resource_->version_; - if (get_global_resource_pool_processor().get_default_cluster_resource() == sm_cluster_resource_) { - // default cluster resource cluster id is always 0, and it is used only for building cluster resource - param.cr_id_ = client_session_->get_session_info().get_cluster_id(); - } else { - param.cr_id_ = sm_cluster_resource_->get_cluster_id(); - } - param.tenant_version_ = sm_cluster_resource_->get_location_tenant_version( - client_session_->get_session_info().get_priv_info().tenant_name_); - param.timeout_us_ = hrtime_to_usec(trans_state_.mysql_config_params_->short_async_task_timeout_); - param.is_partition_table_route_supported_ = is_partition_table_route_supported(); - param.is_oracle_mode_ = client_session_->get_session_info().is_oracle_mode(); - param.client_request_ = &trans_state_.trans_info_.client_request_; // priv parse result - param.client_info_ = &client_session_->get_session_info(); - param.need_pl_route_ = (is_pl_route_supported() && (param.client_request_->get_parse_result().is_call_stmt() - || param.client_request_->get_parse_result().is_text_ps_call_stmt())); - param.current_idc_name_ = client_session_->get_current_idc_name();//shallow copy - if (trans_state_.pll_info_.is_cached_dummy_force_renew()) { - param.need_pl_route_ = false; - param.name_.shallow_copy(name.cluster_name_, name.tenant_name_, - ObString::make_string(OB_SYS_DATABASE_NAME), - ObString::make_string(OB_ALL_DUMMY_TNAME)); - } else { - param.name_.shallow_copy(name); + int ret = OB_SUCCESS; + ObAction *pl_lookup_action_handle = NULL; + bool find_entry = false; + bool need_pl_route = (is_pl_route_supported() && (trans_state_.trans_info_.client_request_.get_parse_result().is_call_stmt() + || trans_state_.trans_info_.client_request_.get_parse_result().is_text_ps_call_stmt())); + // Get it from table_map first + if (OB_LIKELY(!trans_state_.pll_info_.is_force_renew() && !name.is_all_dummy_table() + && !trans_state_.pll_info_.is_cached_dummy_force_renew()) + && !need_pl_route) { + ObTableRefHashMap &table_map = self_ethread().get_table_map(); + ObTableEntry *tmp_entry = NULL; + int64_t cr_id = 0; + if (get_global_resource_pool_processor().get_default_cluster_resource() == sm_cluster_resource_) { + // default cluster resource cluster id is always 0, and it is used only for building cluster resource + cr_id = client_session_->get_session_info().get_cluster_id(); + } else { + cr_id = sm_cluster_resource_->get_cluster_id(); + } + ObTableEntryKey key(name, sm_cluster_resource_->version_, cr_id); + tmp_entry = table_map.get(key); + if (NULL != tmp_entry && !tmp_entry->is_partition_table() + && (tmp_entry->is_avail_state() || tmp_entry->is_updating_state()) + && !(get_global_table_cache().is_table_entry_expired(*tmp_entry))) { + tmp_entry->renew_last_access_time(); + ObMysqlRouteResult result; + result.table_entry_ = tmp_entry; + result.is_table_entry_from_remote_ = false; + result.has_dup_replica_ = tmp_entry->has_dup_replica(); + tmp_entry->set_need_force_flush(false); + find_entry = true; + LOG_DEBUG("get table entry from thread map", KPC(tmp_entry)); + state_partition_location_lookup(TABLE_ENTRY_EVENT_LOOKUP_DONE, &result); + } else if (NULL != tmp_entry) { + tmp_entry->dec_ref(); + } } - LOG_DEBUG("Doing partition location Lookup", K_(sm_id), K(param)); - ObAction *pl_lookup_action_handle = NULL; - int ret = ObMysqlRoute::get_route_entry(param, sm_cluster_resource_, pl_lookup_action_handle); + if (OB_UNLIKELY(!find_entry)) { + ObRouteParam param; + param.cont_ = this; + param.force_renew_ = trans_state_.pll_info_.is_force_renew(); + param.is_need_force_flush_ = trans_state_.pll_info_.is_need_force_flush(); + param.use_lower_case_name_ = client_session_->get_session_info().need_use_lower_case_names(); + param.mysql_proxy_ = &sm_cluster_resource_->mysql_proxy_; + param.cr_version_ = sm_cluster_resource_->version_; + if (get_global_resource_pool_processor().get_default_cluster_resource() == sm_cluster_resource_) { + // default cluster resource cluster id is always 0, and it is used only for building cluster resource + param.cr_id_ = client_session_->get_session_info().get_cluster_id(); + } else { + param.cr_id_ = sm_cluster_resource_->get_cluster_id(); + } + if (OB_UNLIKELY(get_global_proxy_config().check_tenant_locality_change)) { + param.tenant_version_ = sm_cluster_resource_->get_location_tenant_version( + client_session_->get_session_info().get_priv_info().tenant_name_); + } + param.timeout_us_ = hrtime_to_usec(trans_state_.mysql_config_params_->short_async_task_timeout_); + param.is_partition_table_route_supported_ = is_partition_table_route_supported(); + param.is_oracle_mode_ = client_session_->get_session_info().is_oracle_mode(); + param.client_request_ = &trans_state_.trans_info_.client_request_; // priv parse result + param.client_info_ = &client_session_->get_session_info(); + param.need_pl_route_ = need_pl_route; + param.current_idc_name_ = client_session_->get_current_idc_name();//shallow copy + if (trans_state_.pll_info_.is_cached_dummy_force_renew()) { + param.need_pl_route_ = false; + param.name_.shallow_copy(name.cluster_name_, name.tenant_name_, + ObString::make_string(OB_SYS_DATABASE_NAME), + ObString::make_string(OB_ALL_DUMMY_TNAME)); + } else { + param.name_.shallow_copy(name); + } + + LOG_DEBUG("Doing partition location Lookup", K_(sm_id), K(param)); + pl_lookup_action_handle = NULL; + ret = ObMysqlRoute::get_route_entry(param, sm_cluster_resource_, pl_lookup_action_handle); + } if (OB_SUCC(ret)) { - if (NULL == pl_lookup_action_handle) { + if (OB_LIKELY(NULL == pl_lookup_action_handle)) { // cache hit and has called back, do nothing } else { pending_action_ = pl_lookup_action_handle; @@ -4720,11 +5259,13 @@ int ObMysqlSM::do_observer_open() int ret = OB_SUCCESS; MYSQL_SM_SET_DEFAULT_HANDLER(&ObMysqlSM::state_observer_open); - milestones_.do_observer_open_begin_ = get_based_hrtime(); - milestones_.do_observer_open_end_ = 0; + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.do_observer_open_begin_ = get_based_hrtime(); + milestones_.do_observer_open_end_ = 0; + } // if sync all variables completed, send request directly through handle_observer_open - if (trans_state_.send_reqeust_direct_) { + if (OB_UNLIKELY(trans_state_.send_reqeust_direct_)) { if (OB_ISNULL(server_session_) || ObMysqlTransact::SERVER_SEND_REQUEST != trans_state_.current_.send_action_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected state", K(server_session_), K(trans_state_.current_.send_action_), K(ret)); @@ -4733,11 +5274,11 @@ int ObMysqlSM::do_observer_open() } } else { // We need to close the previous attempt, except reroute - if (trans_state_.is_rerouted_) { + if (OB_UNLIKELY(trans_state_.is_rerouted_)) { release_server_session(); } - if (NULL != server_entry_) { + if (OB_UNLIKELY(NULL != server_entry_)) { if (MYSQL_SERVER_VC != server_entry_->vc_type_) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("invlid internal state, server entry vc type is unexpected", @@ -4760,7 +5301,7 @@ int ObMysqlSM::do_observer_open() // dependency, obproxy must ensure that the client finishes sending its // request and for this reason, the inactivity timeout cannot be // cancelled. - if (NULL != client_session_) { + if (OB_LIKELY(NULL != client_session_)) { client_session_->get_netvc()->cancel_inactivity_timeout(); } } @@ -4782,15 +5323,15 @@ int ObMysqlSM::do_observer_open() "force_retry_congested", trans_state_.force_retry_congested_); // we check this version only when safe_weak_read is enable - if (client_session_->get_session_info().is_oceanbase_server() + if (OB_UNLIKELY(client_session_->get_session_info().is_oceanbase_server() && is_causal_order_read_enabled() && NULL != client_session_ && WEAK == trans_state_.get_trans_consistency_level(client_session_->get_session_info()) - && trans_state_.trans_info_.client_request_.get_parse_result().is_select_stmt()) { + && trans_state_.trans_info_.client_request_.get_parse_result().is_select_stmt())) { ObMysqlTransact::check_safe_read_snapshot(trans_state_); } - if (NULL != api_.plugin_tunnel_) { + if (OB_UNLIKELY(NULL != api_.plugin_tunnel_)) { ObPluginVCCore *t = api_.plugin_tunnel_; api_.plugin_tunnel_ = NULL; @@ -4799,7 +5340,7 @@ int ObMysqlSM::do_observer_open() if (OB_FAIL(ret) || NULL != pvc_action_handle) { LOG_WARN("failed plugin vc to connect_re", K(pvc_action_handle), K_(sm_id), K(ret)); } - } else if (MYSQL_NO_PLUGIN_TUNNEL != api_.plugin_tunnel_type_) { + } else if (OB_UNLIKELY(MYSQL_NO_PLUGIN_TUNNEL != api_.plugin_tunnel_type_)) { ret = OB_INNER_STAT_ERROR; LOG_ERROR("invalid tunnel type, should be MYSQL_NO_PLUGIN_TUNNEL", K_(api_.plugin_tunnel_type), K_(sm_id), K(ret)); @@ -4809,7 +5350,7 @@ int ObMysqlSM::do_observer_open() } } - if (OB_SESSION_POOL_FULL_ERROR == ret) { + if (OB_UNLIKELY(OB_SESSION_POOL_FULL_ERROR == ret)) { ret = handle_retry_acquire_svr_session(); } @@ -4952,7 +5493,7 @@ inline int ObMysqlSM::do_oceanbase_internal_observer_open(ObMysqlServerSession * } */ LOG_DEBUG("sql cmd", K(trans_state_.trans_info_.sql_cmd_), K(client_session_->is_session_pool_client())); - if (trans_state_.trans_info_.sql_cmd_ == OB_MYSQL_COM_LOGIN && client_session_->is_session_pool_client() && !client_session_->can_direct_ok()) { + if (OB_UNLIKELY(trans_state_.trans_info_.sql_cmd_ == OB_MYSQL_COM_LOGIN && client_session_->is_session_pool_client() && !client_session_->can_direct_ok())) { LOG_DEBUG("OB_MYSQL_COM_LOGIN here not use pool"); // only proxy_mysql_client for session pool use pool_sever_addr // this logic is for pre connection create @@ -4960,12 +5501,15 @@ inline int ObMysqlSM::do_oceanbase_internal_observer_open(ObMysqlServerSession * ret = use_set_pool_addr(); } } else { - const bool need_close_last_ss = need_close_last_used_ss(); + bool need_close_last_ss = false; + if (OB_UNLIKELY(client_session_->get_session_info().is_read_consistency_set())) { + need_close_last_ss = need_close_last_used_ss(); + } ret = client_session_->acquire_svr_session(trans_state_.server_info_.addr_.sa_, need_close_last_ss, selected_session); - if ((OB_SUCCESS == ret && NULL != selected_session) - || (OB_SESSION_NOT_FOUND == ret && NULL == selected_session)) { + if (OB_LIKELY((OB_SUCCESS == ret && NULL != selected_session) + || (OB_SESSION_NOT_FOUND == ret && NULL == selected_session))) { ret = OB_SUCCESS; - } else if (OB_SESSION_POOL_FULL_ERROR == ret) { + } else if (OB_UNLIKELY(OB_SESSION_POOL_FULL_ERROR == ret)) { // now last is release client_session_->attach_server_session(NULL); server_entry_ = NULL; @@ -5031,7 +5575,7 @@ inline int ObMysqlSM::do_internal_observer_open() int ret = OB_SUCCESS; ObMysqlServerSession *selected_session = NULL; - if (client_session_->get_session_info().is_oceanbase_server()) { + if (OB_LIKELY(client_session_->get_session_info().is_oceanbase_server())) { if (OB_FAIL(do_oceanbase_internal_observer_open(selected_session))) { LOG_WARN("failed to do oceanbase internal observer open", K(ret)); } @@ -5045,10 +5589,10 @@ inline int ObMysqlSM::do_internal_observer_open() if (OB_SUCC(ret)) { // clear client session's bound_ss_ client_session_->attach_server_session(NULL); - if (NULL != selected_session) { + if (OB_LIKELY(NULL != selected_session)) { selected_session->state_ = MSS_ACTIVE; selected_session->set_client_session(*client_session_); - if (client_session_->can_direct_ok()) { + if (OB_UNLIKELY(client_session_->can_direct_ok())) { // sharding user maybe skip real connect, set flag now proxy::ObMysqlAuthRequest& login_req = client_session_->get_session_info().get_login_req(); ObMySQLCapabilityFlags& cap_flag = login_req.get_hsr_result().response_.get_capability_flags(); @@ -5062,7 +5606,7 @@ inline int ObMysqlSM::do_internal_observer_open() } if (OB_SUCC(ret)) { - if (NULL != server_session_) { + if (OB_LIKELY(NULL != server_session_)) { ObClientSessionInfo &client_info = client_session_->get_session_info(); ObServerSessionInfo &server_info = server_session_->get_session_info(); // ObSqlParseResult &sql_result = trans_state_.trans_info_.client_request_.get_parse_result(); @@ -5080,24 +5624,25 @@ inline int ObMysqlSM::do_internal_observer_open() K(server_session_->ss_id_), "client_session version", client_info.get_session_version(), "server version", server_info.get_session_var_version()); - if (OB_MYSQL_COM_STMT_CLOSE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) { + if (OB_UNLIKELY(OB_MYSQL_COM_STMT_CLOSE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_)) { // no need sync var on OB_MYSQL_COM_STMT_CLOSE trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_REQUEST; - } else if (client_info.need_reset_database(server_info)) { + } else if (OB_UNLIKELY(client_info.need_reset_database(server_info))) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_USE_DATABASE; - } else if (client_info.need_reset_session_vars(server_info)) { + } else if (OB_UNLIKELY(client_info.need_reset_session_vars(server_info))) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_SESSION_VARS; - } else if (client_info.need_reset_last_insert_id(server_info)) { + } else if (OB_UNLIKELY(client_info.need_reset_last_insert_id(server_info))) { // TODO: current version proxy parse can't judge last_insert_id exactly, // so we do not judge, whether sql_reuslt has_last_insert_id here trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_LAST_INSERT_ID; - } else if (trans_state_.is_hold_start_trans_) { + } else if (OB_UNLIKELY(trans_state_.is_hold_start_trans_)) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_START_TRANS; - } else if (((OB_MYSQL_COM_STMT_EXECUTE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) - || (OB_MYSQL_COM_STMT_SEND_PIECE_DATA == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_)) - && client_info.need_do_prepare(server_info)) { + } else if (OB_UNLIKELY(((OB_MYSQL_COM_STMT_EXECUTE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) + || (OB_MYSQL_COM_STMT_SEND_PIECE_DATA == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) + || (OB_MYSQL_COM_STMT_SEND_LONG_DATA == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_)) + && client_info.need_do_prepare(server_info))) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_PREPARE; - } else if (client_info.is_text_ps_execute() && client_info.need_do_text_ps_prepare(server_info)) { + } else if (OB_UNLIKELY(client_info.is_text_ps_execute() && client_info.need_do_text_ps_prepare(server_info))) { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_TEXT_PS_PREPARE; } else { trans_state_.current_.send_action_ = ObMysqlTransact::SERVER_SEND_REQUEST; @@ -5142,7 +5687,13 @@ inline int ObMysqlSM::connect_observer() // Set the inactivity timeout to the connect timeout so that we // we fail this server if it doesn't start sending the response // convert to ns - const int64_t connect_timeout = trans_state_.mysql_config_params_->short_async_task_timeout_; + ObShardProp *shard_prop = client_session_->get_session_info().get_shard_prop(); + int64_t connect_timeout = 0; + if (OB_NOT_NULL(shard_prop)) { + connect_timeout = HRTIME_MSECONDS(shard_prop->get_connect_timeout()); + } else { + connect_timeout = trans_state_.mysql_config_params_->short_async_task_timeout_; + } LOG_DEBUG("calling g_net_processor.connect", K_(sm_id), K(trans_state_.server_info_.addr_)); ret = g_net_processor.connect(*this, trans_state_.server_info_.addr_.sa_, @@ -5163,6 +5714,177 @@ inline int ObMysqlSM::connect_observer() return ret; } +int ObMysqlSM::do_internal_request_for_sharding_init_db(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + // handle use db stmt or OB_MYSQL_COM_INIT_DB cmd when sharding + LOG_DEBUG("sharding init db"); + ObHSRResult &hsr = client_info.get_login_req().get_hsr_result(); + ObString db_name; + if (client_request.get_parse_result().is_use_db_stmt()) { + db_name = client_request.get_parse_result().get_database_name(); + } else { + db_name = client_request.get_sql(); + } + if (OB_FAIL(ObProxyShardUtils::handle_shard_use_db(trans_state_, *client_session_, db_name))) { + if (OB_ENTRY_NOT_EXIST == ret) { + // response unknown db + trans_state_.mysql_errcode_ = OB_ERR_BAD_DATABASE; + trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_BAD_DATABASE); + if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to build err resp", K_(sm_id), K(ret)); + } + } else if (OB_ERR_NO_DB_PRIVILEGE == ret) { + if (OB_FAIL(ObMysqlTransact::build_no_privilege_message(trans_state_, *client_session_, db_name))) { + LOG_WARN("fail to build_no_privilege_message", K(db_name), K(ret)); + } + } else { + LOG_WARN("handle shard use db failed", K(ret)); + } + } else if (OB_FAIL(save_user_login_info(client_info, hsr))) { + LOG_WARN("fail to save user login info", K_(sm_id), K(ret)); + } else { + // if start trans is hold, we treat this resp is in trans + bool is_in_trans = (trans_state_.is_hold_start_trans_ || ObMysqlTransact::is_in_trans(trans_state_)); + if (OB_FAIL(ObMysqlResponseBuilder::build_ok_resq_with_state_changed( + *buf, client_request, client_info, is_in_trans))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to build ok res", K_(sm_id), K(ret)); + } + } + + return ret; +} + +int ObMysqlSM::do_internal_request_for_sharding_show_db_version(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + // handle show db version when sharding + LOG_DEBUG("sharding show db_version"); + ObString logic_tenant_name; + ObString logic_database_name; + if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { + LOG_WARN("fail to get_logic_tenant_name", K(ret)); + } else if (OB_FAIL(client_info.get_logic_database_name(logic_database_name))) { + LOG_WARN("fail to get_logic_database_name", K(ret)); + trans_state_.mysql_errcode_ = OB_ERR_NO_DB_SELECTED; + trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_NO_DB_SELECTED); + if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to build not err resp", K_(sm_id), K(ret)); + } + } else if (OB_FAIL(ObShowDBVersionHandler::show_db_version_cmd_callback(buf, + static_cast(client_request.get_packet_meta().pkt_seq_ + 1), + trans_state_.mysql_config_params_->internal_cmd_mem_limited_, + logic_tenant_name, logic_database_name))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show db version", K_(sm_id), K(ret)); + } + + return ret; +} + +int ObMysqlSM::do_internal_request_for_sharding_show_db(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + // handle show databases when sharding + LOG_DEBUG("sharding show databases"); + ObString logic_tenant_name; + if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { + LOG_WARN("fail to get_logic_tenant_name", K(ret)); + } else if (OB_FAIL(ObShowDatabasesHandler::show_databases_cmd_callback(buf, + static_cast(client_request.get_packet_meta().pkt_seq_ + 1), + trans_state_.mysql_config_params_->internal_cmd_mem_limited_, logic_tenant_name))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show databases", K_(sm_id), K(ret)); + } + + return ret; +} + +int ObMysqlSM::do_internal_request_for_sharding_show_table(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + // handle show tables when sharding + LOG_DEBUG("sharding show tables"); + ObString logic_tenant_name; + ObString logic_database_name; + if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { + LOG_WARN("fail to get_logic_tenant_name", K(ret)); + } else if (OB_FAIL(client_info.get_logic_database_name(logic_database_name))) { + LOG_WARN("fail to get_logic_database_name", K(ret)); + trans_state_.mysql_errcode_ = OB_ERR_NO_DB_SELECTED; + trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_NO_DB_SELECTED); + if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to build not err resp", K_(sm_id), K(ret)); + } + } else if (OB_FAIL(ObShowTablesHandler::show_tables_cmd_callback(buf, + static_cast(client_request.get_packet_meta().pkt_seq_ + 1), + trans_state_.mysql_config_params_->internal_cmd_mem_limited_, + logic_tenant_name, logic_database_name))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show databases", K_(sm_id), K(ret)); + } + + return ret; +} + +int ObMysqlSM::do_internal_request_for_sharding_show_topology(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + // handle show topology when sharding + ObInternalCmdInfo &cmd_info = *client_request.cmd_info_; + ObString logic_db_name = cmd_info.get_like_string(); + ObString logic_tenant_name; + ObString group_name = cmd_info.get_value_string(); + LOG_DEBUG("sharding show topology", K(logic_db_name), K(group_name)); + if (logic_db_name.empty()) { + if (OB_FAIL(client_info.get_logic_database_name(logic_db_name))) { + LOG_WARN("fail to get_logic_db_name", K(ret)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { + LOG_WARN("fail to get_logic_tenant_name", K(ret)); + } else if (OB_FAIL(ObShowTopologyHandler::show_topology_cmd_callback(buf, + static_cast(client_request.get_packet_meta().pkt_seq_ + 1), + trans_state_.mysql_config_params_->internal_cmd_mem_limited_, + logic_tenant_name, logic_db_name, group_name))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show topology", K_(sm_id), K(ret)); + } + + return ret; +} + +int ObMysqlSM::do_internal_request_for_sharding_select_db(ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info) +{ + int ret = OB_SUCCESS; + + ObString logic_database_name; + client_info.get_logic_database_name(logic_database_name); + if (OB_FAIL(ObSelectDatabaseHandler::select_database_cmd_callback(buf, + static_cast(client_request.get_packet_meta().pkt_seq_ + 1), + trans_state_.mysql_config_params_->internal_cmd_mem_limited_, + logic_database_name))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle select database()", K_(sm_id), K(ret)); + } + + return ret; +} + void ObMysqlSM::do_internal_request() { int ret = OB_SUCCESS; @@ -5191,7 +5913,11 @@ void ObMysqlSM::do_internal_request() switch (trans_state_.trans_info_.sql_cmd_) { case OB_MYSQL_COM_HANDSHAKE: { OMPKHandshake handshake; - if (g_ssl_processor.is_client_ssl_supported() && !client_session_->is_proxy_mysql_client_) { + if (!client_session_->is_proxy_mysql_client_ && + get_global_ssl_config_table_processor().is_ssl_supported( + client_session_->get_vip_cluster_name(), + client_session_->get_vip_tenant_name(), + true)) { handshake.enable_ssl(); } // current not support compress to client @@ -5449,129 +6175,47 @@ void ObMysqlSM::do_internal_request() } // 12. handle some special stmt or cmd when sharding - } else if (client_session_->get_session_info().is_sharding_user() && - !trans_state_.trans_info_.client_request_.get_parse_result().is_dual_request()) { - // handle use db stmt or OB_MYSQL_COM_INIT_DB cmd when sharding - if (OB_MYSQL_COM_INIT_DB == trans_state_.trans_info_.sql_cmd_ || client_request.get_parse_result().is_use_db_stmt()) { - LOG_DEBUG("init db"); - ObHSRResult &hsr = client_info.get_login_req().get_hsr_result(); - ObString db_name; - if (client_request.get_parse_result().is_use_db_stmt()) { - db_name = client_request.get_parse_result().get_database_name(); - } else { - db_name = client_request.get_sql(); - } - if (OB_FAIL(ObProxyShardUtils::handle_shard_use_db(trans_state_, *client_session_, db_name))) { - if (OB_ENTRY_NOT_EXIST == ret) { - // response unknown db - trans_state_.mysql_errcode_ = OB_ERR_BAD_DATABASE; - trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_BAD_DATABASE); - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to build err resp", K_(sm_id), K(ret)); - } - } else if (OB_ERR_NO_DB_PRIVILEGE == ret) { - if (OB_FAIL(ObMysqlTransact::build_no_privilege_message(trans_state_, *client_session_, db_name))) { - LOG_WARN("fail to build_no_privilege_message", K(db_name), K(ret)); - } - } else { - LOG_WARN("handle shard use db failed", K(ret)); - } - } else if (OB_FAIL(save_user_login_info(client_info, hsr))) { - LOG_WARN("fail to save user login info", K_(sm_id), K(ret)); - } else { - // if start trans is hold, we treat this resp is in trans - bool is_in_trans = (trans_state_.is_hold_start_trans_ || ObMysqlTransact::is_in_trans(trans_state_)); - if (OB_FAIL(ObMysqlResponseBuilder::build_ok_resq_with_state_changed( - *buf, client_request, client_info, is_in_trans))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to build ok res", K_(sm_id), K(ret)); - } - } - - } else if (client_request.get_parse_result().is_show_db_version_stmt()) { - // handle show db version when sharding - LOG_DEBUG("show db_version"); - ObString logic_tenant_name; - ObString logic_database_name; - if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { - LOG_WARN("fail to get_logic_tenant_name", K(ret)); - } else if (OB_FAIL(client_info.get_logic_database_name(logic_database_name))) { - LOG_WARN("fail to get_logic_database_name", K(ret)); - trans_state_.mysql_errcode_ = OB_ERR_NO_DB_SELECTED; - trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_NO_DB_SELECTED); - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to build not err resp", K_(sm_id), K(ret)); - } - } else if (OB_FAIL(ObShowDBVersionHandler::show_db_version_cmd_callback(buf, - static_cast(client_request.get_packet_meta().pkt_seq_ + 1), - trans_state_.mysql_config_params_->internal_cmd_mem_limited_, - logic_tenant_name, logic_database_name))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show db version", K_(sm_id), K(ret)); - } - } else if (client_request.get_parse_result().is_show_databases_stmt()) { - // handle show databases when sharding - LOG_DEBUG("show databases"); - ObString logic_tenant_name; - if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { - LOG_WARN("fail to get_logic_tenant_name", K(ret)); - } else if (OB_FAIL(ObShowDatabasesHandler::show_databases_cmd_callback(buf, - static_cast(client_request.get_packet_meta().pkt_seq_ + 1), - trans_state_.mysql_config_params_->internal_cmd_mem_limited_, logic_tenant_name))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show databases", K_(sm_id), K(ret)); - } - // handle show tables when sharding - } else if (client_request.get_parse_result().is_show_tables_stmt()) { - LOG_DEBUG("show tables"); - ObString logic_tenant_name; - ObString logic_database_name; - if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { - LOG_WARN("fail to get_logic_tenant_name", K(ret)); - } else if (OB_FAIL(client_info.get_logic_database_name(logic_database_name))) { - LOG_WARN("fail to get_logic_database_name", K(ret)); - trans_state_.mysql_errcode_ = OB_ERR_NO_DB_SELECTED; - trans_state_.mysql_errmsg_ = ob_strerror(OB_ERR_NO_DB_SELECTED); - if (OB_FAIL(ObMysqlTransact::build_error_packet(trans_state_))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to build not err resp", K_(sm_id), K(ret)); - } - } else if (OB_FAIL(ObShowTablesHandler::show_tables_cmd_callback(buf, - static_cast(client_request.get_packet_meta().pkt_seq_ + 1), - trans_state_.mysql_config_params_->internal_cmd_mem_limited_, - logic_tenant_name, logic_database_name))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show databases", K_(sm_id), K(ret)); - } - } else if (client_request.get_parse_result().is_show_topology_stmt()) { - // handle show topology when sharding - ObInternalCmdInfo &cmd_info = *client_request.cmd_info_; - ObString logic_db_name = cmd_info.get_like_string(); - ObString logic_tenant_name; - ObString group_name = cmd_info.get_value_string(); - LOG_DEBUG("show topology", K(logic_db_name), K(group_name)); - if (logic_db_name.empty()) { - if (OB_FAIL(client_info.get_logic_database_name(logic_db_name))) { - LOG_WARN("fail to get_logic_db_name", K(ret)); - } - } - if (OB_FAIL(ret)) { - // do nothing - } if (OB_FAIL(client_info.get_logic_tenant_name(logic_tenant_name))) { - LOG_WARN("fail to get_logic_tenant_name", K(ret)); - } else if (OB_FAIL(ObShowTopologyHandler::show_topology_cmd_callback(buf, - static_cast(client_request.get_packet_meta().pkt_seq_ + 1), - trans_state_.mysql_config_params_->internal_cmd_mem_limited_, - logic_tenant_name, logic_db_name, group_name))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle show topology", K_(sm_id), K(ret)); - } - } else if (client_request.get_parse_result().is_select_database_stmt()) { - ObString logic_database_name; - client_info.get_logic_database_name(logic_database_name); - if (OB_FAIL(ObSelectDatabaseHandler::select_database_cmd_callback(buf, - static_cast(client_request.get_packet_meta().pkt_seq_ + 1), - trans_state_.mysql_config_params_->internal_cmd_mem_limited_, - logic_database_name))) { - LOG_WARN("[ObMysqlSM::do_internal_request] fail to handle select database()", K_(sm_id), K(ret)); - } + } else if (client_session_->get_session_info().is_sharding_user() + && (OB_MYSQL_COM_INIT_DB == trans_state_.trans_info_.sql_cmd_ + || client_request.get_parse_result().is_use_db_stmt())) { + if (OB_FAIL(do_internal_request_for_sharding_init_db(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding init db", K_(sm_id), K(ret)); + } + } else if (client_session_->get_session_info().is_sharding_user() + && client_request.get_parse_result().is_show_db_version_stmt()) { + if (OB_FAIL(do_internal_request_for_sharding_show_db_version(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding show db version", K_(sm_id), K(ret)); + } + } else if (client_session_->get_session_info().is_sharding_user() + && client_request.get_parse_result().is_show_databases_stmt()) { + if (OB_FAIL(do_internal_request_for_sharding_show_db(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding show db", K_(sm_id), K(ret)); + } + } else if (client_session_->get_session_info().is_sharding_user() + && client_request.get_parse_result().is_show_tables_stmt()) { + if (OB_FAIL(do_internal_request_for_sharding_show_table(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding show db", K_(sm_id), K(ret)); + } + } else if (client_session_->get_session_info().is_sharding_user() + && client_request.get_parse_result().is_show_topology_stmt()) { + if (OB_FAIL(do_internal_request_for_sharding_show_topology(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding show db", K_(sm_id), K(ret)); + } + } else if (client_session_->get_session_info().is_sharding_user() + && client_request.get_parse_result().is_select_database_stmt()) { + if (OB_FAIL(do_internal_request_for_sharding_select_db(buf, client_request, client_info))) { + LOG_WARN("fail to do internal request for sharding show db", K_(sm_id), K(ret)); + } + // 13. select_proxy_version + } else if (client_request.get_parse_result().is_select_proxy_version()) { + // if start trans is hold, we treat this resp is in trans + bool is_in_trans = (trans_state_.is_hold_start_trans_ || ObMysqlTransact::is_in_trans(trans_state_)); + if (OB_FAIL(ObMysqlResponseBuilder::build_select_proxy_version_resp( + *buf, client_request, client_info, is_in_trans))) { + LOG_WARN("[ObMysqlSM::do_internal_request] fail to build select proxy_version", K_(sm_id), K(ret)); } - // 13. internal cmd + // 14. internal cmd } else { send_response_direct = false; if (OB_ISNULL(client_request.cmd_info_)) { @@ -5824,9 +6468,11 @@ void ObMysqlSM::handle_observer_open() LOG_DEBUG("[ObMysqlSM::handle_observer_open]", K_(sm_id), K(trans_state_.send_reqeust_direct_), "server_ip", trans_state_.server_info_.addr_); - milestones_.do_observer_open_end_ = get_based_hrtime(); - cmd_time_stats_.do_observer_open_time_ += milestone_diff(milestones_.do_observer_open_begin_, - milestones_.do_observer_open_end_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + milestones_.do_observer_open_end_ = get_based_hrtime(); + cmd_time_stats_.do_observer_open_time_ += milestone_diff(milestones_.do_observer_open_begin_, + milestones_.do_observer_open_end_); + } // applying per-transaction observer netVC options // here IFF they differ from the netVC's current @@ -5835,11 +6481,12 @@ void ObMysqlSM::handle_observer_open() // transaction. if (OB_LIKELY(NULL != server_session_)) { ObNetVConnection *vc = server_session_->get_netvc(); + trans_state_.server_info_.set_obproxy_addr(server_session_->get_netvc()->get_local_addr()); - if (OB_LIKELY(NULL != vc) && + if (OB_UNLIKELY(NULL != vc && (vc->options_.sockopt_flags_ != trans_state_.mysql_config_params_->sock_option_flag_out_ || vc->options_.packet_mark_ != trans_state_.mysql_config_params_->sock_packet_mark_out_ - || vc->options_.packet_tos_ != trans_state_.mysql_config_params_->sock_packet_tos_out_)) { + || vc->options_.packet_tos_ != trans_state_.mysql_config_params_->sock_packet_tos_out_))) { vc->options_.sockopt_flags_ = static_cast(trans_state_.mysql_config_params_->sock_option_flag_out_); vc->options_.packet_mark_ = static_cast(trans_state_.mysql_config_params_->sock_packet_mark_out_); vc->options_.packet_tos_ = static_cast(trans_state_.mysql_config_params_->sock_packet_tos_out_); @@ -5859,6 +6506,17 @@ void ObMysqlSM::handle_observer_open() ObMysqlTransact::update_sql_cmd(trans_state_); switch (trans_state_.current_.send_action_) { + case ObMysqlTransact::SERVER_SEND_REQUEST: + if (OB_UNLIKELY(need_setup_client_transfer())) { + if (OB_FAIL(setup_client_transfer(MYSQL_TRANSFORM_VC))) { + LOG_WARN("failed to setup_client_transfer", K_(sm_id), K(ret)); + } + } else { + skip_plugin_ = true; + callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_SEND_REQUEST); + } + break; + case ObMysqlTransact::SERVER_SEND_HANDSHAKE: // normally, we will set query_timeout before we send request, // but in SERVER_SEND_HANDSHAK nothing to be sent to observer, @@ -5870,16 +6528,6 @@ void ObMysqlSM::handle_observer_open() } break; - case ObMysqlTransact::SERVER_SEND_REQUEST: - if (need_setup_client_transfer()) { - if (OB_FAIL(setup_client_transfer(MYSQL_TRANSFORM_VC))) { - LOG_WARN("failed to setup_client_transfer", K_(sm_id), K(ret)); - } - } else { - callout_api_and_start_next_action(ObMysqlTransact::SM_ACTION_API_SEND_REQUEST); - } - break; - case ObMysqlTransact::SERVER_SEND_ALL_SESSION_VARS: //fall through case ObMysqlTransact::SERVER_SEND_USE_DATABASE: @@ -5919,12 +6567,13 @@ void ObMysqlSM::handle_observer_open() inline bool ObMysqlSM::need_setup_client_transfer() { bool need = false; - if (client_session_->get_session_info().is_oceanbase_server()) { + if (OB_LIKELY(client_session_->get_session_info().is_oceanbase_server())) { need = (!trans_state_.is_auth_request_ && trans_state_.trans_info_.request_content_length_ > 0 && (PROTOCOL_CHECKSUM == use_compression_protocol() || PROTOCOL_OB20 == use_compression_protocol() - || obmysql::OB_MYSQL_COM_STMT_PREPARE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) + || obmysql::OB_MYSQL_COM_STMT_PREPARE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_ + || obmysql::OB_MYSQL_COM_STMT_EXECUTE == trans_state_.trans_info_.client_request_.get_packet_meta().cmd_) && NULL != api_.do_request_transform_open()); if (need) { LOG_DEBUG("[need_setup_client_transfer] will setup client transfer", K_(sm_id)); @@ -6230,7 +6879,7 @@ int ObMysqlSM::attach_server_session(ObMysqlServerSession &s) LOG_ERROR("invalid server session state", K_(s.state), K_(sm_id), K(ret)); dump_history_state(); } else { - if (trans_state_.server_info_.addr_ != s.server_ip_) { + if (OB_UNLIKELY(trans_state_.server_info_.addr_ != s.server_ip_)) { trans_state_.server_info_.set_addr(s.get_netvc()->get_remote_addr()); } server_session_ = &s; @@ -6251,7 +6900,7 @@ int ObMysqlSM::attach_server_session(ObMysqlServerSession &s) server_entry_->vc_type_ = MYSQL_SERVER_VC; server_entry_->vc_handler_ = &ObMysqlSM::state_server_request_send; - if (client_session_->is_session_pool_client() && client_session_->is_proxy_mysql_client_) { + if (OB_UNLIKELY(client_session_->is_session_pool_client() && client_session_->is_proxy_mysql_client_)) { if (server_session_->get_session_info().get_ob_capability() == 0) { server_session_->get_session_info().set_ob_capability(client_session_->get_session_info().get_ob_capability()); LOG_DEBUG("set_ob_capability", K(client_session_->get_session_info().get_ob_capability())); @@ -6329,35 +6978,40 @@ int ObMysqlSM::setup_server_request_send() ObIOBufferReader *buf_start = NULL; int64_t request_len = 0; - int64_t build_server_request_begin = get_based_hrtime(); + int64_t build_server_request_begin = 0; + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + build_server_request_begin = get_based_hrtime(); + } if (OB_FAIL(ObMysqlTransact::build_server_request(trans_state_, buf_start, request_len))) { LOG_WARN("failed to build server request", K(buf_start), K(ret)); } else if (OB_ISNULL(buf_start) || OB_UNLIKELY(request_len <= 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid request buf", K(buf_start), K(request_len), K(ret)); } else { - int64_t build_server_request_end = get_based_hrtime(); - cmd_time_stats_.build_server_request_time_ += milestone_diff(build_server_request_begin, build_server_request_end); - //reset before send request - milestones_.server_.reset(); - cmd_time_stats_.server_request_write_time_ = 0; - cmd_time_stats_.server_response_read_time_ = 0; - cmd_time_stats_.plugin_decompress_response_time_ = 0; - cmd_time_stats_.server_response_analyze_time_ = 0; - cmd_time_stats_.ok_packet_trim_time_ = 0; - cmd_time_stats_.client_response_write_time_ = 0; - - MYSQL_INCREMENT_TRANS_STAT(SERVER_REQUESTS); - milestones_.server_.reset(); - cmd_size_stats_.server_request_bytes_ = request_len; - milestones_.server_.server_write_begin_ = get_based_hrtime(); - - if (0 == milestones_.server_first_write_begin_ - && (trans_state_.is_auth_request_ - || ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { - milestones_.server_first_write_begin_ = milestones_.server_.server_write_begin_; - cmd_time_stats_.prepare_send_request_to_server_time_ = - milestone_diff(milestones_.client_.client_begin_, milestones_.server_first_write_begin_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + int64_t build_server_request_end = get_based_hrtime(); + cmd_time_stats_.build_server_request_time_ += milestone_diff(build_server_request_begin, build_server_request_end); + //reset before send request + milestones_.server_.reset(); + cmd_time_stats_.server_request_write_time_ = 0; + cmd_time_stats_.server_response_read_time_ = 0; + cmd_time_stats_.plugin_decompress_response_time_ = 0; + cmd_time_stats_.server_response_analyze_time_ = 0; + cmd_time_stats_.ok_packet_trim_time_ = 0; + cmd_time_stats_.client_response_write_time_ = 0; + + MYSQL_INCREMENT_TRANS_STAT(SERVER_REQUESTS); + milestones_.server_.reset(); + cmd_size_stats_.server_request_bytes_ = request_len; + milestones_.server_.server_write_begin_ = get_based_hrtime(); + + if (0 == milestones_.server_first_write_begin_ + && (trans_state_.is_auth_request_ + || ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_)) { + milestones_.server_first_write_begin_ = milestones_.server_.server_write_begin_; + cmd_time_stats_.prepare_send_request_to_server_time_ = + milestone_diff(milestones_.client_.client_begin_, milestones_.server_first_write_begin_); + } } // if write_buffer_ is not NULL, when clean server entry, will free the write_buffer_. @@ -6365,7 +7019,7 @@ int ObMysqlSM::setup_server_request_send() server_entry_->write_buffer_ = (buf_start != client_buffer_reader_) ? buf_start->writer() : NULL; // set query_timeout to each request, ddl stmt will never timeout - if (trans_state_.trans_info_.client_request_.get_parse_result().is_ddl_stmt()) { + if (OB_UNLIKELY(trans_state_.trans_info_.client_request_.get_parse_result().is_ddl_stmt())) { cancel_server_query_timeout(); // if PL/SQL: // 1. if in trans, use trx timeout @@ -6373,8 +7027,8 @@ int ObMysqlSM::setup_server_request_send() // Attention: // 1. if send begin or start, think as in trans // 2. if autocommit = 0, first SQL think as not in trans - } else if (trans_state_.trans_info_.client_request_.get_parse_result().is_call_stmt() - || trans_state_.trans_info_.client_request_.get_parse_result().has_anonymous_block()) { + } else if (OB_UNLIKELY(trans_state_.trans_info_.client_request_.get_parse_result().is_call_stmt() + || trans_state_.trans_info_.client_request_.get_parse_result().has_anonymous_block())) { if (trans_state_.is_hold_start_trans_ || ObMysqlTransact::is_in_trans(trans_state_)) { set_server_trx_timeout(); } else { @@ -6384,9 +7038,9 @@ int ObMysqlSM::setup_server_request_send() set_server_query_timeout(); } - if (!trans_state_.is_auth_request_ + if (OB_UNLIKELY(!trans_state_.is_auth_request_ && ObMysqlTransact::SERVER_SEND_REQUEST == trans_state_.current_.send_action_ - && trans_state_.trans_info_.request_content_length_ > 0) { + && trans_state_.trans_info_.request_content_length_ > 0)) { if (NULL != api_.request_transform_info_.vc_) { if (OB_FAIL(api_.setup_transform_to_server_transfer())) { LOG_WARN("failed to setup_transform_to_server_transfer", K_(sm_id), K(ret)); @@ -6441,7 +7095,7 @@ int ObMysqlSM::setup_server_response_read() milestones_.server_.server_read_end_ = 0; // The tunnel from observer to client is now setup. Ready to read the response - if (NULL != trans_state_.cache_block_) { + if (OB_UNLIKELY(NULL != trans_state_.cache_block_)) { // use the cached block to read server response server_session_->read_buffer_->append_block_internal(trans_state_.cache_block_); trans_state_.cache_block_ = NULL; @@ -6676,10 +7330,10 @@ int ObMysqlSM::setup_server_transfer() ObMysqlResp &server_response = trans_state_.trans_info_.server_response_; ObIMysqlRespAnalyzer *analyzer = NULL; bool is_resultset = server_response.get_analyze_result().is_resultset_resp(); - if ((PROTOCOL_CHECKSUM == use_compression_protocol()) + if (OB_UNLIKELY((PROTOCOL_CHECKSUM == use_compression_protocol()) && (NULL != client_session_) // inner sql's compressed response has tranfer to normal mysql packet - && (!client_session_->is_proxy_mysql_client_)) { + && (!client_session_->is_proxy_mysql_client_))) { if (is_resultset) { LOG_ERROR("compress protocol's never reach here", K(is_resultset)); } @@ -6768,12 +7422,16 @@ int ObMysqlSM::setup_cmd_complete() LOG_DEBUG("transaction is complete, wait next transaction", K_(sm_id)); } else { is_in_trans_ = true; - update_cmd_stats(); - milestones_.cmd_reset(); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + update_cmd_stats(); + milestones_.cmd_reset(); + } LOG_DEBUG("still in transaction, wait next request", K_(sm_id)); } cmd_size_stats_.reset(); - cmd_time_stats_.reset(); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + cmd_time_stats_.reset(); + } trans_state_.reset(); // reset ps info if (NULL != server_session_) { @@ -7243,12 +7901,19 @@ inline void ObMysqlSM::get_monitor_error_info(int32_t &error_code, ObString &err inline void ObMysqlSM::update_monitor_log() { + ObMySQLCmd request_cmd = OB_MYSQL_COM_END; + if (trans_state_.is_auth_request_) { + request_cmd = trans_state_.trans_info_.sql_cmd_; + } else { + request_cmd = trans_state_.trans_info_.client_request_.get_packet_meta().cmd_; + } + if (OB_NOT_NULL(client_session_) && !client_session_->is_proxy_mysql_client_ && !client_session_->is_proxysys_tenant() - && OB_MYSQL_COM_HANDSHAKE != trans_state_.trans_info_.sql_cmd_ - && OB_MYSQL_COM_END != trans_state_.trans_info_.sql_cmd_ - && OB_MYSQL_COM_QUIT != trans_state_.trans_info_.sql_cmd_) { + && OB_MYSQL_COM_HANDSHAKE != request_cmd + && OB_MYSQL_COM_END != request_cmd + && OB_MYSQL_COM_QUIT != request_cmd) { int64_t slow_time_threshold = trans_state_.mysql_config_params_->slow_query_time_threshold_; int64_t query_digest_time_threshold = trans_state_.mysql_config_params_->query_digest_time_threshold_; bool is_error_resp = false; @@ -7260,7 +7925,7 @@ inline void ObMysqlSM::update_monitor_log() || (slow_time_threshold > 0 && slow_time_threshold < cmd_time_stats_.request_total_time_) || is_error_resp || get_global_proxy_config().enable_monitor_stat - || (get_global_proxy_config().enable_qos && OB_MYSQL_COM_QUERY == trans_state_.trans_info_.sql_cmd_) + || (get_global_proxy_config().enable_qos && OB_MYSQL_COM_QUERY == request_cmd) || (get_global_proxy_config().enable_prometheus && g_ob_prometheus_processor.is_inited())) { ObClientSessionInfo &cs_info = client_session_->get_session_info(); @@ -7318,7 +7983,7 @@ inline void ObMysqlSM::update_monitor_log() const ObString rpc_id = parse_result.get_rpc_id(); const char *database_type_str = ObProxyMonitorUtils::get_database_type_name(database_type); const ObString &logic_table_name = parse_result.get_origin_table_name(); - const char *sql_cmd = ObProxyParserUtils::get_sql_cmd_name(trans_state_.trans_info_.sql_cmd_); + const char *sql_cmd = ObProxyParserUtils::get_sql_cmd_name(request_cmd); // for compatible sharding mode bool is_enc_beyond_trust = false; @@ -7330,16 +7995,17 @@ inline void ObMysqlSM::update_monitor_log() const char *stmt_type_str = ""; ObString new_sql; - char new_sql_buf[PRINT_SQL_LEN] = "\0"; - if (OB_MYSQL_COM_QUERY == trans_state_.trans_info_.sql_cmd_ - || OB_MYSQL_COM_STMT_PREPARE == trans_state_.trans_info_.sql_cmd_ - || OB_MYSQL_COM_STMT_PREPARE_EXECUTE == trans_state_.trans_info_.sql_cmd_) { + const int32_t print_len = static_cast(get_global_proxy_config().digest_sql_length); + char new_sql_buf[print_len] = "\0"; + int32_t new_sql_len = 0; + if (OB_MYSQL_COM_QUERY == request_cmd + || OB_MYSQL_COM_STMT_PREPARE == request_cmd + || OB_MYSQL_COM_STMT_PREPARE_EXECUTE == request_cmd) { stmt_type_str = get_print_stmt_name(stmt_type); - const ObString &origin_sql = trans_state_.trans_info_.get_print_sql(); - int32_t new_sql_len = 0; + const ObString &origin_sql = trans_state_.trans_info_.get_print_sql(print_len); ObProxyMonitorUtils::sql_escape(origin_sql.ptr(), origin_sql.length(), - new_sql_buf, PRINT_SQL_LEN, new_sql_len); + new_sql_buf, print_len, new_sql_len); new_sql.assign_ptr(new_sql_buf, new_sql_len); } @@ -7376,7 +8042,7 @@ inline void ObMysqlSM::update_monitor_log() } if (get_global_proxy_config().enable_monitor_stat) { - if (OB_MYSQL_COM_LOGIN == trans_state_.trans_info_.sql_cmd_) { + if (OB_MYSQL_COM_LOGIN == request_cmd) { stmt_type = OBPROXY_T_LOGIN; } update_monitor_stats(logic_tenant_name, logic_database_name, @@ -7386,7 +8052,7 @@ inline void ObMysqlSM::update_monitor_log() // Temporarily only supports OB_MYSQL_COM_QUERY // TODO: support ps sql - if (get_global_proxy_config().enable_qos && OB_MYSQL_COM_QUERY == trans_state_.trans_info_.sql_cmd_) { + if (get_global_proxy_config().enable_qos && OB_MYSQL_COM_QUERY == request_cmd) { int32_t table_name_length = table_name.length(); ObString new_table_name; if (trans_stats_.is_in_testload_trans_ && !ObMysqlTransact::is_in_trans(trans_state_)) { @@ -7407,38 +8073,55 @@ inline void ObMysqlSM::update_monitor_log() hrtime_to_usec(cmd_time_stats_.request_total_time_)); } - SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - stmt_type, PROMETHEUS_REQUEST_COUNT, is_slow_query, is_error_resp); - SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - stmt_type, PROMETHEUS_REQUEST_TOTAL_TIME, - hrtime_to_usec(cmd_time_stats_.request_total_time_)); - SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - stmt_type, PROMETHEUS_SERVER_PROCESS_REQUEST_TIME, - hrtime_to_usec(cmd_time_stats_.server_process_request_time_)); - SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - stmt_type, PROMETHEUS_PREPARE_SEND_REQUEST_TIME, - hrtime_to_usec(cmd_time_stats_.prepare_send_request_to_server_time_)); - - NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - PROMETHEUS_REQUEST_BYTE, true, true, - cmd_size_stats_.client_request_bytes_); - NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - PROMETHEUS_REQUEST_BYTE, true, false, - cmd_size_stats_.server_request_bytes_); - NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - PROMETHEUS_REQUEST_BYTE, false, true, - cmd_size_stats_.client_response_bytes_); - NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, - cluster_name, tenant_name, database_name, - PROMETHEUS_REQUEST_BYTE, false, false, - cmd_size_stats_.server_response_bytes_); + if (client_session_->is_need_convert_vip_to_tname() + && !cs_info.is_sharding_user() + && !is_slow_query + && !is_error_resp) { + SQLMonitorInfo monitor_info; + monitor_info.request_count_ = 1; + monitor_info.request_total_time_ = cmd_time_stats_.request_total_time_; + monitor_info.server_process_request_time_ = cmd_time_stats_.server_process_request_time_; + monitor_info.prepare_send_request_to_server_time_ = cmd_time_stats_.prepare_send_request_to_server_time_; + self_ethread().thread_prometheus_->set_sql_monitor_info(tenant_name, cluster_name, monitor_info); + } else { + if (!client_session_->is_need_convert_vip_to_tname()) { + NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + PROMETHEUS_REQUEST_BYTE, true, true, + cmd_size_stats_.client_request_bytes_); + NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + PROMETHEUS_REQUEST_BYTE, true, false, + cmd_size_stats_.server_request_bytes_); + NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + PROMETHEUS_REQUEST_BYTE, false, true, + cmd_size_stats_.client_response_bytes_); + NET_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + PROMETHEUS_REQUEST_BYTE, false, false, + cmd_size_stats_.server_response_bytes_); + } else { + // This field is not used on the public cloud + stmt_type = OBPROXY_T_INVALID; + } + + SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + stmt_type, PROMETHEUS_REQUEST_COUNT, is_slow_query, is_error_resp); + SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + stmt_type, PROMETHEUS_REQUEST_TOTAL_TIME, + hrtime_to_usec(cmd_time_stats_.request_total_time_)); + SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + stmt_type, PROMETHEUS_SERVER_PROCESS_REQUEST_TIME, + hrtime_to_usec(cmd_time_stats_.server_process_request_time_)); + SQL_PROMETHEUS_STAT(logic_tenant_name, logic_database_name, + cluster_name, tenant_name, database_name, + stmt_type, PROMETHEUS_PREPARE_SEND_REQUEST_TIME, + hrtime_to_usec(cmd_time_stats_.prepare_send_request_to_server_time_)); + } } } } @@ -7557,6 +8240,7 @@ inline void ObMysqlSM::update_cmd_stats() LOG_INFO(log_head, "client_ip", trans_state_.client_info_.addr_, "server_ip", trans_state_.server_info_.addr_, + "obproxy_client_port", trans_state_.server_info_.obproxy_addr_, "server_trace_id", get_server_trace_id(), "route_type", get_route_type_string(trans_state_.pll_info_.route_.cur_chosen_route_type_), K(user_name), @@ -7577,6 +8261,7 @@ inline void ObMysqlSM::update_cmd_stats() LOG_WARN(log_head, "client_ip", trans_state_.client_info_.addr_, "server_ip", trans_state_.server_info_.addr_, + "obproxy_client_port", trans_state_.server_info_.obproxy_addr_, "server_trace_id", get_server_trace_id(), "route_type", get_route_type_string(trans_state_.pll_info_.route_.cur_chosen_route_type_), K(user_name), @@ -7599,6 +8284,7 @@ inline void ObMysqlSM::update_cmd_stats() OBPROXY_XF_LOG(INFO, xf_head, "client_ip", trans_state_.client_info_.addr_, "server_ip", trans_state_.server_info_.addr_, + "obproxy_client_port", trans_state_.server_info_.obproxy_addr_, "route_type", get_route_type_string(trans_state_.pll_info_.route_.cur_chosen_route_type_), K(user_name), K(tenant_name), @@ -7640,7 +8326,9 @@ inline void ObMysqlSM::update_cmd_stats() inline void ObMysqlSM::update_stats() { if (!is_updated_stat_) { - update_cmd_stats(); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + update_cmd_stats(); + } milestones_.trans_finish_ = get_based_hrtime(); @@ -7872,7 +8560,7 @@ inline void ObMysqlSM::set_next_state() case ObMysqlTransact::SM_ACTION_SERVER_READ: { trans_state_.source_ = ObMysqlTransact::SOURCE_OBSERVER; - if (NULL != api_.response_transform_info_.vc_) { + if (OB_UNLIKELY(NULL != api_.response_transform_info_.vc_)) { if (OB_FAIL(api_.setup_server_transfer_to_transform())) { LOG_WARN("failed to setup_server_transfer_to_transform", K_(sm_id), K(ret)); setup_error_transfer(); diff --git a/src/obproxy/proxy/mysql/ob_mysql_sm.h b/src/obproxy/proxy/mysql/ob_mysql_sm.h index 49027e49fdb6c9171944ceeb331dafee91005331..e30031e67c7edfc04952f7003469a973f22a7721 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_sm.h +++ b/src/obproxy/proxy/mysql/ob_mysql_sm.h @@ -44,6 +44,7 @@ #include "proxy/mysql/ob_mysql_tunnel.h" #include "proxy/mysql/ob_mysql_client_session.h" #include "proxy/mysql/ob_mysql_sm_time_stat.h" +#include "proxy/shard/obproxy_shard_ddl_cont.h" #include "obutils/ob_tenant_stat_struct.h" #include "engine/ob_proxy_operator_result.h" @@ -150,19 +151,26 @@ public: void add_history_entry(const char *fileline, const int event, const int32_t reentrant); int get_mysql_schedule(int event, void *data); + int setup_handle_shard_ddl(event::ObAction *action); + int state_handle_shard_ddl(int event, void *data); + int process_shard_ddl_result(ObShardDDLStatus *ddl_status); int setup_handle_execute_plan(); int state_handle_execute_plan(int event, void *data); int process_executor_result(engine::ObProxyResultResp *result_resp); int build_executor_resp(event::ObMIOBuffer *write_buf, uint8_t &seq, engine::ObProxyResultResp *result_resp); - int handle_shard_request(bool &need_direct_response_for_dml); + int handle_shard_request(bool &need_response_for_stmt, bool &need_wait_callback); int check_user_identity(const ObString &user_name, const ObString &tenant_name, const ObString &cluster_name); int save_user_login_info(ObClientSessionInfo &session_info, ObHSRResult &hsr_result); void analyze_mysql_request(ObMysqlAnalyzeStatus &status); + int analyze_login_request(ObRequestAnalyzeCtx &ctx, ObMysqlAnalyzeStatus &status); int analyze_ps_prepare_request(); int do_analyze_ps_prepare_request(const ObString &ps_sql); - int analyze_ps_execute_request(); + int analyze_ps_execute_request(bool is_large_request = false); + int do_analyze_ps_execute_request(ObPsIdEntry *entry, bool is_large_request); + int do_analyze_ps_execute_request_with_flag(ObPsIdEntry *entry); + int do_analyze_ps_execute_request_without_flag(ObPsIdEntry *entry); int analyze_text_ps_prepare_request(); int analyze_text_ps_execute_request(); int analyze_fetch_request(); @@ -170,11 +178,11 @@ public: int analyze_ps_prepare_execute_request(); bool need_setup_client_transfer(); bool check_connection_throttle(); + bool can_pass_white_list(); + bool check_vt_connection_throttle(); bool is_partition_table_route_supported(); bool is_pl_route_supported(); - int encode_throttle_message(); - int encode_unsupport_ps_message(); - int encode_unsupport_change_user_message(); + int encode_error_message(int err_code); int handle_saved_session_variables(); void print_mysql_complete_log(ObMysqlTunnelProducer *p); @@ -231,8 +239,10 @@ public: bool is_cloud_user() const; bool need_reject_user_login(const common::ObString &user, const common::ObString &tenant, - const bool has_full_username, + const bool has_tenant_username, + const bool has_cluster_username, const bool is_cloud_user) const; + inline void set_skip_plugin(const bool bvalue) { skip_plugin_ = bvalue; } public: static const int64_t OP_LOCAL_NUM = 32; static const int64_t MAX_SCATTER_LEN; @@ -269,7 +279,7 @@ public: ObMysqlCompressOB20Analyzer compress_ob20_analyzer_; ObMysqlRequestAnalyzer request_analyzer_; -private: +public: static uint32_t get_next_sm_id(); void remove_client_entry(); void remove_server_entry(); @@ -322,6 +332,24 @@ private: int do_normal_internal_observer_open(ObMysqlServerSession *&selected_session); int do_internal_observer_open(); void do_internal_request(); + int do_internal_request_for_sharding_init_db(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); + int do_internal_request_for_sharding_show_db_version(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); + int do_internal_request_for_sharding_show_db(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); + int do_internal_request_for_sharding_show_table(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); + int do_internal_request_for_sharding_show_topology(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); + int do_internal_request_for_sharding_select_db(event::ObMIOBuffer *buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &client_info); int connect_observer(); int setup_client_transfer(ObMysqlVCType to_vc_type); void handle_api_return(); @@ -434,6 +462,12 @@ private: bool is_in_trans_; int32_t retry_acquire_server_session_count_; int64_t start_acquire_server_session_time_; + bool skip_plugin_; + + // private functions + int handle_server_request_send_long_data(); + int do_analyze_ps_execute_request_with_remain_value(event::ObMIOBuffer *writer, int64_t read_avail, + int64_t param_type_pos); }; inline ObMysqlSM *ObMysqlSM::allocate() @@ -526,14 +560,19 @@ inline int64_t ObMysqlSM::get_query_timeout() { int64_t timeout = HRTIME_NSECONDS(trans_state_.mysql_config_params_->observer_query_timeout_delta_); if (OB_LIKELY(NULL != client_session_) && OB_LIKELY(NULL != server_session_)) { - int64_t hint_query_timeout = trans_state_.trans_info_.client_request_.get_parse_result().get_hint_query_timeout(); - // if the request contains query_timeout in hint, we use it - if (hint_query_timeout > 0) { - // the query timeout in hint is in microseconds(us), so convert it into nanoseconds - timeout += HRTIME_USECONDS(hint_query_timeout); + dbconfig::ObShardProp *shard_prop = client_session_->get_session_info().get_shard_prop(); + if (OB_NOT_NULL(shard_prop)) { + timeout = HRTIME_MSECONDS(shard_prop->get_socket_timeout()); } else { - // we do parse in trans now, so we can use query_timeout in anycase - timeout += client_session_->get_session_info().get_query_timeout(); + int64_t hint_query_timeout = trans_state_.trans_info_.client_request_.get_parse_result().get_hint_query_timeout(); + // if the request contains query_timeout in hint, we use it + if (hint_query_timeout > 0) { + // the query timeout in hint is in microseconds(us), so convert it into nanoseconds + timeout += HRTIME_USECONDS(hint_query_timeout); + } else { + // we do parse in trans now, so we can use query_timeout in anycase + timeout += client_session_->get_session_info().get_query_timeout(); + } } } else {} return timeout; @@ -620,7 +659,7 @@ inline void ObMysqlSM::callout_api_and_start_next_action( const ObMysqlTransact::ObStateMachineActionType api_next_action) { trans_state_.api_next_action_ = api_next_action; - if (hooks_set_) { + if (hooks_set_ && !skip_plugin_) { api_.do_api_callout_internal(); } else { handle_api_return(); @@ -639,11 +678,11 @@ inline ObHRTime ObMysqlSM::get_based_hrtime() return time; } -inline bool ObMysqlSM::is_causal_order_read_enabled() -{ - return trans_state_.mysql_config_params_->enable_causal_order_read_ - && NULL != client_session_ - && client_session_->get_session_info().is_safe_read_weak_supported(); +inline bool ObMysqlSM::is_causal_order_read_enabled() +{ + return trans_state_.mysql_config_params_->enable_causal_order_read_ + && NULL != client_session_ + && client_session_->get_session_info().is_safe_read_weak_supported(); } inline bool ObMysqlSM::need_print_trace_stat() const diff --git a/src/obproxy/proxy/mysql/ob_mysql_transact.cpp b/src/obproxy/proxy/mysql/ob_mysql_transact.cpp index 5ab6e3dcb47807637bba444c5793b606933673f3..3260763e11ab62e3dc074236afe2cfdae7659f7f 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_transact.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_transact.cpp @@ -33,7 +33,7 @@ #include "prometheus/ob_sql_prometheus.h" #include "rpc/obmysql/packet/ompk_prepare.h" #include "proxy/mysql/ob_cursor_struct.h" -#include "iocore/net/ob_ssl_processor.h" +#include "omt/ob_ssl_config_table_processor.h" #include "dbconfig/ob_proxy_db_config_info.h" #include "lib/encrypt/ob_encrypted_helper.h" #include "proxy/shard/obproxy_shard_utils.h" @@ -49,7 +49,7 @@ using namespace oceanbase::obproxy::net; using namespace oceanbase::obproxy::obutils; using namespace oceanbase::obproxy::prometheus; using namespace oceanbase::obproxy::dbconfig; - +using namespace oceanbase::obproxy::omt; namespace oceanbase { namespace obproxy @@ -76,8 +76,6 @@ void ObMysqlTransact::handle_error_jump(ObTransState &s) void ObMysqlTransact::bad_request(ObTransState &s) { LOG_INFO("[ObMysqlTransact::bad_request] parser marked request bad"); - s.client_request_time_ = get_based_hrtime(s); - // no error message send, just close client session and bound server session TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); } @@ -149,12 +147,6 @@ ObConsistencyLevel ObMysqlTransact::ObTransState::get_trans_consistency_level( if (INVALID_CONSISTENCY != pll_info_.route_.consistency_level_) { ret_level = pll_info_.route_.consistency_level_; } else { - ObString sql; - if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == trans_info_.client_request_.get_packet_meta().cmd_) { - cs_info.get_ps_sql(sql); - } else { - sql = trans_info_.client_request_.get_sql(); - } if (trans_info_.client_request_.get_parse_result().is_select_stmt()) { const ObConsistencyLevel sql_hint = trans_info_.client_request_.get_parse_result().get_hint_consistency_level(); const ObConsistencyLevel sys_var = static_cast(cs_info.get_read_consistency()); @@ -169,26 +161,32 @@ ObConsistencyLevel ObMysqlTransact::ObTransState::get_trans_consistency_level( } } if (common::WEAK == ret_level) { - if (!cs_info.is_read_weak_supported()) { + ObString sql; + if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == trans_info_.client_request_.get_packet_meta().cmd_) { + cs_info.get_ps_sql(sql); + } else { + sql = trans_info_.client_request_.get_sql(); + } + if (OB_UNLIKELY(!cs_info.is_read_weak_supported())) { ret_level = common::STRONG; PROXY_LOG(DEBUG, "ObServer do not support read weak, treat it as strong read", - "sql_hint", get_consistency_level_str(sql_hint), - "sys_var", get_consistency_level_str(sys_var), - "ret_level", get_consistency_level_str(ret_level), - "sql", trans_info_.client_request_.get_sql()); - } else if (is_for_update_sql(sql)) { + "sql_hint", get_consistency_level_str(sql_hint), + "sys_var", get_consistency_level_str(sys_var), + "ret_level", get_consistency_level_str(ret_level), + "sql", trans_info_.client_request_.get_sql()); + } else if (OB_UNLIKELY(is_for_update_sql(sql))) { ret_level = common::STRONG; PROXY_LOG(DEBUG, "For select .. for update sql, treat it as strong read", - "sql_hint", get_consistency_level_str(sql_hint), - "sys_var", get_consistency_level_str(sys_var), - "ret_level", get_consistency_level_str(ret_level), - "sql", sql); + "sql_hint", get_consistency_level_str(sql_hint), + "sys_var", get_consistency_level_str(sys_var), + "ret_level", get_consistency_level_str(ret_level), + "sql", sql); } else { PROXY_LOG(DEBUG, "current use weak read", - "sql_hint", get_consistency_level_str(sql_hint), - "sys_var", get_consistency_level_str(sys_var), - "ret_level", get_consistency_level_str(ret_level), - "sql", sql); + "sql_hint", get_consistency_level_str(sql_hint), + "sys_var", get_consistency_level_str(sys_var), + "ret_level", get_consistency_level_str(ret_level), + "sql", sql); } } } @@ -237,17 +235,18 @@ bool ObMysqlTransact::ObTransState::is_request_readonly_zone_support(ObClientSes void ObMysqlTransact::modify_request(ObTransState &s) { - if (s.sm_->client_session_->get_session_info().is_oceanbase_server() - && OB_ISNULL(s.sm_->sm_cluster_resource_) + if (OB_ISNULL(s.sm_->sm_cluster_resource_) + && OB_UNLIKELY(s.sm_->client_session_->get_session_info().is_oceanbase_server() && !s.sm_->client_session_->is_proxysys_tenant() && (s.mysql_config_params_->is_mysql_routing_mode() - || (OB_MYSQL_COM_HANDSHAKE != s.trans_info_.sql_cmd_))) { + || (OB_MYSQL_COM_HANDSHAKE != s.trans_info_.sql_cmd_)))) { LOG_WARN("[modify_request] cluster resource is NULL, will disconnect"); TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); } else { - s.client_request_time_ = get_based_hrtime(s); - TRANSACT_RETURN(SM_ACTION_API_READ_REQUEST, ObMysqlTransact::handle_request); + if (!s.trans_info_.client_request_.is_large_request()) { + s.sm_->set_skip_plugin(true); + } } } @@ -292,7 +291,8 @@ void ObMysqlTransact::acquire_cached_server_session(ObTransState &s) selected_session = lii_session; LOG_DEBUG("[ObMysqlTransact::acquire_cached_server_session] last_insert_id_session is alive, pick it"); - } else if (NULL != last_session && OB_LIKELY(!s.mysql_config_params_->is_random_routing_mode())) { + } else if (get_global_proxy_config().enable_cached_server + && NULL != last_session && OB_LIKELY(!s.mysql_config_params_->is_random_routing_mode())) { const int32_t ip = ops_ip4_addr_host_order(last_session->get_netvc()->get_remote_addr()); const int32_t port = static_cast(ops_ip_port_host_order(last_session->get_netvc()->get_remote_addr())); ObAddr last_addr; @@ -559,138 +559,151 @@ void ObMysqlTransact::handle_mysql_request(ObTransState &s) } } -void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) +void ObMysqlTransact::handle_ps_close(ObTransState &s) { int ret = OB_SUCCESS; - ObClientSessionInfo &cs_info = get_client_session_info(s); - if (cs_info.is_allow_use_last_session()) { - s.need_pl_lookup_ = need_pl_lookup(s); + ObMysqlClientSession *client_session = s.sm_->get_client_session(); + + // At the end of each request, server_entry_ and server_session_ will be placed, + // including the internal jump of the close command multiple times. + if (!client_session->is_first_handle_close_request()) { + s.sm_->release_server_session(); + // If it is not the first time to come in, it will also be judged according to the transaction status of the first time. + s.need_pl_lookup_ = s.need_pl_lookup_ && !client_session->is_in_trans_for_close_request(); } else { - s.need_pl_lookup_ = true; + // The first time you come in, record the previous transaction status + client_session->set_in_trans_for_close_request(is_in_trans(s)); } - if (cs_info.is_sharding_user() && OB_FAIL(ObProxyShardUtils::update_sys_read_consistency_if_need(cs_info))) { - LOG_WARN("fail to update_sys_read_consistency_if_need", K(ret)); - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); // disconnect - } else if (OB_UNLIKELY(need_server_session_lookup(s))) { - TRANSACT_RETURN(SM_ACTION_SERVER_ADDR_LOOKUP, handle_server_addr_lookup); - } else if (obmysql::OB_MYSQL_COM_STMT_CLOSE == s.trans_info_.sql_cmd_) { - ObMysqlClientSession *client_session = s.sm_->get_client_session(); + ObMysqlServerSession *last_session = client_session->get_server_session(); + ObMysqlServerSession *last_bound_session = client_session->get_last_bound_server_session(); - if (!client_session->is_first_handle_close_request()) { - s.sm_->release_server_session(); - // not first in, use trans state - s.need_pl_lookup_ = s.need_pl_lookup_ && !client_session->is_in_trans_for_close_request(); - } else { - // first in, record trans state - client_session->set_in_trans_for_close_request(is_in_trans(s)); - } - - ObMysqlServerSession *last_session = client_session->get_server_session(); - ObMysqlServerSession *last_bound_session = client_session->get_last_bound_server_session(); + /* If no routing is required, and it is the first time to come in, the following situations need to be disconnected: + * 1. last_session does not exist. + * 2. last_bound_session is not null. + */ + if (!s.need_pl_lookup_ && client_session->is_first_handle_close_request() + && (NULL == last_session || NULL != last_bound_session)) { + LOG_ERROR("[ObMysqlTransact::handle request] something is wrong, we have to disconnect", + "is_first_handle_close_request_", client_session->is_first_handle_close_request(), + KP(last_session), KP(last_bound_session)); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); + } else { + uint32_t client_ps_id = client_session->get_session_info().get_client_ps_id(); + bool is_need_send_close_cmd = false; + bool is_need_send_to_bound_ss = false; + ObIpEndpoint addr; - /* below case need disconnect: - * 1. last session not exist - * 2. last_bound_session not NULL - */ - if (!s.need_pl_lookup_ && client_session->is_first_handle_close_request() - && (NULL == last_session || NULL != last_bound_session)) { - LOG_ERROR("[ObMysqlTransact::handle request] something is wrong, we have to disconnect", - "is_first_handle_close_request_", client_session->is_first_handle_close_request(), - KP(last_session), KP(last_bound_session)); - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); - } else { - uint32_t client_ps_id = client_session->get_session_info().get_client_ps_id(); - bool is_need_send_close_cmd = false; - bool is_need_send_to_bound_ss = false; - ObIpEndpoint addr; - - // cursor_id greater equal 1 << 31L; ps_id less than 1 << 31L - if (client_ps_id >= (CURSOR_ID_START)) { - ObCursorIdAddr *cursor_id_addr = client_session->get_session_info().get_cursor_id_addr(client_ps_id); - if (NULL != cursor_id_addr) { - is_need_send_close_cmd = true; - addr = cursor_id_addr->get_addr(); - // if first in, send to bound_ss - if (!s.need_pl_lookup_ - && client_session->is_first_handle_close_request() - && addr == ObIpEndpoint(last_session->get_netvc()->get_remote_addr())) { - is_need_send_to_bound_ss = true; - } + // If greater than 1 << 31L, it means cursor_id; otherwise, it is ps_id + if (client_ps_id >= (CURSOR_ID_START)) { + ObCursorIdAddr *cursor_id_addr = client_session->get_session_info().get_cursor_id_addr(client_ps_id); + if (NULL != cursor_id_addr) { + is_need_send_close_cmd = true; + addr = cursor_id_addr->get_addr(); + // If no routing is required, and it is the first time, the bound_ss is sent first. + // If the server to be sent is bound_ss, the flag is set, and no migration is required later. + if (!s.need_pl_lookup_ + && client_session->is_first_handle_close_request() + && addr == ObIpEndpoint(last_session->get_netvc()->get_remote_addr())) { + is_need_send_to_bound_ss = true; } - } else { - ObPsIdAddrs *ps_id_addrs = client_session->get_session_info().get_ps_id_addrs(client_ps_id); - if (NULL != ps_id_addrs && 0 != ps_id_addrs->get_addrs().size()) { - is_need_send_close_cmd = true; - // if first in, send to bound_ss - if (!s.need_pl_lookup_ - && client_session->is_first_handle_close_request()) { - ObPsIdAddrs::ADDR_HASH_SET::iterator iter = ps_id_addrs->get_addrs().begin(); - ObPsIdAddrs::ADDR_HASH_SET::iterator iter_end = ps_id_addrs->get_addrs().end(); - - for (; iter != iter_end; iter++) { - if (iter->first == ObIpEndpoint(last_session->get_netvc()->get_remote_addr())) { - is_need_send_to_bound_ss = true; - addr = iter->first; - break; - } + } + } else { + ObPsIdAddrs *ps_id_addrs = client_session->get_session_info().get_ps_id_addrs(client_ps_id); + if (NULL != ps_id_addrs && 0 != ps_id_addrs->get_addrs().size()) { + is_need_send_close_cmd = true; + /* If no routing is required, and it is the first time to come in + * Then check if there is any need to send to bound_ss, if so, send it first, + * otherwise migrate the connection, and then migrate back + */ + if (!s.need_pl_lookup_ + && client_session->is_first_handle_close_request()) { + ObPsIdAddrs::ADDR_HASH_SET::iterator iter = ps_id_addrs->get_addrs().begin(); + ObPsIdAddrs::ADDR_HASH_SET::iterator iter_end = ps_id_addrs->get_addrs().end(); + + for (; iter != iter_end; iter++) { + if (iter->first == ObIpEndpoint(last_session->get_netvc()->get_remote_addr())) { + is_need_send_to_bound_ss = true; + addr = iter->first; + break; } } + } - if (!is_need_send_to_bound_ss) { - addr = ps_id_addrs->get_addrs().begin()->first; - } + if (!is_need_send_to_bound_ss) { + addr = ps_id_addrs->get_addrs().begin()->first; } } + } - if (is_need_send_close_cmd) { - // move bound_ss to last_bound_ss - ObMysqlServerSession *last_bound_session = client_session->get_last_bound_server_session(); - if (!s.need_pl_lookup_ - && !is_need_send_to_bound_ss - && NULL == last_bound_session) { - client_session->attach_server_session(NULL); - last_session->do_io_read(client_session, 0, NULL); - client_session->set_last_bound_server_session(last_session); - client_session->set_need_return_last_bound_ss(true); - } + if (is_need_send_close_cmd) { + // If no routing is required, and there is no last server session in the server to be sent, migrate + ObMysqlServerSession *last_bound_session = client_session->get_last_bound_server_session(); + if (!s.need_pl_lookup_ + && !is_need_send_to_bound_ss + && NULL == last_bound_session) { + client_session->attach_server_session(NULL); + last_session->do_io_read(client_session, 0, NULL); + client_session->set_last_bound_server_session(last_session); + client_session->set_need_return_last_bound_ss(true); + } - if (OB_SUCC(ret)) { - client_session->set_first_handle_close_request(false); - s.server_info_.set_addr(addr); - s.pll_info_.lookup_success_ = true; - start_access_control(s); - } - } else { - // move last_bound_ss to bound_ss - if (client_session->is_need_return_last_bound_ss()) { - if (NULL != last_bound_session) { - if (OB_FAIL(return_last_bound_server_session(client_session))) { - LOG_WARN("fail to return last bond server session", K(ret)); - } - } else { - LOG_WARN("[ObMysqlTransact::handle request] last bound session is NULL, we have to disconnect"); - ret = OB_ERR_UNEXPECTED; + if (OB_SUCC(ret)) { + client_session->set_first_handle_close_request(false); + s.server_info_.set_addr(addr); + s.pll_info_.lookup_success_ = true; + start_access_control(s); + } + } else { + // This indicates that session migration has occurred and needs to be migrated back + if (client_session->is_need_return_last_bound_ss()) { + if (NULL != last_bound_session) { + if (OB_FAIL(return_last_bound_server_session(client_session))) { + LOG_WARN("fail to return last bond server session", K(ret)); } + } else { + LOG_WARN("[ObMysqlTransact::handle request] last bound session is NULL, we have to disconnect"); + ret = OB_ERR_UNEXPECTED; } + } - if (OB_SUCC(ret)) { - if (!client_session->is_in_trans_for_close_request()) { - s.sm_->trans_state_.current_.state_ = ObMysqlTransact::TRANSACTION_COMPLETE; - } else { - s.sm_->trans_state_.current_.state_ = ObMysqlTransact::CMD_COMPLETE; - } - - // if close all, move to internal request and clear session cache - TRANSACT_RETURN(SM_ACTION_INTERNAL_REQUEST, handle_internal_request); + if (OB_SUCC(ret)) { + if (!client_session->is_in_trans_for_close_request()) { + s.sm_->trans_state_.current_.state_ = ObMysqlTransact::TRANSACTION_COMPLETE; } else { - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); + s.sm_->trans_state_.current_.state_ = ObMysqlTransact::CMD_COMPLETE; } + + // If the backend has not been executed or all closed, transfer the internal request and clear the relevant cache on the client session + TRANSACT_RETURN(SM_ACTION_INTERNAL_REQUEST, handle_internal_request); + } else { + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); } } - } else if (s.need_pl_lookup_) { - s.pll_info_.reset(); + } +} +void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) +{ + int ret = OB_SUCCESS; + ObClientSessionInfo &cs_info = get_client_session_info(s); + if (OB_LIKELY(cs_info.is_allow_use_last_session())) { + s.need_pl_lookup_ = need_pl_lookup(s); + } else { + s.need_pl_lookup_ = true; + } + + obmysql::ObMySQLCmd cmd = s.trans_info_.sql_cmd_; + + if (OB_UNLIKELY(cs_info.is_sharding_user()) + && OB_FAIL(ObProxyShardUtils::update_sys_read_consistency_if_need(cs_info))) { + LOG_WARN("fail to update_sys_read_consistency_if_need", K(ret)); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); // disconnect + } else if (OB_UNLIKELY(need_server_session_lookup(s))) { + TRANSACT_RETURN(SM_ACTION_SERVER_ADDR_LOOKUP, handle_server_addr_lookup); + } else if (OB_UNLIKELY(obmysql::OB_MYSQL_COM_STMT_CLOSE == cmd)) { + handle_ps_close(s); + } else if (OB_LIKELY(s.need_pl_lookup_)) { // if need pl lookup, we should extract pl info first if (OB_FAIL(extract_partition_info(s))) { LOG_WARN("fail to extract partition info", K(ret)); @@ -698,17 +711,17 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); // disconnect } else { int64_t addr = cs_info.get_obproxy_route_addr(); - if (0 != addr) { + if (OB_UNLIKELY(0 != addr)) { uint32_t ip = 0; uint16_t port = 0; get_ip_port_from_addr(addr, ip, port); s.server_info_.set_addr(ip, port); s.pll_info_.lookup_success_ = true; LOG_DEBUG("@obproxy_route_addr is set", "address", s.server_info_.addr_, K(addr)); - } else if (obmysql::OB_MYSQL_COM_STMT_FETCH == s.trans_info_.sql_cmd_ - || obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == s.trans_info_.sql_cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_FETCH == cmd + || obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == cmd) { ObCursorIdAddr *cursor_id_addr = NULL; - if (OB_FAIL(s.sm_->get_client_session()->get_session_info().get_cursor_id_addr(cursor_id_addr))) { + if (OB_FAIL(cs_info.get_cursor_id_addr(cursor_id_addr))) { LOG_WARN("fail to get client cursor id addr", K(ret)); if (OB_HASH_NOT_EXIST == ret) { handle_fetch_request(s); @@ -720,10 +733,15 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) s.pll_info_.lookup_success_ = true; LOG_DEBUG("succ to set cursor target addr", "address", s.server_info_.addr_, KPC(cursor_id_addr)); } - } else if (obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == s.trans_info_.sql_cmd_ - || obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == s.trans_info_.sql_cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == cmd + || obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == cmd + || obmysql::OB_MYSQL_COM_STMT_EXECUTE == cmd) { + /* use piece info to record the last server addr */ + /* send piece data/send long data/prepare execute/execute should send to the same server as chosen before */ ObPieceInfo *info = NULL; - if (OB_FAIL(s.sm_->get_client_session()->get_session_info().get_piece_info(info))) { + if (OB_FAIL(cs_info.get_piece_info(info))) { + // maybe the first time to send, ignore not exist err, choose path by pl lookup if (OB_HASH_NOT_EXIST == ret) { LOG_DEBUG("fail to get piece info from hash map", K(ret)); ret = OB_SUCCESS; @@ -743,7 +761,8 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) } else { s.server_info_.set_addr(info->get_addr()); s.pll_info_.lookup_success_ = true; - LOG_DEBUG("succ to set send piece data target addr", "address", s.server_info_.addr_, KPC(info)); + LOG_DEBUG("succ to set target addr for send piece/prepare execute/send long data", "address", + s.server_info_.addr_, KPC(info)); } } else if ((s.mysql_config_params_->is_mock_routing_mode() && !s.sm_->client_session_->is_proxy_mysql_client_) || s.mysql_config_params_->is_mysql_routing_mode()) { @@ -756,10 +775,10 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) LOG_DEBUG("mysql mode, test server is valid, just use it and skip pl lookup", "address", s.server_info_.addr_); } - } else if (OB_LIKELY(!s.mysql_config_params_->is_random_routing_mode()) - && !s.api_server_addr_set_ - && s.pll_info_.te_name_.is_all_dummy_table() - && cs_info.is_allow_use_last_session()) { + } else if (OB_UNLIKELY(!s.mysql_config_params_->is_random_routing_mode() + && !s.api_server_addr_set_ + && s.pll_info_.te_name_.is_all_dummy_table() + && cs_info.is_allow_use_last_session())) { acquire_cached_server_session(s); } // end of !is_in_test_mode @@ -773,9 +792,9 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) ObMysqlServerSession *last_session = s.sm_->client_session_->get_server_session(); if (OB_LIKELY(NULL != last_session)) { - if (obmysql::OB_MYSQL_COM_STMT_FETCH == s.trans_info_.sql_cmd_) { + if (obmysql::OB_MYSQL_COM_STMT_FETCH == cmd) { ObCursorIdAddr *cursor_id_addr = NULL; - if (OB_FAIL(s.sm_->get_client_session()->get_session_info().get_cursor_id_addr(cursor_id_addr))) { + if (OB_FAIL(cs_info.get_cursor_id_addr(cursor_id_addr))) { LOG_WARN("fail to get client cursor id addr", K(ret)); if (OB_HASH_NOT_EXIST == ret) { handle_fetch_request(s); @@ -794,9 +813,9 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) s.server_info_.set_addr(cursor_id_addr->get_addr()); s.pll_info_.lookup_success_ = true; } - } else if (obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == s.trans_info_.sql_cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == cmd) { ObCursorIdAddr *cursor_id_addr = NULL; - if (OB_FAIL(s.sm_->get_client_session()->get_session_info().get_cursor_id_addr(cursor_id_addr))) { + if (OB_FAIL(cs_info.get_cursor_id_addr(cursor_id_addr))) { LOG_WARN("fail to get client cursor id addr", K(ret)); if (OB_HASH_NOT_EXIST == ret) { handle_fetch_request(s); @@ -825,9 +844,10 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) s.server_info_.set_addr(last_session->get_netvc()->get_remote_addr()); s.pll_info_.lookup_success_ = true; } - } else if (obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == s.trans_info_.sql_cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == cmd) { ObPieceInfo *info = NULL; - if (OB_FAIL(s.sm_->get_client_session()->get_session_info().get_piece_info(info))) { + if (OB_FAIL(cs_info.get_piece_info(info))) { if (OB_HASH_NOT_EXIST == ret) { ret = OB_SUCCESS; // do nothing @@ -849,8 +869,8 @@ void ObMysqlTransact::handle_oceanbase_request(ObTransState &s) if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = build_error_packet(s)))) { LOG_WARN("fail to build err packet", K(tmp_ret)); } else { - LOG_WARN("send piece target server is not the trans server", - "send piece target server", info->get_addr(), + LOG_WARN("send piece/long data target server is not the trans server", + "target server", info->get_addr(), "trans server", ObIpEndpoint(last_session->get_netvc()->get_remote_addr())); } @@ -941,7 +961,7 @@ void ObMysqlTransact::handle_request(ObTransState &s) if (OB_UNLIKELY(MYSQL_PLUGIN_AS_INTERCEPT == s.sm_->api_.plugin_tunnel_type_)) { setup_plugin_request_intercept(s); - } else if (is_sequence_request(s)) { + } else if (OB_UNLIKELY(s.trans_info_.client_request_.is_sharding_user() && is_sequence_request(s))) { LOG_DEBUG("is a sequence request"); int ret = OB_SUCCESS; s.trans_info_.client_request_.get_parse_result().set_stmt_type(OBPROXY_T_ICMD_DUAL); @@ -952,41 +972,43 @@ void ObMysqlTransact::handle_request(ObTransState &s) } else { TRANSACT_RETURN(SM_ACTION_INTERNAL_REQUEST, handle_internal_request); } - } else if (is_internal_request(s)) { + } else if (OB_UNLIKELY(is_internal_request(s))) { if (s.trans_info_.client_request_.get_parse_result().is_internal_select()) { MYSQL_INCREMENT_TRANS_STAT(CLIENT_USE_LOCAL_SESSION_STATE_REQUESTS); } // so it's an internal request TRANSACT_RETURN(SM_ACTION_INTERNAL_REQUEST, handle_internal_request); } else { - if (obmysql::OB_MYSQL_COM_QUERY == s.trans_info_.sql_cmd_) { - ObProxyBasicStmtType stmt_type = s.trans_info_.client_request_.get_parse_result().get_stmt_type(); - switch (stmt_type) { - case OBPROXY_T_SELECT: - MYSQL_INCREMENT_TRANS_STAT(CLIENT_SELECT_REQUESTS); - break; - case OBPROXY_T_DELETE: - MYSQL_INCREMENT_TRANS_STAT(CLIENT_DELETE_REQUESTS); - break; - case OBPROXY_T_INSERT: - case OBPROXY_T_MERGE: - // fall through - case OBPROXY_T_REPLACE: - MYSQL_INCREMENT_TRANS_STAT(CLIENT_INSERT_REQUESTS); - break; - case OBPROXY_T_UPDATE: - MYSQL_INCREMENT_TRANS_STAT(CLIENT_UPDATE_REQUESTS); - break; - default: - MYSQL_INCREMENT_TRANS_STAT(CLIENT_OTHER_REQUESTS); - break; + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { + if (obmysql::OB_MYSQL_COM_QUERY == s.trans_info_.sql_cmd_) { + ObProxyBasicStmtType stmt_type = s.trans_info_.client_request_.get_parse_result().get_stmt_type(); + switch (stmt_type) { + case OBPROXY_T_SELECT: + MYSQL_INCREMENT_TRANS_STAT(CLIENT_SELECT_REQUESTS); + break; + case OBPROXY_T_DELETE: + MYSQL_INCREMENT_TRANS_STAT(CLIENT_DELETE_REQUESTS); + break; + case OBPROXY_T_INSERT: + case OBPROXY_T_MERGE: + // fall through + case OBPROXY_T_REPLACE: + MYSQL_INCREMENT_TRANS_STAT(CLIENT_INSERT_REQUESTS); + break; + case OBPROXY_T_UPDATE: + MYSQL_INCREMENT_TRANS_STAT(CLIENT_UPDATE_REQUESTS); + break; + default: + MYSQL_INCREMENT_TRANS_STAT(CLIENT_OTHER_REQUESTS); + break; + } } } if (OB_ISNULL(s.sm_->client_session_)) { LOG_WARN("[ObMysqlTransact::handle request] client session is NULL, we have to disconnect"); TRANSACT_RETURN(SM_ACTION_SEND_ERROR_NOOP, NULL); - } else if (is_session_memory_overflow(s)) { + } else if (OB_UNLIKELY(is_session_memory_overflow(s))) { s.mysql_errcode_ = OB_EXCEED_MEM_LIMIT; int tmp_ret = OB_SUCCESS; if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = build_error_packet(s)))) { @@ -998,10 +1020,10 @@ void ObMysqlTransact::handle_request(ObTransState &s) } s.current_.state_ = ObMysqlTransact::INTERNAL_ERROR; TRANSACT_RETURN(SM_ACTION_INTERNAL_NOOP, NULL); - } else if (!s.sm_->client_session_->get_session_info().is_oceanbase_server()) { - handle_mysql_request(s); - } else { + } else if (OB_LIKELY(s.sm_->client_session_->get_session_info().is_oceanbase_server())) { handle_oceanbase_request(s); + } else { + handle_mysql_request(s); }// end of NULL != s.sm_->client_session_ } } @@ -1013,10 +1035,10 @@ inline bool ObMysqlTransact::need_use_last_server_session(ObTransState &s) // 2. a func depend on last execute sql // 3. has already specified transaction characteristics (set transaction xxx), not commit yet, return (is_in_trans(s) - || (NULL != s.sm_->client_session_ && !s.sm_->client_session_->is_session_pool_client() - && s.trans_info_.client_request_.get_parse_result().has_dependent_func()) - || (NULL != s.sm_->client_session_ - && s.sm_->client_session_->get_session_info().is_trans_specified())); + || OB_UNLIKELY(NULL != s.sm_->client_session_ + && !s.sm_->client_session_->is_session_pool_client() + && (s.trans_info_.client_request_.get_parse_result().has_dependent_func() + || s.sm_->client_session_->get_session_info().is_trans_specified()))); } inline bool ObMysqlTransact::need_pl_lookup(ObTransState &s) @@ -1025,6 +1047,42 @@ inline bool ObMysqlTransact::need_pl_lookup(ObTransState &s) return !need_use_last_server_session(s); } +//reroute conditions: +// 1. Transaction first SQL +// 2. No reroute has occurred +// 3. It is not a large request, and the request is less than 4K +// (because the data in the read_buffer will be consumed again here, and the second routing will be copied from the 4K Buffer) +// 4. In case of EXECUTE or PREPARE_EXECUTE request, no pieceInfo structure exists +inline bool ObMysqlTransact::is_need_reroute(ObMysqlTransact::ObTransState &s) +{ + int ret = OB_SUCCESS; + int64_t total_request_packet_len = s.trans_info_.client_request_.get_packet_len(); + int64_t cached_request_packet_len = s.trans_info_.client_request_.get_req_pkt().length(); + bool is_need_reroute = false; + + is_need_reroute = s.mysql_config_params_->enable_reroute_ + && !s.is_rerouted_ && s.need_pl_lookup_ + && s.is_trans_first_request_ + && !s.trans_info_.client_request_.is_large_request() + && total_request_packet_len == cached_request_packet_len; + + if (is_need_reroute + && (obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == s.trans_info_.client_request_.get_packet_meta().cmd_ + || obmysql::OB_MYSQL_COM_STMT_EXECUTE == s.trans_info_.client_request_.get_packet_meta().cmd_)) { + ObPieceInfo *info = NULL; + ObClientSessionInfo &cs_info = s.sm_->client_session_->get_session_info(); + if (OB_FAIL(cs_info.get_piece_info(info))) { + if (OB_HASH_NOT_EXIST != ret) { + is_need_reroute = false; + } + } else { + is_need_reroute = false; + } + } + + return is_need_reroute; +} + int ObMysqlTransact::extract_partition_info(ObTransState &s) { int ret = OB_SUCCESS; @@ -1043,10 +1101,10 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) ObString database_name; // sys tenant just use __all_dummy entry, must not fetch table entry from remote - if (s.mysql_config_params_->is_mysql_routing_mode()) { + if (OB_UNLIKELY(s.mysql_config_params_->is_mysql_routing_mode())) { table_name = OB_ALL_DUMMY_TNAME; database_name = OB_SYS_DATABASE_NAME; - } else if (s.sm_->client_session_->is_proxy_mysql_client_) { + } else if (OB_UNLIKELY(s.sm_->client_session_->is_proxy_mysql_client_)) { if (tenant_name == OB_SYS_TENANT_NAME) { table_name = OB_ALL_DUMMY_TNAME; database_name = OB_SYS_DATABASE_NAME; @@ -1064,7 +1122,7 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) } } - if (OB_SUCC(ret) && table_name.empty() && database_name.empty()) { + if (OB_SUCC(ret) && OB_LIKELY(table_name.empty()) && OB_LIKELY(database_name.empty())) { ObSqlParseResult &parse_result = s.trans_info_.client_request_.get_parse_result(); table_name = parse_result.get_table_name(); @@ -1073,14 +1131,15 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) table_name = OB_ALL_DUMMY_TNAME; database_name = OB_SYS_DATABASE_NAME; - } else if (OB_UNLIKELY(table_name == OB_ALL_DUMMY_TNAME)) { + } else if (OB_UNLIKELY(table_name[0] == '_' && + table_name == OB_ALL_DUMMY_TNAME)) { // if table is __all_dummy, just set the db to oceanbase database_name = OB_SYS_DATABASE_NAME; } else { is_table_name_from_parser = true; database_name = parse_result.get_database_name(); - if (OB_UNLIKELY(database_name.empty())) { + if (OB_LIKELY(database_name.empty())) { if (OB_SUCCESS != cs_info.get_database_name(database_name)) { database_name = OB_SYS_DATABASE_NAME; } @@ -1094,7 +1153,7 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) } // if run here, means table name and db name all come from parse result - if (s.sm_->client_session_->get_session_info().is_oracle_mode()) { + if (OB_UNLIKELY(s.sm_->client_session_->get_session_info().is_oracle_mode())) { if (is_table_name_from_parser && OBPROXY_QUOTE_T_INVALID == parse_result.get_table_name_quote()) { string_to_upper_case(table_name.ptr(), table_name.length()); } @@ -1107,8 +1166,9 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) } } - if (is_need_use_sql_table_cache(s) - && table_name != OB_ALL_DUMMY_TNAME) { + if (OB_UNLIKELY(s.mysql_config_params_->enable_index_route_ + && is_need_use_sql_table_cache(s) + && table_name != OB_ALL_DUMMY_TNAME)) { ObString sql_id; // parse sql id if (OB_FAIL(ObMysqlRequestAnalyzer::analyze_sql_id(s.trans_info_.client_request_, sql_id))) { @@ -1170,7 +1230,7 @@ int ObMysqlTransact::extract_partition_info(ObTransState &s) // print info LOG_DEBUG("current extracted table entry name", "table_entry_name", te_name, K(ret)); - if (OB_FAIL(ret) || !te_name.is_valid()) { + if (OB_FAIL(ret) || OB_UNLIKELY(!te_name.is_valid())) { LOG_WARN("fail to extract_partition_info", K(te_name), K(ret)); te_name.reset(); } @@ -1189,7 +1249,6 @@ inline void ObMysqlTransact::setup_plugin_request_intercept(ObTransState &s) // We just want to write the request straight to the plugin s.current_.attempts_ = 1; - s.request_sent_time_ = get_based_hrtime(s); TRANSACT_RETURN(SM_ACTION_OBSERVER_OPEN, NULL); } @@ -1324,7 +1383,7 @@ inline void ObMysqlTransact::ObPartitionLookupInfo::pl_update_if_necessary(const const bool has_table_but_no_leader = (!route_.is_no_route_info_found() && !is_leader_existent()); if (is_leader != is_partition_hit || has_table_but_no_leader) { LOG_INFO("will set strong read route dirty", K(is_leader), K(is_partition_hit), K(has_table_but_no_leader), - "origin_name", te_name_, "route_info", route_); + "origin_name", te_name_, "route_info", route_); //NOTE:: if leader was congested from server, it will set_dirty in ObMysqlTransact::handle_congestion_control_lookup // here we only care response if (route_.set_target_dirty()) { @@ -1372,14 +1431,16 @@ void ObMysqlTransact::handle_congestion_control_lookup(ObTransState &s) { int ret = OB_SUCCESS; - s.sm_->milestones_.congestion_control_end_ = get_based_hrtime(s); - s.sm_->cmd_time_stats_.congestion_control_time_ += - milestone_diff(s.sm_->milestones_.congestion_control_begin_, s.sm_->milestones_.congestion_control_end_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + s.sm_->milestones_.congestion_control_end_ = get_based_hrtime(s); + s.sm_->cmd_time_stats_.congestion_control_time_ += + milestone_diff(s.sm_->milestones_.congestion_control_begin_, s.sm_->milestones_.congestion_control_end_); - s.sm_->milestones_.congestion_process_begin_ = s.sm_->milestones_.congestion_control_end_; - s.sm_->milestones_.congestion_process_end_ = 0; + s.sm_->milestones_.congestion_process_begin_ = s.sm_->milestones_.congestion_control_end_; + s.sm_->milestones_.congestion_process_end_ = 0; + } - if (!s.need_congestion_lookup_) { // no need congestion lookup + if (OB_LIKELY(!s.need_congestion_lookup_)) { // no need congestion lookup start_access_control(s); // continue } else { if (!s.congestion_lookup_success_) { // congestion entry lookup failed @@ -1471,9 +1532,11 @@ void ObMysqlTransact::handle_congestion_control_lookup(ObTransState &s) } } - s.sm_->milestones_.congestion_process_end_ = get_based_hrtime(s); - s.sm_->cmd_time_stats_.congestion_process_time_ += - milestone_diff(s.sm_->milestones_.congestion_process_begin_, s.sm_->milestones_.congestion_process_end_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + s.sm_->milestones_.congestion_process_end_ = get_based_hrtime(s); + s.sm_->cmd_time_stats_.congestion_process_time_ += + milestone_diff(s.sm_->milestones_.congestion_process_begin_, s.sm_->milestones_.congestion_process_end_); + } if (OB_FAIL(ret)) { s.inner_errcode_ = ret; @@ -1652,7 +1715,7 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) { int ret = OB_SUCCESS; ++s.pll_info_.pl_attempts_; - if (!s.pll_info_.lookup_success_) { // partition location lookup failed + if (OB_UNLIKELY(!s.pll_info_.lookup_success_)) { // partition location lookup failed ret = OB_ERR_UNEXPECTED; LOG_WARN("[ObMysqlTransact::handle_pl_lookup] " "fail to lookup partition location, will disconnect", K(ret)); @@ -1661,7 +1724,7 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) LOG_DEBUG("[ObMysqlTransact::handle_pl_lookup] Partition location Lookup successful", "pl_attempts", s.pll_info_.pl_attempts_); const bool is_server_addr_set = s.server_info_.addr_.is_valid(); - if (s.api_server_addr_set_ && !is_server_addr_set) { + if (OB_UNLIKELY(s.api_server_addr_set_ && !is_server_addr_set)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[ObMysqlTransact::handle_pl_lookup] " "observer ip/port doesn't set correctly, will disconnect"); @@ -1670,7 +1733,7 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) ret = OB_ERR_UNEXPECTED; LOG_WARN("[ObMysqlTransact::handle_pl_lookup] " "it should not arrive here, will disconnect", K(ret)); - } else if (is_server_addr_set && s.mysql_config_params_->is_mysql_routing_mode()) { + } else if (OB_UNLIKELY(is_server_addr_set && s.mysql_config_params_->is_mysql_routing_mode())) { LOG_DEBUG("[ObMysqlTransact::handle_pl_lookup] use mysql route mode and server addr is set, " "use cached session", "addr", s.server_info_.addr_, @@ -1687,8 +1750,10 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) s.sm_->cmd_time_stats_.debug_consistency_time_ += milestone_diff(t1, t2); t1 = t2; #endif + bool fill_addr = false; + bool use_dup_replica = need_use_dup_replica(consistency_level, s); // if not support safe_weak_read snapshot version, we should disable sort by priority - if (common::WEAK == consistency_level) { + if (OB_UNLIKELY(common::WEAK == consistency_level)) { if (!s.sm_->is_causal_order_read_enabled()) { LOG_DEBUG("safe weak read is disabled", "proxy_enable_causal_order_read", @@ -1700,103 +1765,120 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) LOG_DEBUG("safe weak read is enabled"); } } else { - // non weak read(login request included) do nothing: + if (common::STRONG == consistency_level && 1 == s.pll_info_.pl_attempts_ + && NULL != s.pll_info_.route_.table_entry_ + && !s.pll_info_.route_.table_entry_->is_dummy_entry() + && !use_dup_replica + && '\0' == s.mysql_config_params_->proxy_primary_zone_name_[0]) { + const ObProxyPartitionLocation *pl = s.pll_info_.route_.table_entry_->get_first_pl(); + for (int64_t i = 0; OB_SUCC(ret) && NULL != pl && i < pl->replica_count(); i++) { + const ObProxyReplicaLocation *replica = pl->get_replica(i); + if (NULL != replica && replica->is_leader()) { + fill_addr = true; + s.server_info_.set_addr(replica->server_.get_ipv4(), static_cast(replica->server_.get_port())); + s.pll_info_.route_.cur_chosen_server_.replica_ = replica; + s.pll_info_.route_.leader_item_.is_used_ = true; + s.pll_info_.route_.skip_leader_item_ = true; + break; + } + } + } } - bool use_dup_replica = need_use_dup_replica(consistency_level, s); - s.pll_info_.route_.need_use_dup_replica_ = use_dup_replica; - const bool disable_merge_status_check = need_disable_merge_status_check(s); - const ObRoutePolicyEnum route_policy = s.get_route_policy(*s.sm_->client_session_, use_dup_replica); - - ModulePageAllocator *allocator = NULL; - ObLDCLocation::get_thread_allocator(allocator); - - ObSEArray region_names(5, *allocator); - ObSEArray simple_servers_info( - ObServerStateRefreshCont::DEFAULT_SERVER_COUNT, *allocator); - get_region_name_and_server_info(s, simple_servers_info, region_names); - ObString proxy_primary_zone_name(s.mysql_config_params_->proxy_primary_zone_name_); - - if (OB_FAIL(s.pll_info_.route_.fill_replicas( - consistency_level, - route_policy, - disable_merge_status_check, - s.sm_->client_session_->dummy_entry_, - s.sm_->client_session_->dummy_ldc_, - simple_servers_info, - region_names, - proxy_primary_zone_name + if (!fill_addr) { + s.pll_info_.route_.need_use_dup_replica_ = use_dup_replica; + const bool disable_merge_status_check = need_disable_merge_status_check(s); + const ObRoutePolicyEnum route_policy = s.get_route_policy(*s.sm_->client_session_, use_dup_replica); + + ModulePageAllocator *allocator = NULL; + ObLDCLocation::get_thread_allocator(allocator); + + ObSEArray region_names(5, *allocator); + ObSEArray simple_servers_info( + ObServerStateRefreshCont::DEFAULT_SERVER_COUNT, *allocator); + get_region_name_and_server_info(s, simple_servers_info, region_names); + ObString proxy_primary_zone_name(s.mysql_config_params_->proxy_primary_zone_name_); + + if (OB_FAIL(s.pll_info_.route_.fill_replicas( + consistency_level, + route_policy, + disable_merge_status_check, + s.sm_->client_session_->dummy_entry_, + s.sm_->client_session_->dummy_ldc_, + simple_servers_info, + region_names, + proxy_primary_zone_name #if OB_DETAILED_SLOW_QUERY - ,s.sm_->cmd_time_stats_.debug_random_time_, - s.sm_->cmd_time_stats_.debug_fill_time_ + , + s.sm_->cmd_time_stats_.debug_random_time_, + s.sm_->cmd_time_stats_.debug_fill_time_ #endif - ))) { - LOG_WARN("fail to fill replicas", K(ret)); - } else if (is_server_addr_set) { - LOG_DEBUG("server addr is set, use cached session", - "addr", s.server_info_.addr_, - "attempts", s.current_.attempts_, - "sm_id", s.sm_->sm_id_); - } else { + ))) { + LOG_WARN("fail to fill replicas", K(ret)); + } else if (is_server_addr_set) { + LOG_DEBUG("server addr is set, use cached session", + "addr", s.server_info_.addr_, + "attempts", s.current_.attempts_, + "sm_id", s.sm_->sm_id_); + } else { #if OB_DETAILED_SLOW_QUERY - t2 = get_based_hrtime(s); - s.sm_->cmd_time_stats_.debug_total_fill_time_ += milestone_diff(t1, t2); - t1 = t2; + t2 = get_based_hrtime(s); + s.sm_->cmd_time_stats_.debug_total_fill_time_ += milestone_diff(t1, t2); + t1 = t2; #endif - int32_t attempt_count = 0; - bool found_leader_force_congested = false; - if (OB_UNLIKELY(s.mysql_config_params_->is_random_routing_mode()) - && (!s.sm_->client_session_->is_proxy_mysql_client_)) { - uint32_t last_ip = 0; - uint16_t last_port = 0; - ObNetVConnection *netvc = NULL; - if (NULL != s.sm_->client_session_->get_server_session() - && NULL != (netvc = s.sm_->client_session_->get_server_session()->get_netvc()) - && s.pll_info_.replica_size() > 1) { - last_ip = ntohl(netvc->get_remote_ip()); - last_port = netvc->get_remote_port(); - } - replica = s.pll_info_.get_next_replica(last_ip, last_port, s.force_retry_congested_); - } else { - replica = s.pll_info_.get_next_avail_replica(s.force_retry_congested_, attempt_count, - found_leader_force_congested); - s.current_.attempts_ += attempt_count; - if ((NULL == replica) && s.pll_info_.is_all_iterate_once()) { // next round - // if we has tried all servers, do force retry in next round - LOG_INFO("all replica is force_congested, force retry congested now", - "route", s.pll_info_.route_); - s.pll_info_.reset_cursor(); - s.force_retry_congested_ = true; + int32_t attempt_count = 0; + bool found_leader_force_congested = false; + if (OB_UNLIKELY(s.mysql_config_params_->is_random_routing_mode()) && (!s.sm_->client_session_->is_proxy_mysql_client_)) { + uint32_t last_ip = 0; + uint16_t last_port = 0; + ObNetVConnection *netvc = NULL; + if (NULL != s.sm_->client_session_->get_server_session() && NULL != (netvc = s.sm_->client_session_->get_server_session()->get_netvc()) && s.pll_info_.replica_size() > 1) { + last_ip = ntohl(netvc->get_remote_ip()); + last_port = netvc->get_remote_port(); + } + replica = s.pll_info_.get_next_replica(last_ip, last_port, s.force_retry_congested_); + } else { + replica = s.pll_info_.get_next_avail_replica(s.force_retry_congested_, attempt_count, + found_leader_force_congested); + s.current_.attempts_ += attempt_count; + if ((NULL == replica) && s.pll_info_.is_all_iterate_once()) { // next round + // if we has tried all servers, do force retry in next round + LOG_INFO("all replica is force_congested, force retry congested now", + "route", s.pll_info_.route_); + s.pll_info_.reset_cursor(); + s.force_retry_congested_ = true; - // get replica again - replica = s.pll_info_.get_next_avail_replica(); + // get replica again + replica = s.pll_info_.get_next_avail_replica(); + } } - } #if OB_DETAILED_SLOW_QUERY - t2 = get_based_hrtime(s); - s.sm_->cmd_time_stats_.debug_get_next_time_ += milestone_diff(t1, t2); - t1 = t2; + t2 = get_based_hrtime(s); + s.sm_->cmd_time_stats_.debug_get_next_time_ += milestone_diff(t1, t2); + t1 = t2; #endif - if (OB_ISNULL(replica)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("no replica avail", K(replica), K(ret)); - } else { - s.server_info_.set_addr(replica->server_.get_ipv4(), - static_cast(replica->server_.get_port())); - } + if (OB_ISNULL(replica)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no replica avail", K(replica), K(ret)); + } else { + s.server_info_.set_addr(replica->server_.get_ipv4(), + static_cast(replica->server_.get_port())); + } - if (found_leader_force_congested) { - ObProxyMutex *mutex_ = s.sm_->mutex_; // for stat - PROCESSOR_INCREMENT_DYN_STAT(UPDATE_ROUTE_ENTRY_BY_CONGESTION); - // when leader first router && (leader was congested from server or dead congested), - // we need set forceUpdateTableEntry to avoid senseless of leader migrate - if (s.pll_info_.set_target_dirty()) { - LOG_INFO("leader is force_congested in strong read, set it to dirty " - "and wait for updating", "addr", s.server_info_.addr_, - "route", s.pll_info_.route_); + if (found_leader_force_congested) { + ObProxyMutex *mutex_ = s.sm_->mutex_; // for stat + PROCESSOR_INCREMENT_DYN_STAT(UPDATE_ROUTE_ENTRY_BY_CONGESTION); + // when leader first router && (leader was congested from server or dead congested), + // we need set forceUpdateTableEntry to avoid senseless of leader migrate + if (s.pll_info_.set_target_dirty()) { + LOG_INFO("leader is force_congested in strong read, set it to dirty " + "and wait for updating", + "addr", s.server_info_.addr_, + "route", s.pll_info_.route_); + } } } } @@ -1807,9 +1889,11 @@ void ObMysqlTransact::handle_pl_lookup(ObTransState &s) "addr", s.server_info_.addr_, "attempts", s.current_.attempts_, "sm_id", s.sm_->sm_id_); - s.sm_->milestones_.pl_process_end_ = get_based_hrtime(s); - s.sm_->cmd_time_stats_.pl_process_time_ += - milestone_diff(s.sm_->milestones_.pl_process_begin_, s.sm_->milestones_.pl_process_end_); + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + s.sm_->milestones_.pl_process_end_ = get_based_hrtime(s); + s.sm_->cmd_time_stats_.pl_process_time_ += + milestone_diff(s.sm_->milestones_.pl_process_begin_, s.sm_->milestones_.pl_process_end_); + } TRANSACT_RETURN(SM_ACTION_CONGESTION_CONTROL_LOOKUP, handle_congestion_control_lookup); } } @@ -1918,6 +2002,7 @@ inline void ObMysqlTransact::start_access_control(ObTransState &s) { if (s.need_pl_lookup_) { TRANSACT_RETURN(SM_ACTION_API_OBSERVER_PL, ObMysqlTransact::lookup_skip_open_server); + s.sm_->set_skip_plugin(true); } else { lookup_skip_open_server(s); } @@ -1925,7 +2010,6 @@ inline void ObMysqlTransact::start_access_control(ObTransState &s) inline void ObMysqlTransact::lookup_skip_open_server(ObTransState &s) { - s.request_sent_time_ = get_based_hrtime(s); TRANSACT_RETURN(SM_ACTION_OBSERVER_OPEN, ObMysqlTransact::handle_response); } @@ -1952,18 +2036,20 @@ inline int ObMysqlTransact::build_user_request( int ObMysqlTransact::rewrite_stmt_id(ObTransState &s, ObIOBufferReader *client_buffer_reader) { int ret = OB_SUCCESS; + obmysql::ObMySQLCmd cmd = s.trans_info_.client_request_.get_packet_meta().cmd_; - // rewrite stmt id for ps execute - if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == s.trans_info_.client_request_.get_packet_meta().cmd_ - || obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == s.trans_info_.client_request_.get_packet_meta().cmd_) { + // rewrite stmt id for ps execute/send piece/send long data + if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == cmd) { ObServerSessionInfo &ss_info = get_server_session_info(s); ObClientSessionInfo &cs_info = get_client_session_info(s); uint32_t client_ps_id = cs_info.get_client_ps_id(); // get server_ps_id by client_ps_id uint32_t server_ps_id = ss_info.get_server_ps_id(client_ps_id); client_buffer_reader->replace(reinterpret_cast(&server_ps_id), sizeof(server_ps_id), MYSQL_NET_META_LENGTH); - } else if (obmysql::OB_MYSQL_COM_STMT_FETCH == s.trans_info_.client_request_.get_packet_meta().cmd_ - || obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == s.trans_info_.client_request_.get_packet_meta().cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_FETCH == cmd + || obmysql::OB_MYSQL_COM_STMT_GET_PIECE_DATA == cmd) { ObServerSessionInfo &ss_info = get_server_session_info(s); ObClientSessionInfo &cs_info = get_client_session_info(s); uint32_t client_cursor_id = cs_info.get_client_cursor_id(); @@ -1975,7 +2061,7 @@ int ObMysqlTransact::rewrite_stmt_id(ObTransState &s, ObIOBufferReader *client_b } else { client_buffer_reader->replace(reinterpret_cast(&server_cursor_id), sizeof(server_cursor_id), MYSQL_NET_META_LENGTH); } - } else if (obmysql::OB_MYSQL_COM_STMT_CLOSE == s.trans_info_.client_request_.get_packet_meta().cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_CLOSE == cmd) { ObServerSessionInfo &ss_info = get_server_session_info(s); ObClientSessionInfo &cs_info = get_client_session_info(s); uint32_t client_ps_id = cs_info.get_client_ps_id(); @@ -1990,7 +2076,7 @@ int ObMysqlTransact::rewrite_stmt_id(ObTransState &s, ObIOBufferReader *client_b } client_buffer_reader->replace(reinterpret_cast(&server_ps_id), sizeof(server_ps_id), MYSQL_NET_META_LENGTH); - } else if (obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == s.trans_info_.client_request_.get_packet_meta().cmd_) { + } else if (obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == cmd) { ObClientSessionInfo &cs_info = get_client_session_info(s); uint32_t recv_client_ps_id = cs_info.get_recv_client_ps_id(); uint32_t client_ps_id = cs_info.get_client_ps_id(); @@ -2308,7 +2394,7 @@ int ObMysqlTransact::build_server_request(ObTransState &s, ObIOBufferReader *&re break; } - if (OB_SUCC(ret) && NULL != build_func) { + if (OB_UNLIKELY(OB_SUCCESS == ret && NULL != build_func)) { ObMIOBuffer *write_buffer = NULL; if (OB_ISNULL(write_buffer = new_miobuffer(MYSQL_BUFFER_SIZE))) { @@ -2360,7 +2446,6 @@ int ObMysqlTransact::build_server_request(ObTransState &s, ObIOBufferReader *&re void ObMysqlTransact::handle_response(ObTransState &s) { s.source_ = SOURCE_OBSERVER; - s.response_received_time_ = get_based_hrtime(s); s.sm_->trans_stats_.server_responses_ += 1; handle_response_from_server(s); @@ -2404,9 +2489,10 @@ inline int ObMysqlTransact::do_handle_prepare_response(ObTransState &s, ObIOBuff compress_analyzer->reset(); const uint8_t req_seq = s.sm_->get_request_seq(); const ObMySQLCmd cmd = s.sm_->get_request_cmd(); + const ObMysqlProtocolMode mysql_mode = s.sm_->client_session_->get_session_info().is_oracle_mode() ? OCEANBASE_ORACLE_PROTOCOL_MODE : OCEANBASE_MYSQL_PROTOCOL_MODE; const bool enable_extra_ok_packet_for_stats = s.sm_->is_extra_ok_packet_for_stats_enabled(); if (OB_FAIL(compress_analyzer->init(req_seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, - cmd, enable_extra_ok_packet_for_stats, req_seq, + cmd, mysql_mode, enable_extra_ok_packet_for_stats, req_seq, s.sm_->get_server_session()->get_server_request_id(), s.sm_->get_server_session()->get_server_sessid()))) { LOG_WARN("fail to init compress analyzer", K(req_seq), K(cmd), K(enable_extra_ok_packet_for_stats), @@ -2973,7 +3059,16 @@ void ObMysqlTransact::handle_error_resp(ObTransState &s) switch(s.current_.send_action_) { case SERVER_SEND_HANDSHAKE: - s.current_.error_type_ = HANDSHAKE_COMMON_ERROR; + if (!s.sm_->client_session_->get_session_info().is_oceanbase_server()) { + if (OB_SUCCESS != s.sm_->get_client_buffer_reader()->consume_all()) { + s.current_.state_ = INTERNAL_ERROR; + LOG_WARN("fail to consume client buffer reader", "state", s.current_.state_); + } else { + is_user_request = true; + } + } else { + s.current_.error_type_ = HANDSHAKE_COMMON_ERROR; + } break; case SERVER_SEND_LOGIN: @@ -3065,6 +3160,8 @@ void ObMysqlTransact::handle_error_resp(ObTransState &s) "server_sessid", s.sm_->get_server_session()->get_server_sessid(), K(resp)); } + } else if (!s.sm_->client_session_->get_session_info().is_oceanbase_server()) { + is_user_request = true; } else { s.current_.error_type_ = SAVED_LOGIN_COMMON_ERROR; } @@ -3277,14 +3374,14 @@ inline void ObMysqlTransact::handle_resultset_resp(ObTransState &s) if (OB_UNLIKELY(obmysql::OB_MYSQL_COM_STMT_PREPARE == s.trans_info_.sql_cmd_ || SERVER_SEND_PREPARE == s.current_.send_action_)) { handle_prepare_succ(s); - } else if (OB_LIKELY(obmysql::OB_MYSQL_COM_STMT_EXECUTE == s.trans_info_.sql_cmd_)) { + } else if (OB_UNLIKELY(obmysql::OB_MYSQL_COM_STMT_EXECUTE == s.trans_info_.sql_cmd_)) { handle_execute_succ(s); - } else if (obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == s.trans_info_.sql_cmd_) { + } else if (OB_UNLIKELY(obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == s.trans_info_.sql_cmd_)) { handle_prepare_execute_succ(s); } // consume response of non-user request - if (SERVER_SEND_REQUEST != s.current_.send_action_) { + if (OB_UNLIKELY(SERVER_SEND_REQUEST != s.current_.send_action_)) { //LOG_WARN("handle_resultset_resp, not expected"); consume_response_packet(s); } @@ -3464,9 +3561,12 @@ inline int ObMysqlTransact::handle_oceanbase_handshake_pkt(ObTransState &s, uint // current proxy support use compress, and inner request also use compress bool use_compress = s.mysql_config_params_->enable_compression_protocol_; bool use_ob_protocol_v2 = s.mysql_config_params_->enable_ob_protocol_v2_; - bool use_ssl = s.trans_info_.server_response_.get_analyze_result().support_ssl() - && g_ssl_processor.is_server_ssl_supported() - && !s.sm_->client_session_->is_proxy_mysql_client_; + bool use_ssl = !s.sm_->client_session_->is_proxy_mysql_client_ + && s.trans_info_.server_response_.get_analyze_result().support_ssl() + && get_global_ssl_config_table_processor().is_ssl_supported( + s.sm_->client_session_->get_vip_cluster_name(), + s.sm_->client_session_->get_vip_cluster_name(), + false); ObRespAnalyzeResult &result = s.trans_info_.server_response_.get_analyze_result(); const ObString &server_scramble = result.get_scramble_string(); const ObString &proxy_scramble = client_session->get_scramble_string(); @@ -3855,18 +3955,20 @@ inline void ObMysqlTransact::handle_response_from_server(ObTransState &s) "error_type", get_server_resp_error_name(s.current_.error_type_), "server_ip", s.server_info_.addr_); - if (CONNECTION_ALIVE == s.current_.state_) { + if (OB_LIKELY(CONNECTION_ALIVE == s.current_.state_)) { handle_first_response_packet(s); } s.server_info_.state_ = s.current_.state_; - if (s.mysql_config_params_->enable_trans_detail_stats_) { + if (OB_UNLIKELY(s.mysql_config_params_->enable_trans_detail_stats_)) { update_sync_session_stat(s); } handle_server_failed(s); - update_trace_stat(s); + if (OB_UNLIKELY(get_global_performance_params().enable_stat_)) { + update_trace_stat(s); + } switch (s.current_.state_) { case CONNECTION_ALIVE: LOG_DEBUG("[ObMysqlTransact::handle_response_from_server] connection alive"); @@ -4272,14 +4374,18 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) switch (s.current_.send_action_) { case SERVER_SEND_HANDSHAKE: { - if (s.mysql_config_params_->is_mysql_routing_mode()) { + ObRespAnalyzeResult &resp = s.trans_info_.server_response_.get_analyze_result(); + if (resp.is_error_resp() || s.mysql_config_params_->is_mysql_routing_mode()) { s.next_action_ = SM_ACTION_SERVER_READ; - s.sm_->api_.do_response_transform_open(); + if (get_client_session_info(s).is_oceanbase_server()) { + s.sm_->api_.do_response_transform_open(); + } } else { - ObMysqlResp &server_response = s.trans_info_.server_response_; - bool server_support_ssl = server_response.get_analyze_result().support_ssl(); - LOG_DEBUG("ssl support", K(g_ssl_processor.is_server_ssl_supported()), K(server_support_ssl)); - if (g_ssl_processor.is_server_ssl_supported() && server_support_ssl + bool server_support_ssl = resp.support_ssl(); + bool is_server_ssl_supported = get_global_ssl_config_table_processor().is_ssl_supported( + s.sm_->client_session_->get_vip_cluster_name(), s.sm_->client_session_->get_vip_tenant_name(), false); + LOG_DEBUG("ssl support", K(is_server_ssl_supported), K(server_support_ssl)); + if (is_server_ssl_supported && server_support_ssl && !s.sm_->client_session_->is_proxy_mysql_client_) { s.current_.send_action_ = SERVER_SEND_SSL_REQUEST; s.next_action_ = SM_ACTION_API_SEND_REQUEST; @@ -4306,7 +4412,14 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) case SERVER_SEND_SAVED_LOGIN: { ObClientSessionInfo &client_info = get_client_session_info(s); - if (client_info.is_oceanbase_server()) { + ObRespAnalyzeResult &resp = s.trans_info_.server_response_.get_analyze_result(); + if (resp.is_error_resp()) { + s.next_action_ = SM_ACTION_SERVER_READ; + if (client_info.is_oceanbase_server()) { + s.sm_->api_.do_response_transform_open(); + } + break; + } else if (client_info.is_oceanbase_server()) { bool is_proxy_mysql_client_ = s.sm_->client_session_->is_proxy_mysql_client_; if (client_info.need_reset_all_session_vars() && !is_proxy_mysql_client_) { // 1.if global vars changed, we need to sync all session vars to keep @@ -4324,6 +4437,7 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) // fall through: } } + __attribute__ ((fallthrough)); case SERVER_SEND_ALL_SESSION_VARS: if (OB_LIKELY(NULL != s.sm_->client_session_) && OB_LIKELY(NULL != s.sm_->get_server_session())) { @@ -4351,6 +4465,7 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) K(s.sm_->client_session_), "server_session", s.sm_->get_server_session()); break; } + __attribute__ ((fallthrough)); case SERVER_SEND_USE_DATABASE: if (OB_LIKELY(NULL != s.sm_->client_session_) && OB_LIKELY(NULL != s.sm_->get_server_session())) { ObClientSessionInfo &client_info = get_client_session_info(s); @@ -4372,6 +4487,7 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) K(s.sm_->client_session_), "server_session", s.sm_->get_server_session()); break; } + __attribute__ ((fallthrough)); case SERVER_SEND_SESSION_VARS: case SERVER_SEND_LAST_INSERT_ID: case SERVER_SEND_PREPARE: @@ -4380,6 +4496,7 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) if (OB_LIKELY(NULL != s.sm_->client_session_) && OB_LIKELY(NULL != s.sm_->get_server_session())) { ObClientSessionInfo &client_info = get_client_session_info(s); ObServerSessionInfo &server_info = get_server_session_info(s); + obmysql::ObMySQLCmd cmd = s.trans_info_.client_request_.get_packet_meta().cmd_; //obutils::ObSqlParseResult &sql_result = s.trans_info_.client_request_.get_parse_result(); if (client_info.need_reset_last_insert_id(server_info)) { // TODO: current version proxy parse can't judge last_insert_id exactly, @@ -4389,8 +4506,9 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) s.current_.send_action_ = SERVER_SEND_LAST_INSERT_ID; } else if (s.is_hold_start_trans_) { s.current_.send_action_ = SERVER_SEND_START_TRANS; - } else if (((obmysql::OB_MYSQL_COM_STMT_EXECUTE == s.trans_info_.client_request_.get_packet_meta().cmd_) || - (obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == s.trans_info_.client_request_.get_packet_meta().cmd_)) + } else if ((obmysql::OB_MYSQL_COM_STMT_EXECUTE == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_PIECE_DATA == cmd + || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == cmd) && client_info.need_do_prepare(server_info)) { s.current_.send_action_ = SERVER_SEND_PREPARE; } else if (client_info.is_text_ps_execute() && client_info.need_do_text_ps_prepare(server_info)) { @@ -4442,7 +4560,7 @@ void ObMysqlTransact::handle_on_forward_server_response(ObTransState &s) } // just print info - if (SM_ACTION_SERVER_READ == s.next_action_) { + if (OB_LIKELY(SM_ACTION_SERVER_READ == s.next_action_)) { LOG_DEBUG("[ObMysqlTransact::handle_on_forward_server_response]" "start send response, wait for reading request", "next_action", ObMysqlTransact::get_action_name(s.next_action_)); @@ -4816,12 +4934,15 @@ int ObMysqlTransact::build_error_packet(ObTransState &s) } break; } + case OB_GET_LOCATION_TIME_OUT: + case OB_CLUSTER_NOT_EXIST: case OB_ERR_OPERATOR_UNKNOWN: case OB_ERR_TOO_MANY_SESSIONS: case OB_EXCEED_MEM_LIMIT: case OB_UNSUPPORTED_PS: case OB_ERR_FETCH_OUT_SEQUENCE: case OB_CURSOR_NOT_EXIST: + case OB_ERR_CON_COUNT_ERROR: case OB_NOT_SUPPORTED: { if (OB_FAIL(ObMysqlPacketUtil::encode_err_packet(*buf, next_seq, errcode))) { LOG_WARN("[ObMysqlTransact::build_error_packet] fail to build not supported err resp", K(ret)); @@ -4921,7 +5042,7 @@ inline ObRoutePolicyEnum ObMysqlTransact::ObTransState::get_route_policy(ObMysql } else { const ObString value = get_global_proxy_config().proxy_route_policy.str(); ObProxyRoutePolicyEnum policy = get_proxy_route_policy(value); - PROXY_CS_LOG(INFO, "succ to global variable proxy_route_policy", + PROXY_CS_LOG(DEBUG, "succ to global variable proxy_route_policy", "policy", get_proxy_route_policy_enum_string(policy)); get_route_policy(policy, ret_policy); } diff --git a/src/obproxy/proxy/mysql/ob_mysql_transact.h b/src/obproxy/proxy/mysql/ob_mysql_transact.h index b029538409f89f0ac799e301f15f6d84e0b965a0..76b827926ff2055cfb476459fc557caa500c3069 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_transact.h +++ b/src/obproxy/proxy/mysql/ob_mysql_transact.h @@ -224,6 +224,7 @@ public: { ObConnectionAttributes() : addr_(), + obproxy_addr_(), state_(STATE_UNDEFINED), abort_(ABORT_UNDEFINED) { @@ -235,11 +236,13 @@ public: void set_addr(const uint32_t ipv4, const uint16_t port) { net::ops_ip_copy(addr_, ipv4, port); } void set_addr(const sockaddr &sa) { net::ops_ip_copy(addr_, sa); } void set_addr(const net::ObIpEndpoint &ip_point) { net::ops_ip_copy(addr_, ip_point.sa_); } + void set_obproxy_addr(const sockaddr &sa) { net::ops_ip_copy(obproxy_addr_, sa); } uint32_t get_ipv4() { return net::ops_ip4_addr_host_order(addr_.sa_); } uint16_t get_port() { return net::ops_ip_port_host_order(addr_); } net::ObIpEndpoint addr_; // use function below to get/set ip and port + net::ObIpEndpoint obproxy_addr_; ObServerStateType state_; ObAbortStateType abort_; @@ -248,6 +251,7 @@ public: state_ = STATE_UNDEFINED; abort_ = ABORT_UNDEFINED; addr_.reset(); + obproxy_addr_.reset(); } private: @@ -441,7 +445,7 @@ public: sql_cmd_ = obmysql::OB_MYSQL_COM_END; } - common::ObString get_print_sql() + common::ObString get_print_sql(const int64_t sql_len = PRINT_SQL_LEN) { common::ObString ret; switch (sql_cmd_) { @@ -452,7 +456,7 @@ public: ret = common::ObString::make_string("OB_MYSQL_COM_LOGIN"); break; default: - ret = client_request_.get_print_sql(); + ret = client_request_.get_print_sql(sql_len); break; } return ret; @@ -492,9 +496,6 @@ public: internal_reader_(NULL), reroute_info_(), pll_info_(), - client_request_time_(0), - request_sent_time_(0), - response_received_time_(0), mysql_errcode_(0), mysql_errmsg_(NULL), inner_errcode_(0), @@ -557,13 +558,15 @@ public: ObMysqlTransact::handle_new_config_acquired(*this); } - if (NULL != sqlaudit_record_queue_) { - sqlaudit_record_queue_->refcount_dec(); - sqlaudit_record_queue_ = NULL; - } + if (OB_UNLIKELY(get_global_performance_params().enable_trace_)) { + if (NULL != sqlaudit_record_queue_) { + sqlaudit_record_queue_->refcount_dec(); + sqlaudit_record_queue_ = NULL; + } - if (mysql_config_params_->sqlaudit_mem_limited_ > 0) { - sqlaudit_record_queue_ = get_global_sqlaudit_processor().acquire(); + if (mysql_config_params_->sqlaudit_mem_limited_ > 0) { + sqlaudit_record_queue_ = get_global_sqlaudit_processor().acquire(); + } } } @@ -621,7 +624,7 @@ public: int alloc_internal_buffer(const int64_t buffer_block_size) { int ret = common::OB_SUCCESS; - if (NULL == internal_buffer_) { + if (OB_UNLIKELY(NULL == internal_buffer_)) { if (OB_ISNULL(internal_buffer_ = event::new_empty_miobuffer(buffer_block_size))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; PROXY_TXN_LOG(ERROR, "failed to new miobuffer", K(ret)); @@ -649,7 +652,7 @@ public: void reset_internal_buffer() { - if (NULL != internal_buffer_) { + if (OB_LIKELY(NULL != internal_buffer_)) { internal_buffer_->dealloc_all_readers(); cache_block_ = internal_buffer_->writer_; if (NULL != cache_block_) { @@ -667,7 +670,7 @@ public: void free_internal_buffer() { - if (NULL != internal_buffer_) { + if (OB_LIKELY(NULL != internal_buffer_)) { free_miobuffer(internal_buffer_); internal_buffer_ = NULL; internal_reader_ = NULL; @@ -792,10 +795,6 @@ public: ObProxyRerouteInfo reroute_info_; ObPartitionLookupInfo pll_info_; - ObHRTime client_request_time_; // internal - ObHRTime request_sent_time_; // internal - ObHRTime response_received_time_; // internal - // used to building error packet, which will be sent to client int mysql_errcode_; const char *mysql_errmsg_; @@ -834,6 +833,7 @@ public: static void handle_mysql_request(ObTransState &s); static int set_server_ip_by_shard_conn(ObTransState &s, dbconfig::ObShardConnector* shard_conn); static void handle_oceanbase_request(ObTransState &s); + static void handle_ps_close(ObTransState &s); static void handle_fetch_request(ObTransState &s); static void handle_request(ObTransState &s); static int build_normal_login_request(ObTransState &s, event::ObIOBufferReader *&reader, @@ -975,21 +975,6 @@ public: static bool is_need_use_sql_table_cache(ObMysqlTransact::ObTransState &s); }; -// conditions for reroute: -// 1. first SQL in transaction -// 2. no reroute -// 3. not large request and request size less than 4K -inline bool ObMysqlTransact::is_need_reroute(ObMysqlTransact::ObTransState &s) -{ - int64_t total_request_packet_len = s.trans_info_.client_request_.get_packet_len(); - int64_t cached_request_packet_len = s.trans_info_.client_request_.get_req_pkt().length(); - return s.mysql_config_params_->enable_reroute_ - && !s.is_rerouted_ && s.need_pl_lookup_ - && s.is_trans_first_request_ - && !s.trans_info_.client_request_.is_large_request() - && total_request_packet_len == cached_request_packet_len; -} - inline bool ObMysqlTransact::is_need_use_sql_table_cache(ObMysqlTransact::ObTransState &s) { obutils::ObSqlParseResult &parse_result = s.trans_info_.client_request_.get_parse_result(); @@ -1106,7 +1091,7 @@ inline void ObMysqlTransact::update_stat( s.current_stats_->stats_[*next_insert].increment_ = increment; ++(*next_insert); } - // ignore error + // ignore error } inline int64_t milestone_diff(const ObHRTime start, const ObHRTime end) @@ -1167,7 +1152,7 @@ inline void ObMysqlTransact::update_sql_cmd(ObTransState &s) break; case SERVER_SEND_REQUEST: - if (s.is_auth_request_) { + if (OB_UNLIKELY(s.is_auth_request_)) { s.trans_info_.sql_cmd_ = obmysql::OB_MYSQL_COM_LOGIN; } else { s.trans_info_.sql_cmd_ = s.trans_info_.client_request_.get_packet_meta().cmd_; diff --git a/src/obproxy/proxy/mysql/ob_mysql_tunnel.cpp b/src/obproxy/proxy/mysql/ob_mysql_tunnel.cpp index 02211463b2f0ab553991ee33baf07d3116f56c21..7cdcf90b40b5a2373db58cfba2863dc2f3b34b5e 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_tunnel.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_tunnel.cpp @@ -239,13 +239,19 @@ inline ObMysqlTunnelProducer *ObMysqlTunnel::alloc_producer() ObMysqlTunnelProducer *ret = NULL; if (OB_LIKELY(num_producers_ < MAX_PRODUCERS)) { - for (int64_t i = 0; i < MAX_PRODUCERS && NULL == ret; ++i) { - if (NULL == producers_[i].vc_) { - ++num_producers_; - ret = producers_ + i; + if (OB_LIKELY(NULL == producers_[0].vc_)) { + ++num_producers_; + ret = producers_; + } else { + for (int64_t i = 1; i < MAX_PRODUCERS && NULL == ret; ++i) { + if (OB_LIKELY(NULL == producers_[i].vc_)) { + ++num_producers_; + ret = producers_ + i; + } } } } + return ret; } @@ -254,13 +260,19 @@ inline ObMysqlTunnelConsumer *ObMysqlTunnel::alloc_consumer() ObMysqlTunnelConsumer *ret = NULL; if (OB_LIKELY(num_consumers_ < MAX_CONSUMERS)) { - for (int64_t i = 0; i < MAX_CONSUMERS && NULL == ret; ++i) { - if (NULL == consumers_[i].vc_) { - ++num_consumers_; - ret = consumers_ + i; + if (OB_LIKELY(NULL == consumers_[0].vc_)) { + ++num_consumers_; + ret = consumers_; + } else { + for (int64_t i = 1; i < MAX_CONSUMERS && NULL == ret; ++i) { + if (OB_LIKELY(NULL == consumers_[i].vc_)) { + ++num_consumers_; + ret = consumers_ + i; + } } } } + return ret; } @@ -297,7 +309,7 @@ ObMysqlTunnelProducer *ObMysqlTunnel::add_producer( } } - if (OB_SUCC(ret) && MYSQL_TUNNEL_STATIC_PRODUCER == vc && OB_UNLIKELY(0 != ntodo)) { + if (OB_UNLIKELY(OB_SUCCESS == ret && MYSQL_TUNNEL_STATIC_PRODUCER == vc && 0 != ntodo)) { ret = OB_ERR_SYS; LOG_ERROR("add_producer, invalid static producer ntodo", K(nbytes_arg), K(read_avail), K(ntodo), K(ret)); @@ -390,9 +402,8 @@ int ObMysqlTunnel::tunnel_run(ObMysqlTunnelProducer *p_arg) int ret = OB_SUCCESS; LOG_DEBUG("tunnel_run started", K_(sm_->sm_id), K(p_arg)); - if (NULL != p_arg) { - ret = producer_run(*p_arg); - if (OB_FAIL(ret)) { + if (OB_LIKELY(NULL != p_arg)) { + if (OB_FAIL(producer_run(*p_arg))) { LOG_WARN("failed to run producer", K(ret)); } } else { @@ -418,7 +429,7 @@ int ObMysqlTunnel::tunnel_run(ObMysqlTunnelProducer *p_arg) // due to a all transfers being zero length // If that is the case, call the state machine // back to say we are done - if (!is_tunnel_alive()) { + if (OB_UNLIKELY(!is_tunnel_alive())) { active_ = false; sm_->handle_event(MYSQL_TUNNEL_EVENT_DONE, this); ret = OB_SUCCESS; // tunnel complete @@ -1096,25 +1107,25 @@ int ObMysqlTunnel::main_handler(int event, void *data) LOG_DEBUG("mysql tunnel main_handler", K_(sm_->sm_id), "event", ObMysqlDebugNames::get_event_name(event), K(data)); - if (MYSQL_SM_MAGIC_ALIVE != sm_->magic_) { + if (OB_UNLIKELY(MYSQL_SM_MAGIC_ALIVE != sm_->magic_)) { LOG_WARN("failed to check sm magic", K_(sm_->magic), "expected", MYSQL_SM_MAGIC_ALIVE); } // Find the appropriate entry if (NULL != (p = get_producer(reinterpret_cast(data)))) { sm_callback = producer_handler(event, *p); - if (!sm_callback) { + if (OB_UNLIKELY(get_global_performance_params().enable_trace_ && !sm_callback)) { current_time = get_based_hrtime(); p->cost_time_ += (current_time - last_handler_event_time_); last_handler_event_time_ = current_time; } } else { - if (NULL != (c = get_consumer(reinterpret_cast(data)))) { + if (OB_LIKELY(NULL != (c = get_consumer(reinterpret_cast(data))))) { if (c->write_vio_ != data) { LOG_WARN("failed to check vio, data must equal to write vio", K_(c->write_vio), K(data)); } sm_callback = consumer_handler(event, *c); - if (!sm_callback) { + if (OB_UNLIKELY(get_global_performance_params().enable_trace_ && !sm_callback)) { current_time = get_based_hrtime(); c->cost_time_ += (current_time - last_handler_event_time_); last_handler_event_time_ = current_time; diff --git a/src/obproxy/proxy/mysql/ob_mysql_tunnel.h b/src/obproxy/proxy/mysql/ob_mysql_tunnel.h index 2878f99e157f1626914701b5da3cb385ce1aca5d..aae24652c48757f6b39648c382b5dc33ed11a074 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_tunnel.h +++ b/src/obproxy/proxy/mysql/ob_mysql_tunnel.h @@ -293,15 +293,17 @@ inline int ObMysqlTunnel::chain_finish_all(ObMysqlTunnelProducer &p) inline bool ObMysqlTunnel::is_tunnel_alive() const { - bool tunnel_alive = false; + bool tunnel_alive = producers_[0].alive_; - for (int64_t i = 0; i < MAX_PRODUCERS && !tunnel_alive; ++i) { - if (producers_[i].alive_) { - tunnel_alive = true; + if (OB_UNLIKELY(!tunnel_alive)) { + for (int64_t i = 1; i < MAX_PRODUCERS && !tunnel_alive; ++i) { + if (producers_[i].alive_) { + tunnel_alive = true; + } } } - if (!tunnel_alive) { + if (OB_UNLIKELY(!tunnel_alive)) { for (int64_t i = 0; i < MAX_CONSUMERS && !tunnel_alive; ++i) { if (consumers_[i].alive_) { tunnel_alive = true; @@ -315,10 +317,15 @@ inline bool ObMysqlTunnel::is_tunnel_alive() const inline ObMysqlTunnelProducer *ObMysqlTunnel::get_producer(event::ObVConnection *vc) { ObMysqlTunnelProducer *ret = NULL; - for (int64_t i = 0; i < MAX_PRODUCERS && NULL == ret; ++i) { - if (producers_[i].vc_ == vc) { - ret = producers_ + i; + if (OB_LIKELY(producers_[0].vc_ == vc)) { + ret = producers_; + } else { + for (int64_t i = 1; i < MAX_PRODUCERS && NULL == ret; ++i) { + if (producers_[i].vc_ == vc) { + ret = producers_ + i; + } } + } return ret; } @@ -326,9 +333,13 @@ inline ObMysqlTunnelProducer *ObMysqlTunnel::get_producer(event::ObVConnection * inline ObMysqlTunnelConsumer *ObMysqlTunnel::get_consumer(event::ObVConnection *vc) { ObMysqlTunnelConsumer *ret = NULL; - for (int64_t i = 0; i < MAX_CONSUMERS && NULL == ret; ++i) { - if (consumers_[i].vc_ == vc) { - ret = consumers_ + i; + if (OB_LIKELY(consumers_[0].vc_ == vc)) { + ret = consumers_; + } else { + for (int64_t i = 1; i < MAX_CONSUMERS && NULL == ret; ++i) { + if (consumers_[i].vc_ == vc) { + ret = consumers_ + i; + } } } return ret; @@ -337,9 +348,13 @@ inline ObMysqlTunnelConsumer *ObMysqlTunnel::get_consumer(event::ObVConnection * inline ObMysqlTunnelProducer *ObMysqlTunnel::get_producer(event::ObVIO *vio) { ObMysqlTunnelProducer *ret = NULL; - for (int64_t i = 0; i < MAX_PRODUCERS && NULL == ret; ++i) { - if (producers_[i].read_vio_ == vio) { - ret = producers_ + i; + if (OB_LIKELY(producers_[0].read_vio_ == vio)) { + ret = producers_; + } else { + for (int64_t i = 1; i < MAX_PRODUCERS && NULL == ret; ++i) { + if (producers_[i].read_vio_ == vio) { + ret = producers_ + i; + } } } return ret; @@ -348,9 +363,13 @@ inline ObMysqlTunnelProducer *ObMysqlTunnel::get_producer(event::ObVIO *vio) inline ObMysqlTunnelConsumer *ObMysqlTunnel::get_consumer(event::ObVIO *vio) { ObMysqlTunnelConsumer *ret = NULL; - for (int64_t i = 0; i < MAX_CONSUMERS && NULL == ret; ++i) { - if (consumers_[i].write_vio_ == vio) { - ret = consumers_ + i; + if (OB_LIKELY(consumers_[0].write_vio_ == vio)) { + ret = consumers_; + } else { + for (int64_t i = 1; i < MAX_CONSUMERS && NULL == ret; ++i) { + if (consumers_[i].write_vio_ == vio) { + ret = consumers_ + i; + } } } return ret; diff --git a/src/obproxy/proxy/mysql/ob_mysql_vctable.cpp b/src/obproxy/proxy/mysql/ob_mysql_vctable.cpp index e6c3159b7700e00ecb3068c8fef5ab7bd32b4d96..3ca48ee0f9ca374c16b44d502f0e6d3652383c0d 100644 --- a/src/obproxy/proxy/mysql/ob_mysql_vctable.cpp +++ b/src/obproxy/proxy/mysql/ob_mysql_vctable.cpp @@ -78,11 +78,11 @@ int ObMysqlVCTable::remove_entry(ObMysqlVCTableEntry *e) } else { e->vc_ = NULL; e->eos_ = false; - if (NULL != e->read_buffer_) { + if (OB_UNLIKELY(NULL != e->read_buffer_)) { free_miobuffer(e->read_buffer_); e->read_buffer_ = NULL; } - if (NULL != e->write_buffer_) { + if (OB_UNLIKELY(NULL != e->write_buffer_)) { free_miobuffer(e->write_buffer_); e->write_buffer_ = NULL; } diff --git a/src/obproxy/proxy/mysql/ob_prepare_statement_struct.cpp b/src/obproxy/proxy/mysql/ob_prepare_statement_struct.cpp index 55accde7830f479ff74db45b503ef5b8279498c0..3453ee2c5057d1936c262b5d3602772df880ecef 100644 --- a/src/obproxy/proxy/mysql/ob_prepare_statement_struct.cpp +++ b/src/obproxy/proxy/mysql/ob_prepare_statement_struct.cpp @@ -14,6 +14,8 @@ #include "proxy/mysql/ob_prepare_statement_struct.h" #include "iocore/eventsystem/ob_buf_allocator.h" #include "proxy/mysqllib/ob_proxy_mysql_request.h" +#include "proxy/mysqllib/ob_proxy_session_info.h" +#include "proxy/mysql/ob_mysql_client_session.h" using namespace oceanbase::common; using namespace oceanbase::common::hash; @@ -40,7 +42,8 @@ DEF_TO_STRING(ObPsSqlMeta) { int64_t pos = 0; J_OBJ_START(); - J_KV(KP(this), K_(column_count), K_(param_count), K_(param_types)); + J_KV(KP(this), K_(column_count), K_(param_count), K_(param_types), + KP_(param_type), K_(param_type_len)); J_OBJ_END(); return pos; } @@ -50,7 +53,7 @@ DEF_TO_STRING(ObPsEntry) int64_t pos = 0; J_OBJ_START(); J_KV(KP(this), "ps_sql_len", base_ps_sql_.length(), "ps_sql", ObProxyMysqlRequest::get_print_sql(base_ps_sql_), - K_(ps_meta), K_(base_ps_parse_result)); + K_(base_ps_parse_result)); J_OBJ_END(); return pos; } @@ -59,7 +62,7 @@ DEF_TO_STRING(ObPsIdEntry) { int64_t pos = 0; J_OBJ_START(); - J_KV(KP(this), K_(ps_id), KPC_(ps_entry)); + J_KV(KP(this), K_(ps_id), KPC_(ps_entry), K_(ps_meta)); J_OBJ_END(); return pos; } @@ -165,6 +168,7 @@ int ObPsIdEntry::alloc_ps_id_entry(uint32_t ps_id, ObPsEntry *ps_entry, ObPsIdEn void ObPsIdEntry::destroy() { LOG_INFO("ps id entry will be destroyed", KPC(this)); + ps_meta_.reset(); int64_t total_len = sizeof(ObPsIdEntry); ps_entry_->dec_ref(); ps_entry_ = NULL; @@ -192,7 +196,30 @@ void ObPsIdPair::destroy() op_fixed_mem_free(this, total_len); } -int ObPsEntry::alloc_and_init_ps_entry(const ObString &ps_sql, const ObSqlParseResult &parse_result, ObPsEntry *&entry) +int ObPsSqlMeta::set_param_type(const char *param_type, int64_t param_type_len) +{ + int ret = OB_SUCCESS; + + if (NULL != param_type_ && param_type_len_ > 0) { + op_fixed_mem_free(param_type_, param_type_len_); + param_type_ = NULL; + param_type_len_ = 0; + } + + if (OB_ISNULL(param_type_ = static_cast(op_fixed_mem_alloc(param_type_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc param type", K(param_type_len), K(ret)); + } else { + memcpy(param_type_, param_type, param_type_len); + param_type_len_ = param_type_len; + } + + return ret; +} + +int ObPsEntry::alloc_and_init_ps_entry(const ObString &ps_sql, + const ObSqlParseResult &parse_result, + ObPsEntry *&entry) { int ret = OB_SUCCESS; int64_t alloc_size = 0; @@ -266,7 +293,6 @@ void ObPsEntry::destroy() if (OB_LIKELY(is_inited_)) { ObBasePsEntry::destroy(); is_inited_ = false; - ps_meta_.reset(); int64_t total_len = sizeof(ObPsEntry) + buf_len_; buf_start_ = NULL; buf_len_ = 0; @@ -286,6 +312,20 @@ void ObBasePsEntryCache::destroy() base_ps_map_.reset(); } +int init_ps_entry_cache_for_thread() +{ + int ret = OB_SUCCESS; + const int64_t event_thread_count = g_event_processor.thread_count_for_type_[ET_CALL]; + for (int64_t i = 0; i < event_thread_count && OB_SUCC(ret); ++i) { + if (OB_ISNULL(g_event_processor.event_thread_[ET_CALL][i]->ps_entry_cache_ + = new (std::nothrow) ObBasePsEntryCache())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + PROXY_NET_LOG(WARN, "fail to new ObBasePsEntryCache", K(i), K(ret)); + } + } + return ret; +} + int ObTextPsEntry::alloc_and_init_text_ps_entry(const ObString &text_ps_sql, const ObSqlParseResult &parse_result, ObTextPsEntry *&entry) diff --git a/src/obproxy/proxy/mysql/ob_prepare_statement_struct.h b/src/obproxy/proxy/mysql/ob_prepare_statement_struct.h index b01448d6e9fae82113587a223f25e20c23b1c93f..5d43ce47490a18d2f025008617a3685d2cf1a79f 100644 --- a/src/obproxy/proxy/mysql/ob_prepare_statement_struct.h +++ b/src/obproxy/proxy/mysql/ob_prepare_statement_struct.h @@ -21,6 +21,9 @@ #include "obutils/ob_proxy_sql_parser.h" #include "iocore/net/ob_inet.h" #include "lib/allocator/ob_mod_define.h" +#include "obproxy/iocore/eventsystem/ob_buf_allocator.h" +#include "lib/lock/ob_drw_lock.h" +#include "obutils/ob_proxy_sql_parser.h" #define PARAM_TYPE_BLOCK_SIZE 1 << 9 // 512 @@ -34,7 +37,7 @@ namespace obproxy { namespace proxy { - +class ObMysqlClientSession; // stored in client session info class ObPsIdAddrs { @@ -138,9 +141,8 @@ class ObPsSqlMeta public: ObPsSqlMeta() - : param_count_(0), - column_count_(0), - param_types_() + : param_count_(0), column_count_(0), + param_types_(), param_type_(NULL), param_type_len_(0) { param_types_.set_mod_id(common::ObModIds::OB_PROXY_PARAM_TYPE_ARRAY); param_types_.set_block_size(PARAM_TYPE_BLOCK_SIZE); @@ -152,6 +154,8 @@ public: int64_t get_param_count() const { return param_count_; } int64_t get_column_count() const { return column_count_; } ObIArray &get_param_types() { return param_types_; } + ObString get_param_type() { return ObString(param_type_len_, param_type_); } + int set_param_type(const char *param_type, int64_t param_type_len); void set_param_count(int64_t param_count) { param_count_ = param_count; } void set_column_count(int64_t column_count) { column_count_ = column_count; } int64_t to_string(char *buf, const int64_t buf_len) const; @@ -160,6 +164,8 @@ private: int64_t param_count_; int64_t column_count_; common::ObArray param_types_; + char *param_type_; + int64_t param_type_len_; DISALLOW_COPY_AND_ASSIGN(ObPsSqlMeta); }; @@ -168,12 +174,19 @@ inline void ObPsSqlMeta::reset() param_count_ = 0; column_count_ = 0; param_types_.reset(); + + if (NULL != param_type_ && param_type_len_ > 0) { + op_fixed_mem_free(param_type_, param_type_len_); + } + + param_type_ = NULL; + param_type_len_ = 0; } class ObPsEntry : public ObBasePsEntry { public: - ObPsEntry() : ObBasePsEntry(), ps_id_(0), ps_meta_() {} + ObPsEntry() : ObBasePsEntry() {} ~ObPsEntry() {} static int alloc_and_init_ps_entry(const common::ObString &ps_sql, @@ -183,16 +196,11 @@ public: void destroy(); int set_sql(const common::ObString &ps_sql); - ObPsSqlMeta &get_ps_sql_meta() { return ps_meta_; } - int64_t get_param_count() const { return ps_meta_.get_param_count(); } - uint32_t get_ps_id() { return ps_id_; } int64_t to_string(char *buf, const int64_t buf_len) const; - + private: const static int64_t PARSE_EXTRA_CHAR_NUM = 2; - uint32_t ps_id_; - ObPsSqlMeta ps_meta_; public: DISALLOW_COPY_AND_ASSIGN(ObPsEntry); }; @@ -201,7 +209,7 @@ public: class ObPsIdEntry { public: - ObPsIdEntry() : ps_id_(0), ps_entry_(NULL) {} + ObPsIdEntry() : ps_id_(0), ps_entry_(NULL), ps_meta_() {} ObPsIdEntry(uint32_t client_id, ObPsEntry *entry) : ps_id_(client_id), ps_entry_(entry) { ps_entry_->inc_ref(); @@ -220,11 +228,14 @@ public: uint32_t get_ps_id() { return ps_id_; } ObPsEntry *get_ps_entry() { return ps_entry_; } + ObPsSqlMeta &get_ps_sql_meta() { return ps_meta_; } + int64_t get_param_count() const { return ps_meta_.get_param_count(); } int64_t to_string(char *buf, const int64_t buf_len) const; public: uint32_t ps_id_; // client ps id ObPsEntry *ps_entry_; + ObPsSqlMeta ps_meta_; LINK(ObPsIdEntry, ps_id_link_); }; @@ -312,7 +323,7 @@ class ObBasePsEntryCache { public: ObBasePsEntryCache() : base_ps_map_() {} - ~ObBasePsEntryCache() {} + ~ObBasePsEntryCache() { destroy(); } void destroy(); public: @@ -343,6 +354,7 @@ public: } return ret; } + int set_ps_entry(ObPsEntry *ps_entry) { ObBasePsEntry *tmp_entry = static_cast(ps_entry); @@ -387,6 +399,8 @@ private: DISALLOW_COPY_AND_ASSIGN(ObBasePsEntryCache); }; +int init_ps_entry_cache_for_thread(); + } // end of namespace proxy } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_common_define.h b/src/obproxy/proxy/mysqllib/ob_mysql_common_define.h index 742c408c1ec2555bec97a8ac95069b2258703bc9..a87bce46d854f9f913546df60c77aa9fa0c9a0e0 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_common_define.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_common_define.h @@ -28,6 +28,8 @@ static const uint32_t MYSQL_PAYLOAD_LENGTH_LENGTH = 3; // standard payl static const int64_t MYSQL_COMP_HEADER_LENGTH = 3; // compression header extra size static const int64_t MYSQL_NET_TYPE_LENGTH = 1; // packet type size static const int64_t MYSQL_PS_EXECUTE_HEADER_LENGTH = 9; // ps packet header size: stmt_id + flag + iteration-count +static const int64_t MYSQL_PS_STMT_ID_LENGTH = 4; // mysql prepare-statement protocol: statement-id +static const int64_t MYSQL_PS_SEND_LONG_DATA_PARAM_ID_LENGTH = 2; // mysql ps send_long_data: param-id // mysql meta info include mysql header and mysql request type static const int64_t MYSQL_NET_META_LENGTH = MYSQL_NET_TYPE_LENGTH + MYSQL_NET_HEADER_LENGTH; @@ -162,7 +164,6 @@ bool is_supported_mysql_cmd(const obmysql::ObMySQLCmd mysql_cmd) case obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE: case obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA: case obmysql::OB_MYSQL_COM_STMT_CLOSE: - case obmysql::OB_MYSQL_COM_STMT_RESET: // Stored Procedures case obmysql::OB_MYSQL_COM_STMT_FETCH: case obmysql::OB_MYSQL_COM_CHANGE_USER: @@ -179,6 +180,8 @@ bool is_supported_mysql_cmd(const obmysql::ObMySQLCmd mysql_cmd) case obmysql::OB_MYSQL_COM_BINLOG_DUMP_GTID: // Stored Procedures case obmysql::OB_MYSQL_COM_SET_OPTION: + // mysql prepare statement + case obmysql::OB_MYSQL_COM_STMT_RESET: ret = false; break; default: diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.cpp index 6de3072dfe7283031b006289781eb1e0131c97a5..6b412927575fbcd2f1a964fb4303271e3b21ed6f 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.cpp @@ -34,6 +34,7 @@ static int64_t const MYSQL_BUFFER_SIZE = BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX int ObMysqlCompressAnalyzer::init( const uint8_t last_seq, const AnalyzeMode mode, const obmysql::ObMySQLCmd mysql_cmd, + const ObMysqlProtocolMode mysql_mode, const bool enable_extra_ok_packet_for_stats, const uint8_t last_ob20_seq, const uint32_t request_id, @@ -41,6 +42,7 @@ int ObMysqlCompressAnalyzer::init( { int ret = OB_SUCCESS; + UNUSED(mysql_mode); UNUSED(last_ob20_seq); UNUSED(request_id); UNUSED(sessid); diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.h b/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.h index f678e95600f31ab52f382c0139bebd8b103e4504..8c988c506a0e1548c6834322d0edd54e6125a053 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_compress_analyzer.h @@ -49,6 +49,7 @@ public: virtual int init(const uint8_t last_seq, const AnalyzeMode mode, const obmysql::ObMySQLCmd mysql_cmd, + const ObMysqlProtocolMode mysql_mode, const bool enable_extra_ok_packet_for_stats, const uint8_t last_ob20_seq, const uint32_t request_id, diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.cpp index d62dc3ce827d88ddb533ce8782d1b28163c03db4..366c8193fd70730e1f7d7972fa525df47e9a3240 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.cpp @@ -33,6 +33,7 @@ static int64_t const MYSQL_BUFFER_SIZE = BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX int ObMysqlCompressOB20Analyzer::init( const uint8_t last_seq, const AnalyzeMode mode, const obmysql::ObMySQLCmd mysql_cmd, + const ObMysqlProtocolMode mysql_mode, const bool enable_extra_ok_packet_for_stats, const uint8_t last_ob20_seq, const uint32_t request_id, @@ -40,7 +41,7 @@ int ObMysqlCompressOB20Analyzer::init( { int ret = OB_SUCCESS; - ObMysqlCompressAnalyzer::init(last_seq, mode, mysql_cmd, enable_extra_ok_packet_for_stats, + ObMysqlCompressAnalyzer::init(last_seq, mode, mysql_cmd, mysql_mode, enable_extra_ok_packet_for_stats, last_ob20_seq, request_id, sessid); last_ob20_seq_ = last_ob20_seq; request_id_ = request_id; @@ -48,9 +49,9 @@ int ObMysqlCompressOB20Analyzer::init( LOG_DEBUG("ObMysqlCompressOB20Analyzer init", K(request_id), "request_id_", request_id_); result_.set_cmd(mysql_cmd); - result_.set_mysql_mode(OCEANBASE_MYSQL_PROTOCOL_MODE); + result_.set_mysql_mode(mysql_mode); result_.set_enable_extra_ok_packet_for_stats(enable_extra_ok_packet_for_stats); - analyzer_.set_mysql_mode(OCEANBASE_MYSQL_PROTOCOL_MODE); + analyzer_.set_mysql_mode(mysql_mode); return ret; } diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.h b/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.h index 75ee2725a6d07aee14aa345d0023304209e9f16a..c520fbe96698afad40a92b14fefb46d1e4a3537c 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_compress_ob20_analyzer.h @@ -45,6 +45,7 @@ public: virtual int init(const uint8_t last_seq, const AnalyzeMode mode, const obmysql::ObMySQLCmd mysql_cmd, + const ObMysqlProtocolMode mysql_mode, const bool enable_extra_ok_packet_for_stats, const uint8_t last_ob20_seq, const uint32_t request_id, diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.cpp index f3f5848f042caa4590194007ca4eaf5b45288524..4ad44b3c5c7046f3a0559df3de6713d5d69c6cc4 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.cpp @@ -31,6 +31,7 @@ namespace proxy #define CONFIG_IP_ASSIGN(ret, x) (ret = (this->x##_.load(proxy_config.x))) ObMysqlConfigProcessor mysql_config_processor; +ObPerformanceParams performance_params; ObMysqlConfigParams::ObMysqlConfigParams() : stat_table_sync_interval_(0), @@ -324,6 +325,29 @@ DEF_TO_STRING(ObMysqlConfigParams) return pos; } +ObPerformanceParams::ObPerformanceParams() + : enable_performance_mode_(false), enable_trace_(false), enable_stat_(false), is_inited_(false) {} + +int ObPerformanceParams::assign_config(const ObProxyConfig &proxy_config) +{ + int ret = OB_SUCCESS; + + CONFIG_ITEM_ASSIGN(enable_performance_mode); + CONFIG_ITEM_ASSIGN(enable_trace); + CONFIG_ITEM_ASSIGN(enable_stat); + + return ret; +} + +DEF_TO_STRING(ObPerformanceParams) +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(enable_performance_mode), K_(enable_trace), K_(enable_stat)); + J_OBJ_END(); + return pos; +} + int ObMysqlConfigProcessor::reconfigure(const ObProxyConfig &proxy_config) { int ret = OB_SUCCESS; @@ -344,6 +368,12 @@ int ObMysqlConfigProcessor::reconfigure(const ObProxyConfig &proxy_config) params->dec_ref(); params = NULL; } + + if (OB_FAIL(performance_params.assign_config(proxy_config))) { + LOG_WARN("fail to set new performance params", K(ret)); + } else { + performance_params.is_inited_ = true; + } return ret; } diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.h b/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.h index f46c00bf8f2cb207509c989e5586a630cbdde702..2b4cb07156317a8dda37b7aee2689f66fb630c50 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_config_processor.h @@ -147,6 +147,23 @@ public: char proxy_primary_zone_name_[common::MAX_ZONE_LENGTH + 1]; }; +// param for performance +struct ObPerformanceParams +{ +public: + ObPerformanceParams(); + virtual ~ObPerformanceParams() {} + virtual void free() { delete this; } + + int assign_config(const obutils::ObProxyConfig &proxy_config); + DECLARE_TO_STRING; +public: + CfgBool enable_performance_mode_; + CfgBool enable_trace_; + CfgBool enable_stat_; + bool is_inited_; +}; + class ObMysqlConfigProcessor { public: @@ -170,6 +187,8 @@ private: extern ObMysqlConfigProcessor mysql_config_processor; inline ObMysqlConfigProcessor &get_global_mysql_config_processor() { return mysql_config_processor; } +extern ObPerformanceParams performance_params; +inline ObPerformanceParams get_global_performance_params() { return performance_params; } inline int ObMysqlConfigParams::get_one_test_server_addr(net::ObIpEndpoint &addr) const { diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.cpp index 8cd1de12e812e26f6ce8625408f75741cd74383a..8c4fbf79d11da751c4bd11a7349ce572c6d1790a 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.cpp @@ -15,10 +15,15 @@ #include "packet/ob_mysql_packet_reader.h" #include "packet/ob_mysql_packet_util.h" #include "rpc/obmysql/ob_mysql_global.h" +#include "rpc/obmysql/ob_mysql_util.h" #include "obutils/ob_cached_variables.h" #include "common/obsm_utils.h" #include "lib/timezone/ob_time_convert.h" +#include "lib/charset/ob_charset.h" #include "opsql/expr_parser/ob_expr_parser.h" +#include "proxy/mysqllib/ob_mysql_config_processor.h" +#include "obproxy/obutils/ob_proxy_sql_parser.h" +#include "proxy/shard/obproxy_shard_utils.h" #include #include @@ -34,7 +39,6 @@ using namespace obutils; using namespace obmysql; using namespace event; using namespace packet; - static const char OBPROXY_SIMPLE_PART_KEY_MARK = '*'; int ObRequestAnalyzeCtx::init_auth_request_analyze_ctx(ObRequestAnalyzeCtx &ctx, @@ -85,7 +89,7 @@ void ObMysqlRequestAnalyzer::analyze_request(const ObRequestAnalyzeCtx &ctx, } else { // 1. determine whether mysql request packet is received complete ObMysqlAnalyzeResult result; - if (ctx.is_auth_) { + if (OB_UNLIKELY(ctx.is_auth_)) { if (OB_FAIL(handle_auth_request(*ctx.reader_, result))) { LOG_WARN("fail to handle auth request", K(ret)); } @@ -111,9 +115,9 @@ void ObMysqlRequestAnalyzer::analyze_request(const ObRequestAnalyzeCtx &ctx, status = result.status_; if (ANALYZE_DONE == status) { LOG_DEBUG("mysql request packet is received complete", "is_auth", ctx.is_auth_, - "analyze status", ObProxyParserUtils::get_analyze_status_name(status), - K(result.meta_), "packet cmd type", - ObProxyParserUtils::get_sql_cmd_name(result.meta_.cmd_)); + "analyze status", ObProxyParserUtils::get_analyze_status_name(status), + K(result.meta_), "packet cmd type", + ObProxyParserUtils::get_sql_cmd_name(result.meta_.cmd_)); } else { LOG_DEBUG("mysql request packet has not yet been received complete", "is_auth", ctx.is_auth_, "analyze status", ObProxyParserUtils::get_analyze_status_name(status), K(avail_bytes), @@ -121,10 +125,10 @@ void ObMysqlRequestAnalyzer::analyze_request(const ObRequestAnalyzeCtx &ctx, ObProxyParserUtils::get_sql_cmd_name(result.meta_.cmd_)); } - if (OB_SUCCESS == ret && (ANALYZE_DONE == status || ANALYZE_CONT == status)) { + if (OB_SUCC(ret) && (ANALYZE_DONE == status || ANALYZE_CONT == status)) { // 3. set mysql request packet meta sql_cmd = result.meta_.cmd_; - if (OB_MYSQL_COM_LOGIN == result.meta_.cmd_ || OB_MYSQL_COM_HANDSHAKE == result.meta_.cmd_) { + if (OB_UNLIKELY(OB_MYSQL_COM_LOGIN == result.meta_.cmd_ || OB_MYSQL_COM_HANDSHAKE == result.meta_.cmd_)) { // add pkt meta to mysql auth request auth_request.set_packet_meta(result.meta_); } else { @@ -135,13 +139,11 @@ void ObMysqlRequestAnalyzer::analyze_request(const ObRequestAnalyzeCtx &ctx, } // 4. dispatch mysql packet by cmd type, and parse mysql request - if (ANALYZE_DONE == status) { + if (OB_LIKELY(ANALYZE_DONE == status)) { if (OB_FAIL(do_analyze_request(ctx, sql_cmd, auth_request, client_request, is_oracle_mode))) { if (OB_ERR_PARSE_SQL == ret) { //ob parse fail will not disconnect status = ANALYZE_OBPARSE_ERROR; - } else if (OB_ERROR_UNSUPPORT_EXPR_TYPE == ret) { - status = ANALYZE_OBUNSUPPORT_ERROR; } else { status = ANALYZE_ERROR; } @@ -155,7 +157,9 @@ void ObMysqlRequestAnalyzer::analyze_request(const ObRequestAnalyzeCtx &ctx, && result.meta_.pkt_len_ > ctx.large_request_threshold_len_ && avail_bytes + ObProxyMysqlRequest::PARSE_EXTRA_CHAR_NUM > ctx.request_buffer_length_ && !client_request.is_sharding_user() - && !ctx.using_ldg_) { + && !client_request.is_proxysys_user() + && !ctx.using_ldg_ + && OB_MYSQL_COM_STMT_SEND_LONG_DATA != sql_cmd) { if (OB_FAIL(do_analyze_request(ctx, sql_cmd, auth_request, client_request, is_oracle_mode))) { status = ANALYZE_ERROR; LOG_WARN("fail to dispatch mysql cmd", "analyze status", @@ -430,9 +434,10 @@ int ObMysqlRequestAnalyzer::parse_sql_fileds(ObProxyMysqlRequest &client_request { int ret = OB_SUCCESS; bool need_parse_fields = true; - ObExprParseMode parse_mode = INVLIAD_PARSE_MODE; + ObExprParseMode parse_mode = INVALID_PARSE_MODE; ObSqlParseResult &sql_parse_result = client_request.get_parse_result(); - if (sql_parse_result.is_select_stmt() && sql_parse_result.get_dbp_route_info().scan_all_) { + bool need_scan_all = ObProxyShardUtils::need_scan_all(sql_parse_result); + if (need_scan_all) { //sharding parser extract by ob_parse LOG_DEBUG("sharding sql no need parse_sql_fileds"); need_parse_fields = false; @@ -458,6 +463,7 @@ int ObMysqlRequestAnalyzer::parse_sql_fileds(ObProxyMysqlRequest &client_request ObExprParser expr_parser(*allocator, parse_mode); ObString sql = client_request.get_sql(); expr_result.part_key_info_.key_num_ = 0; + expr_result.target_mask_ = 0; ObString expr_sql = ObProxyMysqlRequest::get_expr_sql(sql, sql_parse_result.get_parsed_length()); if (SELECT_STMT_PARSE_MODE == parse_mode) { const char *expr_sql_str = expr_sql.ptr(); @@ -500,9 +506,9 @@ inline int ObMysqlRequestAnalyzer::do_analyze_request( int ret = OB_SUCCESS; switch (sql_cmd) { + case OB_MYSQL_COM_QUERY: case OB_MYSQL_COM_STMT_PREPARE_EXECUTE: - case OB_MYSQL_COM_STMT_PREPARE: - case OB_MYSQL_COM_QUERY: { + case OB_MYSQL_COM_STMT_PREPARE: { // add packet's buffer to mysql common request, for parsing later if (OB_FAIL(client_request.add_request(ctx.reader_, ctx.request_buffer_length_))) { LOG_WARN("fail to add com request", K(ret)); @@ -510,7 +516,7 @@ inline int ObMysqlRequestAnalyzer::do_analyze_request( LOG_DEBUG("the request sql", "sql", client_request.get_sql()); ObString sql = client_request.get_parse_sql(); - if ((OB_SUCC(ret)) && !sql.empty()) { + if ((OB_SUCC(ret)) && OB_LIKELY(!sql.empty())) { ObProxySqlParser sql_parser; ObSqlParseResult &sql_parse_result = client_request.get_parse_result(); // we will handle parser error in parse_sql @@ -530,9 +536,6 @@ inline int ObMysqlRequestAnalyzer::do_analyze_request( ctx.drop_origin_db_table_name_, client_request.is_sharding_user()))) { LOG_WARN("fail to parse sql", K(sql), K(ret)); - } else if (client_request.is_sharding_user() - && OB_FAIL(parse_sql_fileds(client_request, ctx.connection_collation_))){ - LOG_WARN("fail to extract_fileds"); } else if (OB_FAIL(handle_internal_cmd(client_request))) { LOG_WARN("fail to handle internal cmd", K(sql), K(ret)); } @@ -557,6 +560,7 @@ inline int ObMysqlRequestAnalyzer::do_analyze_request( case OB_MYSQL_COM_STMT_FETCH: case OB_MYSQL_COM_STMT_CLOSE: case OB_MYSQL_COM_STMT_EXECUTE: + case OB_MYSQL_COM_STMT_SEND_LONG_DATA: case OB_MYSQL_COM_STMT_SEND_PIECE_DATA: case OB_MYSQL_COM_STMT_GET_PIECE_DATA: { // add packet's buffer to mysql common request, for parsing later @@ -702,8 +706,11 @@ int ObMysqlRequestAnalyzer::handle_internal_cmd(ObProxyMysqlRequest &client_requ if (client_request.is_proxysys_user()) { // allow follow type to support java connect if (parse_result.is_show_stmt() || parse_result.is_set_stmt() || parse_result.is_select_stmt() + || parse_result.is_select_proxy_version() || parse_result.is_set_names_stmt() - || parse_result.is_update_stmt()) { + || parse_result.is_update_stmt() + || parse_result.is_replace_stmt() + || parse_result.is_delete_stmt()) { is_internal_cmd = true; } } @@ -764,52 +771,43 @@ int ObMysqlRequestAnalyzer::rewrite_part_key_comment(ObIOBufferReader *reader, return ret; } -int ObMysqlRequestAnalyzer::do_analyze_execute_param(const char *buf, - int64_t data_len, - const int64_t param_num, - ObIArray ¶m_types, - ObProxyMysqlRequest &client_request, - const int64_t target_index, - ObObj &target_obj) +int ObMysqlRequestAnalyzer::analyze_execute_header(const int64_t param_num, + const char *&bitmap, + int8_t &new_param_bound_flag, + const char *&buf, int64_t &data_len) { int ret = OB_SUCCESS; - int8_t new_param_bound_flag = 0; - ObIAllocator &allocator = client_request.get_param_allocator(); - - int64_t bitmap_types = (param_num + 7) /8; - const char *bitmap = buf; + int64_t bitmap_types = (param_num + 7) / 8; + bitmap = buf; buf += bitmap_types; data_len -= bitmap_types; + if (OB_FAIL(ObMysqlPacketUtil::get_int1(buf, data_len, new_param_bound_flag))) { LOG_WARN("fail to get int1", K(data_len), K(ret)); - } else if (1 == new_param_bound_flag) { - param_types.reset(); } + + return ret; +} + +// The default data of this method is complete and can only come in once +int ObMysqlRequestAnalyzer::parse_param_type(const int64_t param_num, + ObIArray ¶m_types, + const char *&buf, int64_t &data_len) +{ + int ret = OB_SUCCESS; + uint8_t type = 0; int8_t flag = 0; - ObObjType ob_type; - bool is_null = false; - ObCharsetType charset = ObCharset::get_default_charset(); // decode all types for (int64_t i = 0; OB_SUCC(ret) && i < param_num; ++i) { - if (1 == new_param_bound_flag) { - if (OB_FAIL(ObMysqlPacketUtil::get_uint1(buf, data_len, type))) { - LOG_WARN("fail to get uint1", K(data_len), K(ret)); - } else if (OB_FAIL(ObMysqlPacketUtil::get_int1(buf, data_len, flag))) { - LOG_WARN("fail to get int1", K(data_len), K(ret)); - } else if (OB_FAIL(param_types.push_back(static_cast(type)))) { - LOG_WARN("fail to push back param type", K(type), K(ret)); - } - } else { - if (param_num != param_types.count()) { - ret = OB_ERR_WRONG_DYNAMIC_PARAM; - LOG_WARN("wrong param num and param_types num", K(param_num), K(param_types), K(ret)); - } else { - type = static_cast(param_types.at(i)); - } - } - if (OB_SUCC(ret) && OB_MYSQL_TYPE_COMPLEX == type) { + if (OB_FAIL(ObMysqlPacketUtil::get_uint1(buf, data_len, type))) { + LOG_WARN("fail to get uint1", K(data_len), K(ret)); + } else if (OB_FAIL(ObMysqlPacketUtil::get_int1(buf, data_len, flag))) { + LOG_WARN("fail to get int1", K(data_len), K(ret)); + } else if (OB_FAIL(param_types.push_back(static_cast(type)))) { + LOG_WARN("fail to push back param type", K(type), K(ret)); + } else if (OB_MYSQL_TYPE_COMPLEX == type) { TypeInfo type_name_info; uint8_t elem_type = 0; // skip all complex type bytes @@ -828,11 +826,111 @@ int ObMysqlRequestAnalyzer::do_analyze_execute_param(const char *buf, } } // end complex type } + + return ret; +} + +// This method can be entered repeatedly, you need to pass the param offset and buf offset +int ObMysqlRequestAnalyzer::parse_param_type_from_reader(int64_t& param_offset, + const int64_t param_num, + ObIArray ¶m_types, + event::ObIOBufferReader* reader, + int64_t& analyzed_len, + bool& is_finished) + +{ + int ret = OB_SUCCESS; + + int64_t analyzed_len_tmp = analyzed_len; + uint8_t type = 0; + int8_t flag = 0; + // decode all types + for (int64_t i = param_offset; OB_SUCC(ret) && i < param_num; ++i) { + if (OB_FAIL(get_uint1_from_reader(reader, analyzed_len_tmp, type))) { + LOG_WARN("fail to get uint1", K(analyzed_len_tmp), K(ret)); + } else if (OB_FAIL(get_int1_from_reader(reader, analyzed_len_tmp, flag))) { + LOG_WARN("fail to get int1", K(analyzed_len_tmp), K(ret)); + } else if (OB_FAIL(param_types.push_back(static_cast(type)))) { + LOG_WARN("fail to push back param type", K(type), K(ret)); + } else if (OB_MYSQL_TYPE_COMPLEX == type) { + TypeInfo type_name_info; + uint8_t elem_type = 0; + // skip all complex type bytes + if (OB_FAIL(decode_type_info_from_reader(reader, analyzed_len_tmp, type_name_info))) { + LOG_WARN("failed to decode type info", K(ret)); + } else if (type_name_info.is_elem_type_) { + if (OB_FAIL(get_uint1_from_reader(reader, analyzed_len_tmp, elem_type))) { + LOG_WARN("fail to get uint1", K(analyzed_len_tmp), K(ret)); + } else if (OB_FAIL(ObSMUtils::get_ob_type(type_name_info.elem_type_, static_cast(elem_type)))) { + LOG_WARN("cast ob type from mysql type failed", K(type_name_info.elem_type_), K(elem_type), K(ret)); + } else if (OB_MYSQL_TYPE_COMPLEX == elem_type + && OB_FAIL(decode_type_info_from_reader(reader, analyzed_len_tmp, type_name_info))) { + LOG_WARN("failed to decode type info", K(ret)); + } + } + } // end complex type + + if (OB_SUCC(ret)) { + param_offset++; + analyzed_len = analyzed_len_tmp; + } + } + + if (OB_SUCC(ret)) { + // if type buff is complete + if (param_num == param_offset) { + is_finished = true; + } + } + + return ret; +} + +int ObMysqlRequestAnalyzer::do_analyze_execute_param(const char *buf, + int64_t data_len, + const int64_t param_num, + ObIArray *param_types, + ObProxyMysqlRequest &client_request, + const int64_t target_index, + ObObj &target_obj) +{ + int ret = OB_SUCCESS; + + int8_t new_param_bound_flag = 0; + common::ObArray param_types_tmp; + ObIAllocator &allocator = client_request.get_param_allocator(); + + const char *bitmap = NULL; + if (OB_FAIL(analyze_execute_header(param_num, bitmap, new_param_bound_flag, buf, data_len))) { + LOG_WARN("fail to analyze execute header", K(param_num), K(ret)); + } else if (1 == new_param_bound_flag) { + if (OB_FAIL(parse_param_type(param_num, param_types_tmp, buf, data_len))) { + LOG_WARN("fail to parse param type", K(param_num), K(ret)); + } else { + // If the flag is 1, anyway, the param type is also parsed here, so use the parsed directly here + // It is mainly the first Execute. If it is a large request, since the param type will not be parsed in the + // do_analyze_ps_execute_request function, when this function is reached, the passed param types array is empty + // If the flag is 0, for example, the second Execute is a large request, then the passed param types parameter is used directly + param_types = ¶m_types_tmp; + } + } + + if (OB_SUCC(ret)) { + // Check the number in param_num and parm_type + if (param_num != param_types->count()) { + ret = OB_ERR_WRONG_DYNAMIC_PARAM; + LOG_WARN("wrong param num and param_types num", K(param_num), KP(param_types), K(ret)); + } + } + + bool is_null = false; + ObObjType ob_type; const char *param_buf = buf; + ObCharsetType charset = ObCharset::get_default_charset(); // decode all values for (int64_t i = 0; OB_SUCC(ret) && i <= target_index; ++i) { target_obj.reset(); - type = param_types.at(i); + uint8_t type = param_types->at(i); if (OB_FAIL(ObSMUtils::get_ob_type(ob_type, static_cast(type)))) { LOG_DEBUG("fail to cast mysql type to ob type, will add param with null", K(i), K(type), K(ret)); } else { @@ -873,7 +971,7 @@ int ObMysqlRequestAnalyzer::analyze_execute_param(const int64_t param_num, const char *buf = data.ptr() + MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH; data_len -= (MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH); if (param_num > 0) { - if (OB_FAIL(do_analyze_execute_param(buf, data_len, param_num, param_types, + if (OB_FAIL(do_analyze_execute_param(buf, data_len, param_num, ¶m_types, client_request, target_index, target_obj))) { LOG_DEBUG("fail to do analyze execute param", K(ret)); } @@ -882,6 +980,105 @@ int ObMysqlRequestAnalyzer::analyze_execute_param(const int64_t param_num, return ret; } +/* + * try to calculate partition key for OB_MYSQL_COM_STMT_SEND_LONG_DATA + * steps: + * 1: get param id from send long data package, compare with param idx + * 2: try to get obj type from execute parse result if exist + * 3: get obj type from prepare parse result + * 4: resolve send long data package, fill to obj, prepare to calculate partition key + * 5: if the procedure failed, choose part key with random optimization + */ +int ObMysqlRequestAnalyzer::analyze_send_long_data_param(ObProxyMysqlRequest &client_request, + const int64_t execute_param_index, + ObProxyPartInfo *part_info, + ObPsIdEntry *ps_id_entry, + ObObj &target_obj) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(ps_id_entry)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ps id entry is NULL", K(ret)); + } else if (OB_UNLIKELY(execute_param_index >= ps_id_entry->get_param_count() + || execute_param_index < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid execute param idx", K(execute_param_index), K(ret)); + } else { + ObString data = client_request.get_req_pkt(); + uint16_t origin_param_id = 0; + const char *param_id_pos = data.ptr() + MYSQL_NET_META_LENGTH + MYSQL_PS_STMT_ID_LENGTH; // stmt_id 4 + obmysql::ObMySQLUtil::get_uint2(param_id_pos, origin_param_id); + uint64_t param_id = static_cast(origin_param_id); + + if (param_id != execute_param_index) { + ret = OB_INVALID_ARGUMENT; + LOG_DEBUG("param id is not part key.", K(ret)); + } else { + // try to get type from execute, if not, then get from part key info + bool get_param_type = false; + common::ObIArray ¶m_types = ps_id_entry->get_ps_sql_meta().get_param_types(); + if (!param_types.empty()) { + if (param_id > param_types.count()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to get param type from execute parse result", K(ret), K(param_id), K(param_types.count())); + } else { + EMySQLFieldType param_type = param_types.at(param_id); + ObObjType ob_type; + if (OB_FAIL(ObSMUtils::get_ob_type(ob_type, param_type))) { + LOG_WARN("fail to get ob type by mysql filed type", K(ret), K(param_type)); + } else { + target_obj.set_type(ob_type); + get_param_type = true; + } + } + } + + if (OB_SUCC(ret) && !get_param_type) { + ObString param_name; + obutils::SqlFieldResult &sql_result = client_request.get_parse_result().get_sql_filed_result(); + if (static_cast(param_id) > sql_result.field_num_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to get param name from prepare result", K(ret), K(param_id), K(sql_result.field_num_)); + } else { + obutils::SqlField &field = sql_result.fields_.at(param_id); + param_name = field.column_name_.string_; + + ObProxyPartKeyInfo &key_info = part_info->get_part_key_info(); + for (uint64_t i = 0; i < key_info.key_num_; ++i) { + ObProxyPartKey &part_key = key_info.part_keys_[i]; + if (param_name == ObString::make_string(part_key.name_.str_)) { + target_obj.set_type(static_cast(part_key.obj_type_)); + get_param_type = true; + } + } + } + } + + if (OB_SUCC(ret) && get_param_type) { + // common net meta + stmt_id + param_id + uint16 value_inc = MYSQL_NET_META_LENGTH + MYSQL_PS_STMT_ID_LENGTH + MYSQL_PS_SEND_LONG_DATA_PARAM_ID_LENGTH; + const char *buf = data.ptr() + value_inc; + int64_t buf_len = data.length() - value_inc; + ObCharsetType charset = ObCharset::get_default_charset(); + ObIAllocator &allocator = client_request.get_param_allocator(); + if (OB_FAIL(ObMysqlRequestAnalyzer::parse_param_value(allocator, + buf, + buf_len, + target_obj.get_type(), + charset, + target_obj))) { + LOG_WARN("fail to parse param value for send long data", K(ret)); + } + } else { + LOG_WARN("fail to analyze obj for send long data", K(ret), K(get_param_type)); + } + } + } + + return ret; +} + int ObMysqlRequestAnalyzer::analyze_prepare_execute_param(ObProxyMysqlRequest &client_request, const int64_t target_index, ObObj &target_obj) @@ -912,7 +1109,7 @@ int ObMysqlRequestAnalyzer::analyze_prepare_execute_param(ObProxyMysqlRequest &c if (OB_SUCC(ret) && param_num > 0) { common::ObArray param_types; - if (OB_FAIL(do_analyze_execute_param(buf, data_len, param_num, param_types, + if (OB_FAIL(do_analyze_execute_param(buf, data_len, param_num, ¶m_types, client_request, target_index, target_obj))) { LOG_DEBUG("fail to do analyze execute param", K(ret)); } @@ -922,7 +1119,7 @@ int ObMysqlRequestAnalyzer::analyze_prepare_execute_param(ObProxyMysqlRequest &c return ret; } -int ObMysqlRequestAnalyzer::parse_param_value(ObIAllocator &allocator, +int ObMysqlRequestAnalyzer::parse_param_value(ObIAllocator &allocator, const char *&data, int64_t &buf_len, const uint8_t type, const ObCharsetType charset, ObObj ¶m) { @@ -1000,6 +1197,10 @@ int ObMysqlRequestAnalyzer::parse_param_value(ObIAllocator &allocator, break; } case OB_MYSQL_TYPE_OB_RAW: + case OB_MYSQL_TYPE_BLOB: + case OB_MYSQL_TYPE_LONG_BLOB: + case OB_MYSQL_TYPE_MEDIUM_BLOB: + case OB_MYSQL_TYPE_TINY_BLOB: case OB_MYSQL_TYPE_STRING: case OB_MYSQL_TYPE_VARCHAR: case OB_MYSQL_TYPE_VAR_STRING: @@ -1259,6 +1460,170 @@ int ObMysqlRequestAnalyzer::decode_type_info(const char*& buf, int64_t &buf_len, return ret; } +int ObMysqlRequestAnalyzer::decode_type_info_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + TypeInfo &type_info) +{ + int ret = OB_SUCCESS; + uint64_t read_avail = reader->read_avail(); + if (OB_SUCC(ret)) { + uint64_t length = 0; + if (OB_FAIL(get_length_from_reader(reader, decoded_offset, length))) { + LOG_WARN("failed to get length", K(ret)); + } else if (read_avail < (decoded_offset + length)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("packet buf size is not enough", K(read_avail), K(decoded_offset), K(length), K(ret)); + } else { + decoded_offset += length; + } + } + + if (OB_SUCC(ret)) { + uint64_t length = 0; + if (OB_FAIL(get_length_from_reader(reader, decoded_offset, length))) { + LOG_WARN("failed to get length", K(ret)); + } else if (read_avail < (decoded_offset + length)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("packet buf size is not enough", K(read_avail), K(decoded_offset), K(length), K(ret)); + } else { + if (length == 0) { + type_info.is_elem_type_ = true; + } + decoded_offset += length; + } + } + + if (OB_SUCC(ret)) { + uint64_t version = 0; + if (OB_FAIL(get_length_from_reader(reader, decoded_offset, version))) { + LOG_WARN("failed to get version", K(ret)); + } + } + return ret; +} + +int ObMysqlRequestAnalyzer::get_length_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint64_t &length) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 1)) { + ret = OB_SIZE_OVERFLOW; + } else { + uint16_t s2 = 0; + uint32_t s4 = 0; + uint8_t sentinel = 0; + get_uint1_from_reader(reader, decoded_offset, sentinel); + if (sentinel < 251) { + length = sentinel; + } else if (sentinel == 251) { + length = UINT64_MAX; // represents a NULL resultset + } else if (sentinel == 252) { + ret = get_uint2_from_reader(reader, decoded_offset, s2); + length = s2; + } else if (sentinel == 253) { + ret = get_uint3_from_reader(reader, decoded_offset, s4); + length = s4; + } else if (sentinel == 254) { + ret = get_uint8_from_reader(reader, decoded_offset, length); + } else { + // 255??? won't get here. + decoded_offset++; // roll back + ret = OB_INVALID_DATA; + } + } + return ret; +} + +int ObMysqlRequestAnalyzer::get_uint1_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint8_t &v) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 1)) { + ret = OB_SIZE_OVERFLOW; + } else { + reader->copy((char*)&v, 1, decoded_offset); + decoded_offset += 1; + } + + return ret; +} + +int ObMysqlRequestAnalyzer::get_uint2_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint16_t &v) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 2)) { + ret = OB_SIZE_OVERFLOW; + } else { + char buf[2] = "\0"; + char *buf_ptr = buf; + reader->copy(buf, 2, decoded_offset); + ObMySQLUtil::get_uint2(buf_ptr, v); + decoded_offset += 2; + } + + return ret; +} + +int ObMysqlRequestAnalyzer::get_uint3_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint32_t &v) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 3)) { + ret = OB_SIZE_OVERFLOW; + } else { + char buf[3] = "\0"; + char *buf_ptr = buf; + reader->copy(buf, 3, decoded_offset); + ObMySQLUtil::get_uint3(buf_ptr, v); + decoded_offset += 3; + } + + return ret; +} + +int ObMysqlRequestAnalyzer::get_uint8_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint64_t &v) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 8)) { + ret = OB_SIZE_OVERFLOW; + } else { + char buf[8] = "\0"; + char *buf_ptr = buf; + reader->copy(buf, 8, decoded_offset); + ObMySQLUtil::get_uint8(buf_ptr, v); + decoded_offset += 8; + } + + return ret; +} + +int ObMysqlRequestAnalyzer::get_int1_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + int8_t &v) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(reader->read_avail() - decoded_offset < 1)) { + ret = OB_SIZE_OVERFLOW; + } else { + reader->copy((char*)&v, 1, decoded_offset); + decoded_offset += 1; + } + + return ret; +} + int ObMysqlRequestAnalyzer::analyze_sql_id(ObProxyMysqlRequest &client_request, ObString &sql_id) { int ret = OB_SUCCESS; @@ -1307,6 +1672,10 @@ int ObMysqlRequestAnalyzer::analyze_sql_id(ObProxyMysqlRequest &client_request, sql_id.assign_ptr(sql_id_buf, static_cast(sql_id_buf_len - 1)); LOG_DEBUG("succ to get sql id", K(sql_id), K(sql)); } + + if (OB_NOT_NULL(allocator)) { + allocator->reuse(); + } } return ret; } diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.h b/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.h index 91ab64b22473231dedcc63c7a9e4a086e311b0ad..a34e22039748e94e7e4423f89aa920b7e4c8fd67 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_request_analyzer.h @@ -17,6 +17,10 @@ #include "ob_proxy_auth_parser.h" #include "ob_proxy_parser_utils.h" #include "rpc/obmysql/ob_mysql_packet.h" +#include "ob_proxy_session_info.h" +#include "obproxy/proxy/route/obproxy_part_info.h" +#include "obproxy/proxy/mysql/ob_prepare_statement_struct.h" + namespace oceanbase { @@ -85,12 +89,29 @@ public: ObMysqlAnalyzeStatus &status, const bool is_oracle_mode = false); static void extract_fileds(const ObExprParseResult& result, obutils::SqlFieldResult &sql_result); + static int parse_sql_fileds(ObProxyMysqlRequest &client_request, + common::ObCollationType connection_collation); static int init_cmd_info(ObProxyMysqlRequest &client_request); + static int analyze_execute_header(const int64_t param_num, + const char *&bitmap, + int8_t &new_param_bound_flag, + const char *&buf, int64_t &data_len); + + static int parse_param_type(const int64_t param_num, + common::ObIArray ¶m_types, + const char *&buf, int64_t &data_len); + + static int parse_param_type_from_reader(int64_t& param_offset, + const int64_t param_num, + common::ObIArray ¶m_types, + event::ObIOBufferReader* reader, + int64_t& analyzed_len, + bool& is_finished); static int do_analyze_execute_param(const char *buf, int64_t data_len, const int64_t param_num, - common::ObIArray ¶m_types, + common::ObIArray *param_types, ObProxyMysqlRequest &client_request, const int64_t target_index, ObObj &target_obj); @@ -99,11 +120,19 @@ public: ObProxyMysqlRequest &client_request, const int64_t target_index, common::ObObj &target_obj); - + static int analyze_send_long_data_param(ObProxyMysqlRequest &client_request, + const int64_t execute_param_index, + ObProxyPartInfo *part_info, + ObPsIdEntry *ps_id_entry, + ObObj &target_obj); static int analyze_prepare_execute_param(ObProxyMysqlRequest &client_request, const int64_t target_index, ObObj &target_obj); + static int parse_param_value(common::ObIAllocator &allocator, + const char *&data, int64_t &buf_len, const uint8_t type, + const ObCharsetType charset, ObObj ¶m); + static int analyze_sql_id(ObProxyMysqlRequest &client_request, common::ObString &sql_id); private: @@ -131,23 +160,43 @@ private: const bool is_oracle_mode = false); static int handle_internal_cmd(ObProxyMysqlRequest &client_request); static void extract_fileds(const ObExprParseResult& result, ObProxyMysqlRequest &client_request); - static int parse_sql_fileds(ObProxyMysqlRequest &client_request, - common::ObCollationType connection_collation); static int rewrite_part_key_comment(event::ObIOBufferReader *reader, ObProxyMysqlRequest &client_request); static void mysql_hex_dump(const void *data, const int64_t size); - static int parse_param_value(common::ObIAllocator &allocator, - const char *&data, int64_t &buf_len, const uint8_t type, - const ObCharsetType charset, ObObj ¶m); + static int parse_mysql_timestamp_value(const obmysql::EMySQLFieldType field_type, const char *&data, int64_t &buf_len, ObObj ¶m); static int parse_mysql_time_value(const char *&data, int64_t &buf_len, ObObj ¶m); static int decode_type_info(const char*& buf, int64_t &buf_len, TypeInfo &type_info); + static int decode_type_info_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + TypeInfo &type_info); + + static int get_uint1_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint8_t &v); + static int get_uint2_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint16_t &v); + static int get_uint3_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint32_t &v); + static int get_uint8_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint64_t &v); + static int get_int1_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + int8_t &v); + + static int get_length_from_reader(event::ObIOBufferReader* reader, + int64_t &decoded_offset, + uint64_t &length); + private: int64_t packet_length_; // request packet length uint8_t packet_seq_; diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.cpp index 0e69324d5ec03857571d86a7c9b7480f26ba170b..b864fb159cd1d885b87b6564bb21f743507dd229 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.cpp @@ -46,7 +46,7 @@ inline int ObBufferReader::read(char *buf, const int64_t len) } //------------------ObMysqlPacketMetaAnalyzer------------------- -inline int ObMysqlPacketMetaAnalyzer::update_cur_type(ObRespResult &result) +inline int ObMysqlPacketMetaAnalyzer::update_cur_type(ObRespResult &result, const ObMysqlProtocolMode mysql_mode) { int ret = OB_SUCCESS; int64_t eof_pkt_cnt = result.get_pkt_cnt(EOF_PACKET_ENDING_TYPE); @@ -98,6 +98,18 @@ inline int ObMysqlPacketMetaAnalyzer::update_cur_type(ObRespResult &result) } else if (result.is_recv_resultset()) { cur_type_ = OK_PACKET_ENDING_TYPE; } + } else if (OB_MYSQL_COM_STMT_FETCH == result.get_cmd()) { + if (1 == err_pkt_cnt) { + cur_type_ = OK_PACKET_ENDING_TYPE; + // After two EOF packages in Oracle mode is the OK package + } else if (OCEANBASE_ORACLE_PROTOCOL_MODE == mysql_mode) { + if (1 != eof_pkt_cnt) { + cur_type_ = OK_PACKET_ENDING_TYPE; + } + // OB MySQL mode and standard MySQL mode are OK packets after an EOF + } else if (1 == eof_pkt_cnt) { + cur_type_ = OK_PACKET_ENDING_TYPE; + } } else if (1 != eof_pkt_cnt) { cur_type_ = OK_PACKET_ENDING_TYPE; // in OCEANBASE_MYSQL_MODE, if we got an erro in ResultSet Protocol(maybe timeout), the packet @@ -279,7 +291,40 @@ int ObRespResult::is_resp_finished(bool &finished, ObMysqlRespEndingType &ending break; } - case OB_MYSQL_COM_STMT_FETCH: + case OB_MYSQL_COM_STMT_FETCH: { + if (RESULT_SET_RESP_TYPE == resp_type_ || OTHERS_RESP_TYPE == resp_type_) { + if (OB_UNLIKELY(is_mysql_mode() || is_oceanbase_mysql_mode())) { + if (1 == pkt_cnt_[EOF_PACKET_ENDING_TYPE]) { + finished = true; + ending_type = EOF_PACKET_ENDING_TYPE; + } else if (1 == pkt_cnt_[ERROR_PACKET_ENDING_TYPE]) { + finished = true; + ending_type = ERROR_PACKET_ENDING_TYPE; + } else if (1 == pkt_cnt_[OK_PACKET_ENDING_TYPE]) { + finished = true; + ending_type = OK_PACKET_ENDING_TYPE; + } + } else if (OB_LIKELY(is_oceanbase_oracle_mode()) && 1 == pkt_cnt_[OK_PACKET_ENDING_TYPE]) { + if (2 == pkt_cnt_[EOF_PACKET_ENDING_TYPE]) { + finished = true; + ending_type = EOF_PACKET_ENDING_TYPE; + } else if (1 == pkt_cnt_[ERROR_PACKET_ENDING_TYPE]) { + finished = true; + ending_type = ERROR_PACKET_ENDING_TYPE; + } else { + finished = true; + ending_type = OK_PACKET_ENDING_TYPE; + } + } + } else if (MAX_RESP_TYPE == resp_type_) { + // have not read resp type, as read data len less than MYSQL_PACKET_HEADER(4) + finished = false; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("infile not supported now", K(ret)); + } + break; + } case OB_MYSQL_COM_STMT_EXECUTE: case OB_MYSQL_COM_QUERY: case OB_MYSQL_COM_STMT_SEND_PIECE_DATA: @@ -511,7 +556,7 @@ inline int ObMysqlRespAnalyzer::read_pkt_type(ObBufferReader &buf_reader, ObResp LOG_INFO("now received large packet", "meta", meta_analyzer_.get_meta(), K_(is_in_multi_pkt)); } - if (OB_FAIL(meta_analyzer_.update_cur_type(result))) { + if (OB_FAIL(meta_analyzer_.update_cur_type(result, mysql_mode_))) { LOG_WARN("fail to update ending type", K(ret)); } else { if (OB_LIKELY(OB_MYSQL_COM_QUERY == result.get_cmd() @@ -672,6 +717,8 @@ inline int ObMysqlRespAnalyzer::analyze_resp_pkt( ok_packet_action_type = OK_PACKET_ACTION_CONSUME; } else if (1 == eof_pkt_cnt && OB_MYSQL_COM_STMT_EXECUTE == result.get_cmd() && result.is_recv_resultset()) { ok_packet_action_type = OK_PACKET_ACTION_CONSUME; + } else if (1 == eof_pkt_cnt && OB_MYSQL_COM_STMT_FETCH == result.get_cmd()) { + ok_packet_action_type = OK_PACKET_ACTION_CONSUME; } else if (0 == err_pkt_cnt || 0 == eof_pkt_cnt) { // last ok packet, no err and eof in front // NOTE: in multi stmt, we will reset pkt cnt if it isn't the last stmt diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.h b/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.h index 1e27923b1132b5818c386e18a849095d6d609bc1..c9b5b682ed0328162838f4298f250218f666fc36 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_resp_analyzer.h @@ -31,6 +31,7 @@ enum ObMysqlProtocolMode UNDEFINED_MYSQL_PROTOCOL_MODE = 0, STANDARD_MYSQL_PROTOCOL_MODE, OCEANBASE_MYSQL_PROTOCOL_MODE, + OCEANBASE_ORACLE_PROTOCOL_MODE, }; enum ObMysqlResponseAnalyzerState @@ -65,7 +66,7 @@ private: }; // this class is binded to Server Command. when new Server -// Command arrive, this class should be reset. +// Command arrive, this class should be reset.` class ObRespResult { public: @@ -74,7 +75,7 @@ public: void reset(); int is_resp_finished(bool &finished, ObMysqlRespEndingType &ending_type) const; - void set_cmd(const obmysql::ObMySQLCmd cmd) { cmd_ = cmd; } + inline void set_cmd(const obmysql::ObMySQLCmd cmd) { cmd_ = cmd; } obmysql::ObMySQLCmd get_cmd() const { return cmd_; } int64_t get_pkt_cnt(const ObMysqlRespEndingType type) const { return pkt_cnt_[type]; } @@ -100,8 +101,10 @@ public: void set_trans_state(const ObRespTransState state) { trans_state_ = state; } ObMysqlProtocolMode get_mysql_mode() const { return mysql_mode_; } - void set_mysql_mode(const ObMysqlProtocolMode mode) { mysql_mode_ = mode; } - bool is_oceanbase_mode() const { return OCEANBASE_MYSQL_PROTOCOL_MODE == mysql_mode_; } + inline void set_mysql_mode(const ObMysqlProtocolMode mode) { mysql_mode_ = mode; } + bool is_oceanbase_mode() const { return OCEANBASE_MYSQL_PROTOCOL_MODE == mysql_mode_ || OCEANBASE_ORACLE_PROTOCOL_MODE == mysql_mode_; } + bool is_oceanbase_mysql_mode() const { return OCEANBASE_MYSQL_PROTOCOL_MODE == mysql_mode_; } + bool is_oceanbase_oracle_mode() const { return OCEANBASE_ORACLE_PROTOCOL_MODE == mysql_mode_; } bool is_mysql_mode() const { return STANDARD_MYSQL_PROTOCOL_MODE == mysql_mode_; } void set_enable_extra_ok_packet_for_stats(const bool enable_extra_ok_packet_for_stats) { @@ -154,7 +157,7 @@ public: bool is_need_copy(ObRespResult &result) const; bool is_need_reserve_packet(ObRespResult &result) const; - int update_cur_type(ObRespResult &result); + int update_cur_type(ObRespResult &result, const ObMysqlProtocolMode mysql_mode); bool is_full_filled() const { return meta_buf_.is_full_filled(); } bool empty() const { return (0 == meta_buf_.len()); } @@ -205,7 +208,7 @@ public: ObMysqlProtocolMode get_mysql_mode() const { return mysql_mode_; } void set_mysql_mode(const ObMysqlProtocolMode mode) { mysql_mode_ = mode; } - bool is_oceanbase_mode() const { return OCEANBASE_MYSQL_PROTOCOL_MODE == mysql_mode_; } + bool is_oceanbase_mode() const { return OCEANBASE_MYSQL_PROTOCOL_MODE == mysql_mode_ || OCEANBASE_ORACLE_PROTOCOL_MODE == mysql_mode_; } bool is_mysql_mode() const { return STANDARD_MYSQL_PROTOCOL_MODE == mysql_mode_; } bool need_wait_more_data() const { return (next_read_len_ > 0); } diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_response.h b/src/obproxy/proxy/mysqllib/ob_mysql_response.h index 722de56f34511900a7d7330bda7eddc30dd09846..b2d915652a92a8d8119e83c1ac8173e35b451fdf 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_response.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_response.h @@ -213,7 +213,7 @@ inline void ObRespAnalyzeResult::reset() has_new_sys_var_ = false; has_proxy_idc_name_user_var_ = false; connection_id_ = 0; - MEMSET(scramble_buf_, 0, sizeof(scramble_buf_)); + scramble_buf_[0] = '\0'; ok_packet_action_type_ = OK_PACKET_ACTION_SEND; reserved_len_ = 0; reserved_len_for_ob20_ok_ = 0; diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.cpp index 202432d179c73f28adc6a061f9ff5a12cdd2190d..fd7eb6ca95f0555f9b403aae617a094ba8afa62d 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.cpp @@ -32,6 +32,7 @@ namespace obproxy namespace proxy { const ObString ObMysqlResponseBuilder::OBPROXY_ROUTE_ADDR_NAME = "@obproxy_route_addr"; +const ObString ObMysqlResponseBuilder::OBPROXY_PROXY_VERSION_NAME = "proxy_version()"; int ObMysqlResponseBuilder::build_ok_resp(ObMIOBuffer &mio_buf, ObProxyMysqlRequest &client_request, @@ -155,6 +156,52 @@ int ObMysqlResponseBuilder::build_select_route_addr_resp(ObMIOBuffer &mio_buf, } return ret; } + +int ObMysqlResponseBuilder::build_select_proxy_version_resp(ObMIOBuffer &mio_buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &info, + const bool is_in_trans) +{ + int ret = OB_SUCCESS; + + // get seq + uint8_t seq = static_cast(client_request.get_packet_meta().pkt_seq_ + 1); + + // get field + ObMySQLField field; + if (client_request.get_parse_result().get_col_name().empty()) { + field.cname_ = OBPROXY_PROXY_VERSION_NAME; + field.org_cname_ = OBPROXY_PROXY_VERSION_NAME; + } else { + field.cname_ = client_request.get_parse_result().get_col_name(); + field.org_cname_ = OBPROXY_PROXY_VERSION_NAME; + } + field.type_ = OB_MYSQL_TYPE_VARCHAR; + field.charsetnr_ = CS_TYPE_BINARY; + field.flags_ = OB_MYSQL_BINARY_FLAG; + + // get filed value + ObObj field_value; + field_value.set_varchar(PACKAGE_VERSION); + + // get status flag + uint16_t status_flag = 0; + int64_t autocommit = info.get_cached_variables().get_autocommit(); + if (0 != autocommit) { + status_flag |= (1 << OB_SERVER_STATUS_AUTOCOMMIT_POS); + } + if (is_in_trans) { + status_flag |= (1 << OB_SERVER_STATUS_IN_TRANS_POS); + } + + // encode to mio_buf + if (OB_FAIL(ObMysqlPacketUtil::encode_kv_resultset(mio_buf, seq, + field, field_value, status_flag))) { + LOG_WARN("fail to encode kv resultset", K(seq), K(ret)); + } + return ret; +} + } // end of namespace proxy } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.h b/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.h index b63ac2c7c7fccee52fc093e1ff0c3a5738ddbe31..73a9fd12e663b2885493bc2ca0b65c1511afe569 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.h +++ b/src/obproxy/proxy/mysqllib/ob_mysql_response_builder.h @@ -30,6 +30,7 @@ class ObMysqlResponseBuilder { public: static const common::ObString OBPROXY_ROUTE_ADDR_NAME; + static const common::ObString OBPROXY_PROXY_VERSION_NAME; static int build_ok_resp(event::ObMIOBuffer &mio_buf, ObProxyMysqlRequest &client_request, @@ -61,6 +62,11 @@ public: ObProxyMysqlRequest &client_request, ObClientSessionInfo &info, const bool is_in_trans); + + static int build_select_proxy_version_resp(event::ObMIOBuffer &mio_buf, + ObProxyMysqlRequest &client_request, + ObClientSessionInfo &info, + const bool is_in_trans); }; inline int ObMysqlResponseBuilder::build_start_trans_resp(event::ObMIOBuffer &mio_buf, diff --git a/src/obproxy/proxy/mysqllib/ob_mysql_transaction_analyzer.cpp b/src/obproxy/proxy/mysqllib/ob_mysql_transaction_analyzer.cpp index a722592627f83b31d3457bbb58853b2b9acd5d60..9a35b65daf762260bee30f47ebbcd8c232633763 100644 --- a/src/obproxy/proxy/mysqllib/ob_mysql_transaction_analyzer.cpp +++ b/src/obproxy/proxy/mysqllib/ob_mysql_transaction_analyzer.cpp @@ -134,7 +134,7 @@ int ObMysqlTransactionAnalyzer::analyze_trans_response(const ObResultBuffer &buf return ret; } -inline int ObMysqlTransactionAnalyzer::analyze_trans_response(ObIOBufferReader &reader, +int ObMysqlTransactionAnalyzer::analyze_trans_response(ObIOBufferReader &reader, ObMysqlResp *resp /*= NULL*/) { int ret = OB_SUCCESS; @@ -223,7 +223,7 @@ int ObMysqlTransactionAnalyzer::analyze_response(ObIOBufferReader &reader, const bool need_receive_complete) { int ret = OB_SUCCESS; - if (need_receive_complete) { + if (OB_UNLIKELY(need_receive_complete)) { if (OB_FAIL(analyze_trans_response(reader, resp))) { LOG_WARN("fail to analyze trans response", K(ret)); result.status_ = ANALYZE_ERROR; @@ -242,7 +242,7 @@ int ObMysqlTransactionAnalyzer::analyze_response(ObIOBufferReader &reader, } } else { ObServerStatusFlags server_status(0); - if (OB_MYSQL_COM_STATISTICS == result_.get_cmd()) { + if (OB_UNLIKELY(OB_MYSQL_COM_STATISTICS == result_.get_cmd())) { // OB_MYSQL_COM_STATISTICS, maybe only have mysql packet header, without packet body if (OB_FAIL(ObProxyParserUtils::analyze_one_packet_only_header(reader, result))) { LOG_WARN("fail to analyze packet", K(&reader), K(ret)); @@ -255,10 +255,8 @@ int ObMysqlTransactionAnalyzer::analyze_response(ObIOBufferReader &reader, } if (ANALYZE_DONE == result.status_) { - if (OB_UNLIKELY(OCEANBASE_MYSQL_PROTOCOL_MODE != analyzer_.get_mysql_mode())) { - // non-oceanbase mode, do nothing - } else { - if (OB_MYSQL_COM_STATISTICS == result_.get_cmd()) { + if (OB_LIKELY(analyzer_.is_oceanbase_mode())) { + if (OB_UNLIKELY(OB_MYSQL_COM_STATISTICS == result_.get_cmd())) { if (result_.is_extra_ok_packet_for_stats_enabled()) { if (OB_FAIL(receive_next_ok_packet(reader, result))) { LOG_WARN("fail to receive next ok packet", K(ret)); @@ -273,7 +271,7 @@ int ObMysqlTransactionAnalyzer::analyze_response(ObIOBufferReader &reader, } else { // do nothing } - } else if (MYSQL_ERR_PACKET_TYPE == result.meta_.pkt_type_) { + } else if (OB_UNLIKELY(MYSQL_ERR_PACKET_TYPE == result.meta_.pkt_type_)) { // 2. if the fist packet is an err pkt, it must be followed by an ok packet. // in this case, we should also confirm the second ok packet is received competed. if (OB_FAIL(receive_next_ok_packet(reader, result))) { diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.cpp b/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.cpp index 28f48406cfaa11804a0512105014587bc18a0d0e..3212eceb945c2eb24fdf5d0bcc67e64578c3eb82 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.cpp +++ b/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.cpp @@ -37,7 +37,8 @@ void ObHSRResult::reset() cluster_name_.reset(); cluster_id_ = OB_DEFAULT_CLUSTER_ID; is_clustername_from_default_ = false; - has_full_username_ = false; + has_tenant_username_ = false; + has_cluster_username_ = false; response_.reset(); full_name_buf_[0] = '\0'; if (NULL != name_buf_) { @@ -67,7 +68,8 @@ int64_t ObHSRResult::to_string(char *buf, const int64_t buf_len) const K_(full_name), K_(response), K_(is_clustername_from_default), - K_(has_full_username)); + K_(has_tenant_username), + K_(has_cluster_username)); J_OBJ_END(); return pos; } @@ -355,8 +357,12 @@ int ObProxyAuthParser::do_parse_full_user_name(const ObString &full_name, } } } else { - // if tenant or cluster is not empty, think as full name - hsr.has_full_username_ = true; + if (!tenant.empty()) { + hsr.has_tenant_username_ = true; + } + if (!cluster.empty()) { + hsr.has_cluster_username_ = true; + } } } if (OB_SUCC(ret)) { diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.h b/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.h index b33d07000c452cca78bf37f871d96fbec392160a..67b0809ac70367d16843b0db2597ca8c7a77223b 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.h +++ b/src/obproxy/proxy/mysqllib/ob_proxy_auth_parser.h @@ -56,7 +56,8 @@ struct ObHSRResult // user_name_ is UserName // cluster_id_ is ClusterId // is_clustername_from_default_ whether login packet has clustername - // has_full_username_ whether login packet has tenant or clustername + // has_tenant_username_ whether login packet has tenant + // has_cluster_username_ whether login packet has clustername common::ObString full_name_; common::ObString user_tenant_name_; common::ObString cluster_name_; @@ -64,7 +65,8 @@ struct ObHSRResult common::ObString user_name_; bool is_clustername_from_default_; int64_t cluster_id_; - bool has_full_username_; + bool has_tenant_username_; + bool has_cluster_username_; obmysql::OMPKHandshakeResponse response_; diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.cpp b/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.cpp index 36a7ec52ce316bbc5c267a39003656f2395c7b35..4af8393b210496156efcf3762c29234164ceaf46 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.cpp +++ b/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.cpp @@ -88,13 +88,13 @@ int ObProxyMysqlRequest::add_request(event::ObIOBufferReader *reader, const int6 LOG_WARN("buffer reader is empty", K(ret)); } else { //OB_MYSQL_COM_STMT_CLOSE always followed other request - if (OB_UNLIKELY(OB_MYSQL_COM_STMT_CLOSE == meta_.cmd_) + if (OB_UNLIKELY(OB_MYSQL_COM_STMT_CLOSE == meta_.cmd_ || OB_MYSQL_COM_STMT_SEND_LONG_DATA == meta_.cmd_) && OB_LIKELY(total_len > meta_.pkt_len_)) { total_len = meta_.pkt_len_; } int64_t copy_len = 0; - if (is_sharding_user()) { + if (OB_UNLIKELY(is_sharding_user() || is_proxysys_user())) { copy_len = total_len; // add two '\0' at the tail for parser req_buf_len = req_buf_len > total_len + PARSE_EXTRA_CHAR_NUM ? req_buf_len : total_len + PARSE_EXTRA_CHAR_NUM; @@ -109,8 +109,6 @@ int ObProxyMysqlRequest::add_request(event::ObIOBufferReader *reader, const int6 } else { LOG_DEBUG("alloc request buf ", K(req_buf_len)); } - } else { - // do nothing } if (OB_SUCC(ret)) { diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.h b/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.h index 360ac4171f1454988146328f1e6380ce91598db0..9f8a606f8273f35cfea1fc88d0d2fac11636ce16 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.h +++ b/src/obproxy/proxy/mysqllib/ob_proxy_mysql_request.h @@ -71,10 +71,10 @@ public: int64_t get_sql_id_buf_len() const { return common::OB_MAX_SQL_ID_LENGTH + 1; } common::ObString get_parse_sql() { return get_parse_sql(get_sql()); } common::ObString get_expr_sql() { return get_expr_sql(get_sql(), result_.get_parsed_length()); } - common::ObString get_print_sql() { return get_print_sql(get_sql()); } + common::ObString get_print_sql(const int64_t sql_len = PRINT_SQL_LEN) { return get_print_sql(get_sql(), sql_len); } static common::ObString get_expr_sql(const common::ObString &req_sql, const int64_t parsed_length); static common::ObString get_parse_sql(const common::ObString &req_sql); - static common::ObString get_print_sql(const common::ObString &req_sql); + static common::ObString get_print_sql(const common::ObString &req_sql, const int64_t sql_len = PRINT_SQL_LEN); common::ObString get_req_pkt(); obutils::ObSqlParseResult &get_parse_result(); obutils::ObSqlParseResult *get_ps_parse_result() { return ps_result_; } @@ -93,6 +93,7 @@ public: bool is_sharding_user() const { return USER_TYPE_SHARDING == user_identity_; } bool is_proxysys_user() const { return USER_TYPE_PROXYSYS == user_identity_; } bool is_inspector_user() const { return USER_TYPE_INSPECTOR == user_identity_; } + bool is_rootsys_user() const { return USER_TYPE_ROOTSYS == user_identity_; } bool is_proxysys_tenant() const { return (is_proxysys_user() || is_inspector_user()); } void set_internal_cmd(const bool flag) { is_internal_cmd_ = flag; } @@ -100,6 +101,7 @@ public: void set_large_request(const bool flag) { is_large_request_ = flag; } void set_enable_analyze_internal_cmd(const bool internal) { enable_analyze_internal_cmd_ = internal; } void set_user_identity(const ObProxyLoginUserType type) { user_identity_ = type; } + inline ObProxyLoginUserType get_user_identity() const { return user_identity_; } int64_t get_packet_len() { return meta_.pkt_len_; } @@ -145,11 +147,11 @@ private: inline void ObProxyMysqlRequest::reuse(bool is_reset_origin_db_table /* true */) { - if (NULL != cmd_info_) { + if (OB_UNLIKELY(NULL != cmd_info_)) { op_fixed_mem_free(cmd_info_, static_cast(sizeof(ObInternalCmdInfo))); cmd_info_ = NULL; } - if (NULL != query_info_) { + if (OB_UNLIKELY(NULL != query_info_)) { op_fixed_mem_free(query_info_, static_cast(sizeof(ObProxyKillQueryInfo))); query_info_ = NULL; } @@ -211,7 +213,8 @@ inline void ObProxyMysqlRequest::reset(bool is_reset_origin_db_table /* true */) inline obutils::ObSqlParseResult &ObProxyMysqlRequest::get_parse_result() { obutils::ObSqlParseResult *result = &result_; - if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == meta_.cmd_ && NULL != ps_result_) { + if ((obmysql::OB_MYSQL_COM_STMT_EXECUTE == meta_.cmd_ || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == meta_.cmd_) + && NULL != ps_result_) { result = ps_result_; } return *result; @@ -304,8 +307,11 @@ inline common::ObString ObProxyMysqlRequest::get_sql() { const char *sql = NULL; int64_t sql_len = 0; - if (NULL != req_buf_ && req_pkt_len_ > MYSQL_NET_META_LENGTH) { - if (obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE == meta_.cmd_) { + if (OB_LIKELY(NULL != req_buf_ && req_pkt_len_ > MYSQL_NET_META_LENGTH)) { + if (OB_LIKELY(obmysql::OB_MYSQL_COM_STMT_PREPARE_EXECUTE != meta_.cmd_)) { + sql = req_buf_ + MYSQL_NET_META_LENGTH; // skip pkt meta(5 bytes) + sql_len = req_pkt_len_ - MYSQL_NET_META_LENGTH; + } else { int ret = OB_SUCCESS; uint64_t query_len = 0; const char *pos = req_buf_ + MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH; // skip 9 bytes @@ -331,9 +337,6 @@ inline common::ObString ObProxyMysqlRequest::get_sql() sql_len = copy_len; } } - } else { - sql = req_buf_ + MYSQL_NET_META_LENGTH; // skip pkt meta(5 bytes) - sql_len = req_pkt_len_ - MYSQL_NET_META_LENGTH; } } common::ObString sql_str(sql_len, sql); @@ -366,11 +369,11 @@ common::ObString ObProxyMysqlRequest::get_expr_sql( return expr_sql; } -common::ObString ObProxyMysqlRequest::get_print_sql(const common::ObString &req_sql) +common::ObString ObProxyMysqlRequest::get_print_sql(const common::ObString &req_sql, const int64_t sql_len) { return (req_sql.empty() ? req_sql - : common::ObString(std::min(req_sql.length(), static_cast(PRINT_SQL_LEN)), req_sql.ptr())); + : common::ObString(std::min(req_sql.length(), static_cast(sql_len)), req_sql.ptr())); } inline common::ObString ObProxyMysqlRequest::get_req_pkt() diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.cpp b/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.cpp index 5aa3c1ebb3042f51b6403a1069d2bf06dfbab60c..23e1d7c45fbc09f352467cb2d57a18efc2edad63 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.cpp +++ b/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.cpp @@ -218,10 +218,6 @@ const char *ObProxyParserUtils::get_analyze_status_name(const ObMysqlAnalyzeStat name = "ANALYZE_OBPARSE_ERROR"; break; - case ANALYZE_OBUNSUPPORT_ERROR: - name = "ANALYZE_OBUNSUPPORT_ERROR"; - break; - case ANALYZE_ERROR: name = "ANALYZE_ERROR"; break; @@ -270,12 +266,11 @@ int ObProxyParserUtils::analyze_one_packet(ObIOBufferReader &reader, ObMysqlAnal } if (OB_SUCC(ret)) { - if (OB_FAIL(analyze_mysql_packet_meta(buf_start, MYSQL_NET_META_LENGTH, result.meta_))) { - LOG_WARN("fail to analyze mysql packet meta", K(ret)); - } else { - if (len >= result.meta_.pkt_len_) { - result.status_ = ANALYZE_DONE; - } + result.meta_.pkt_len_ = ob_uint3korr(buf_start) + MYSQL_NET_HEADER_LENGTH; + result.meta_.pkt_seq_ = ob_uint1korr(buf_start + 3); + result.meta_.data_ = static_cast(buf_start[4]); + if (OB_LIKELY(len >= result.meta_.pkt_len_)) { + result.status_ = ANALYZE_DONE; } } } @@ -320,7 +315,7 @@ int ObProxyParserUtils::analyze_one_packet_only_header(ObIOBufferReader &reader, int ObProxyParserUtils::analyze_mysql_packet_meta(const char *ptr, const int64_t len, ObMysqlPacketMeta &meta) { int ret = OB_SUCCESS; - if (OB_ISNULL(ptr) || (len < MYSQL_NET_META_LENGTH)) { + if (OB_ISNULL(ptr) || (OB_UNLIKELY(len < MYSQL_NET_META_LENGTH))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid input value", KP(ptr), K(len), K(ret)); } else { diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.h b/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.h index 156b6aa41d57712f759d96468f271688d548cf30..c9ece90512b610e82b2cfdaf42118bcdd0a2e7c4 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.h +++ b/src/obproxy/proxy/mysqllib/ob_proxy_parser_utils.h @@ -32,7 +32,7 @@ namespace proxy enum ObMysqlAnalyzeStatus { - ANALYZE_OBUNSUPPORT_ERROR = -3, + ANALYZE_CAN_NOT_PASS_WHITE_LIST_ERROR = -3, ANALYZE_OBPARSE_ERROR = -2, ANALYZE_ERROR = -1, ANALYZE_DONE = 0, diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_session_info.cpp b/src/obproxy/proxy/mysqllib/ob_proxy_session_info.cpp index 1694a5b16e7474f3ac5b1c34b68d7c03312a50b9..bfe77e3e5eed5246b836bda24030a734af424c6f 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_session_info.cpp +++ b/src/obproxy/proxy/mysqllib/ob_proxy_session_info.cpp @@ -168,11 +168,13 @@ ObClientSessionInfo::ObClientSessionInfo() global_vars_version_(OB_INVALID_VERSION), obproxy_route_addr_(0), var_set_processor_(NULL), cluster_id_(OB_INVALID_CLUSTER_ID), real_meta_cluster_name_(), real_meta_cluster_name_str_(NULL), - server_type_(DB_OB_MYSQL), shard_conn_(NULL), group_id_(OBPROXY_MAX_DBMESH_ID), is_allow_use_last_session_(true), + server_type_(DB_OB_MYSQL), shard_conn_(NULL), shard_prop_(NULL), + group_id_(OBPROXY_MAX_DBMESH_ID), is_allow_use_last_session_(true), consistency_level_prop_(INVALID_CONSISTENCY), recv_client_ps_id_(0), ps_id_(0), ps_entry_(NULL), ps_id_entry_map_(), text_ps_name_entry_map_(), is_text_ps_execute_(false), - cursor_id_(0), cursor_id_addr_map_(), ps_id_addrs_map_(), + cursor_id_(0), cursor_id_addr_map_(), + ps_id_addrs_map_(), is_read_only_user_(false), is_request_follower_user_(false) { @@ -287,6 +289,19 @@ int ObClientSessionInfo::set_tenant_name(const ObString &tenant_name) return field_mgr_.set_tenant_name(tenant_name); } +int ObClientSessionInfo::set_vip_addr_name(const common::ObAddr &vip_addr) +{ + int ret = OB_SUCCESS; + char vip_name[OB_IP_STR_BUFF]; + if (OB_UNLIKELY(!vip_addr.ip_to_string(vip_name, static_cast(sizeof(vip_name))))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to covert ip to string", K(vip_name), K(ret)); + } else { + return field_mgr_.set_vip_addr_name(vip_name); + } + return ret; +} + int ObClientSessionInfo::set_logic_tenant_name(const ObString &logic_tenant_name) { return field_mgr_.set_logic_tenant_name(logic_tenant_name); @@ -360,6 +375,11 @@ int ObClientSessionInfo::get_database_name(ObString &database_name) const return field_mgr_.get_database_name(database_name); } +int ObClientSessionInfo::get_vip_addr_name(ObString &vip_addr_name) const +{ + return field_mgr_.get_vip_addr_name(vip_addr_name); +} + ObString ObClientSessionInfo::get_database_name() const { ObString database_name; @@ -1163,6 +1183,10 @@ void ObClientSessionInfo::destroy() shard_conn_->dec_ref(); shard_conn_ = NULL; } + if (NULL != shard_prop_) { + shard_prop_->dec_ref(); + shard_prop_ = NULL; + } group_id_ = OBPROXY_MAX_DBMESH_ID; is_allow_use_last_session_ = true; up_info_.reset(); diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_session_info.h b/src/obproxy/proxy/mysqllib/ob_proxy_session_info.h index a7829eb4b2f2c40ad26337f174397f88513feb1c..9bbad103ecfc4294c62b98cbbc5752d316da813b 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_session_info.h +++ b/src/obproxy/proxy/mysqllib/ob_proxy_session_info.h @@ -457,6 +457,7 @@ public: //set and get methord int set_cluster_name(const common::ObString &cluster_name); int set_tenant_name(const common::ObString &tenant_name); + int set_vip_addr_name(const common::ObAddr &vip_addr); int set_database_name(const common::ObString &database_name, const bool inc_db_version = true); int set_user_name(const common::ObString &user_name); int set_ldg_logical_cluster_name(const common::ObString &cluster_name); @@ -465,6 +466,7 @@ public: int set_logic_database_name(const common::ObString &logic_database_name); int get_cluster_name(common::ObString &cluster_name) const; int get_tenant_name(common::ObString &tenant_name) const; + int get_vip_addr_name(common::ObString &vip_addr_name) const; int get_database_name(common::ObString &database_name) const; common::ObString get_database_name() const; common::ObString get_full_username(); @@ -669,10 +671,23 @@ public: shard_conn_ = shard_conn; } + dbconfig::ObShardProp *get_shard_prop() { return shard_prop_; } + void set_shard_prop(dbconfig::ObShardProp *shard_prop) { + if (NULL != shard_prop_) { + shard_prop_->dec_ref(); + shard_prop_ = NULL; + } + + if (NULL != shard_prop) { + shard_prop->inc_ref(); + } + shard_prop_ = shard_prop; + } + DBServerType get_server_type() const { return server_type_; } void set_server_type(DBServerType server_type) { server_type_ = server_type; } - bool is_oceanbase_server() const { return DB_OB_MYSQL == server_type_ || DB_OB_ORACLE == server_type_; } + inline bool is_oceanbase_server() const { return DB_OB_MYSQL == server_type_ || DB_OB_ORACLE == server_type_; } void set_allow_use_last_session(const bool is_allow_use_last_session) { is_allow_use_last_session_ = is_allow_use_last_session; } bool is_allow_use_last_session() { return is_allow_use_last_session_; } @@ -716,6 +731,17 @@ public: set_client_ps_id(ps_id_entry->ps_id_); return ps_id_entry_map_.unique_set(ps_id_entry); } + ObPsIdEntry *get_ps_id_entry() { return ps_id_entry_; } + ObPsIdEntry *get_ps_id_entry(uint32_t ps_id) { + int ret = OB_SUCCESS; + ObPsIdEntry *ps_id_entry = NULL; + if (OB_FAIL(ps_id_entry_map_.get_refactored(ps_id, ps_id_entry))) { + if (OB_HASH_NOT_EXIST != ret) { + PROXY_LOG(WARN, "fail to get ps id entry with ps id", K(ret)); + } + } + return ps_id_entry; + } void remove_ps_id_entry(uint32_t client_ps_id) { ObPsIdEntry *ps_id_entry = ps_id_entry_map_.remove(client_ps_id); if (NULL != ps_id_entry) { @@ -725,6 +751,7 @@ public: } void destroy_ps_id_entry_map(); void set_ps_entry(ObPsEntry *entry) { ps_entry_ = entry; } + void set_ps_id_entry(ObPsIdEntry *ps_id_entry) { ps_id_entry_ = ps_id_entry; } int get_ps_sql(common::ObString &ps_sql); bool need_do_prepare(ObServerSessionInfo &server_info) const; @@ -896,6 +923,7 @@ private: DBServerType server_type_; dbconfig::ObShardConnector *shard_conn_; + dbconfig::ObShardProp *shard_prop_; int64_t group_id_; bool is_allow_use_last_session_; @@ -911,6 +939,7 @@ private: uint32_t recv_client_ps_id_; uint32_t ps_id_; ObPsEntry *ps_entry_; + ObPsIdEntry *ps_id_entry_; ObPsIdEntryMap ps_id_entry_map_; common::ObString text_ps_name_; ObTextPsEntry *text_ps_entry_; @@ -979,7 +1008,7 @@ inline void ObClientSessionInfo::set_idc_name(const ObString &name) inline bool ObClientSessionInfo::need_reset_database(const ObServerSessionInfo &server_info) const { - if (!is_session_pool_client_) { + if (OB_LIKELY(!is_session_pool_client_)) { return get_db_name_version() > server_info.get_db_name_version() && enable_reset_db_; } if (get_database_name().empty()) return false; @@ -995,10 +1024,10 @@ inline bool ObClientSessionInfo::need_reset_database(const ObServerSessionInfo & inline bool ObClientSessionInfo::need_reset_common_hot_session_vars(const ObServerSessionInfo &server_info) const { bool bret = false; - bool bret_hash = false; - if (!is_session_pool_client_) { + if (OB_LIKELY(!is_session_pool_client_)) { bret = get_common_hot_sys_var_version() > server_info.get_common_hot_sys_var_version(); } else { + bool bret_hash = false; if (is_common_hot_sys_version_changed()) { bret_hash = true; } else if (val_hash_.common_hot_sys_var_hash_ != server_info.val_hash_.common_hot_sys_var_hash_) { @@ -1013,7 +1042,7 @@ inline bool ObClientSessionInfo::need_reset_common_hot_session_vars(const ObServ inline bool ObClientSessionInfo::need_reset_common_cold_session_vars(const ObServerSessionInfo &server_info) const { bool bret = false; - if (!is_session_pool_client_) { + if (OB_LIKELY(!is_session_pool_client_)) { bret = get_common_sys_var_version() > server_info.get_common_sys_var_version(); } else { bool is_changed = is_common_cold_sys_version_changed(); @@ -1068,10 +1097,10 @@ inline bool ObClientSessionInfo::need_reset_mysql_cold_session_vars(const ObServ inline bool ObClientSessionInfo::need_reset_hot_session_vars(const ObServerSessionInfo &server_info) const { bool bret = false; - bool bret_hash_diff = false; - if (!is_session_pool_client_) { + if (OB_LIKELY(!is_session_pool_client_)) { bret = get_hot_sys_var_version() > server_info.get_hot_sys_var_version(); } else { + bool bret_hash_diff = false; bool is_changed = is_sys_hot_version_changed(); if (is_changed) { bret = true; @@ -1087,7 +1116,7 @@ inline bool ObClientSessionInfo::need_reset_hot_session_vars(const ObServerSessi inline bool ObClientSessionInfo::need_reset_cold_session_vars(const ObServerSessionInfo &server_info) const { bool bret = false; - if (!is_session_pool_client_) { + if (OB_LIKELY(!is_session_pool_client_)) { bret = get_sys_var_version() > server_info.get_sys_var_version(); } else { bool is_changed= is_sys_cold_version_changed(); @@ -1106,8 +1135,8 @@ inline bool ObClientSessionInfo::need_reset_cold_session_vars(const ObServerSess inline bool ObClientSessionInfo::need_reset_user_session_vars(const ObServerSessionInfo &server_info) const { bool bret = false; - if (!is_session_pool_client_) { - if (get_user_var_version() > server_info.get_user_var_version()) { + if (OB_LIKELY(!is_session_pool_client_)) { + if (OB_UNLIKELY(get_user_var_version() > server_info.get_user_var_version())) { int ret = OB_SUCCESS; common::ObSEArray names; ObClientSessionInfo* client_info = const_cast(this); @@ -1135,8 +1164,8 @@ inline bool ObClientSessionInfo::need_reset_user_session_vars(const ObServerSess inline bool ObClientSessionInfo::need_reset_last_insert_id(const ObServerSessionInfo &server_info) const { - if (is_oceanbase_server()) { - if (!is_session_pool_client_) { + if (OB_LIKELY(is_oceanbase_server())) { + if (OB_LIKELY(!is_session_pool_client_)) { return get_last_insert_id_version() > server_info.get_last_insert_id_version(); } return !(const_cast(this))->field_mgr_.is_same_last_insert_id_var(server_info.field_mgr_); @@ -1153,7 +1182,7 @@ inline bool ObClientSessionInfo::need_reset_safe_read_snapshot(const ObServerSes inline bool ObClientSessionInfo::need_reset_session_vars(const ObServerSessionInfo &server_info) const { bool bret = true; - if (is_oceanbase_server()) { + if (OB_LIKELY(is_oceanbase_server())) { bret = need_reset_common_hot_session_vars(server_info) || need_reset_common_cold_session_vars(server_info) || need_reset_hot_session_vars(server_info) diff --git a/src/obproxy/proxy/mysqllib/ob_proxy_session_info_handler.cpp b/src/obproxy/proxy/mysqllib/ob_proxy_session_info_handler.cpp index 53f1e7d74ce515b16a8dbb500c4724bd25f3c3a6..b458af2b5e2dd4b94f07c3a9be2891e35e26cc00 100644 --- a/src/obproxy/proxy/mysqllib/ob_proxy_session_info_handler.cpp +++ b/src/obproxy/proxy/mysqllib/ob_proxy_session_info_handler.cpp @@ -100,7 +100,7 @@ int ObProxySessionInfoHandler::rebuild_ok_packet(ObIOBufferReader &reader, const ObMySQLCapabilityFlags cap = server_info.get_compatible_capability_flags(); OMPKOK src_ok; int64_t offset = reader.read_avail() - pkt_len; - LOG_DEBUG("rebuild_ok_packet", K(reader.read_avail()), K(pkt_len)); + LOG_DEBUG("rebuild_ok_packet", K(reader.read_avail()), K(pkt_len), K(offset)); // 1. get ok packet from buffer pkt_reader.get_ok_packet(reader, offset, cap, src_ok); @@ -168,8 +168,6 @@ int ObProxySessionInfoHandler::rewrite_query_req_by_sharding(ObClientSessionInfo LOG_WARN("fail to analyze request", K(status), K(ret)); if (ANALYZE_OBPARSE_ERROR == status) { ret = OB_ERR_PARSER_SYNTAX; - } else if (ANALYZE_OBUNSUPPORT_ERROR == status) { - ret = OB_ERROR_UNSUPPORT_EXPR_TYPE; } else { ret = OB_ERR_UNEXPECTED; } @@ -405,11 +403,12 @@ int ObProxySessionInfoHandler::rewrite_ldg_login_req(ObClientSessionInfo &clien PROXY_CS_LOG(DEBUG, "rewrite ldg login req", K(auth_req.get_hsr_result())); } } - if (OB_LIKELY(NULL != target_hsr_buf)) { - free_miobuffer(target_hsr_buf); - target_hsr_buf = NULL; - target_hsr_reader = NULL; - } + } + + if (OB_LIKELY(NULL != target_hsr_buf)) { + free_miobuffer(target_hsr_buf); + target_hsr_buf = NULL; + target_hsr_reader = NULL; } return ret; } diff --git a/src/obproxy/proxy/mysqllib/ob_session_field_mgr.cpp b/src/obproxy/proxy/mysqllib/ob_session_field_mgr.cpp index b96e465a18f8721208f2edef8404b475372c8be5..b71db0a362df1377dfd2b6dd74470cc0c9a30007 100644 --- a/src/obproxy/proxy/mysqllib/ob_session_field_mgr.cpp +++ b/src/obproxy/proxy/mysqllib/ob_session_field_mgr.cpp @@ -508,6 +508,18 @@ int ObSessionFieldMgr::set_tenant_name(const ObString &tenant_name) return ret; } +int ObSessionFieldMgr::set_vip_addr_name(const ObString &vip_addr_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(replace_str_field(OB_V_FIELD_VIP_ADDR_NAME, vip_addr_name))) { + LOG_WARN("fail to set vip addr name", K(vip_addr_name), K(ret)); + } + return ret; +} + int ObSessionFieldMgr::set_user_name(const ObString &user_name) { int ret = OB_SUCCESS; @@ -619,6 +631,18 @@ int ObSessionFieldMgr::get_tenant_name(ObString &tenant_name) const return ret; } +int ObSessionFieldMgr::get_vip_addr_name(ObString &vip_addr_name) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_FAIL(get_str_field_value(OB_V_FIELD_VIP_ADDR_NAME, vip_addr_name))) { + LOG_DEBUG("fail to get vip addr name", K(ret)); + } + return ret; +} + int ObSessionFieldMgr::get_user_name(ObString &user_name) const { int ret = OB_SUCCESS; diff --git a/src/obproxy/proxy/mysqllib/ob_session_field_mgr.h b/src/obproxy/proxy/mysqllib/ob_session_field_mgr.h index 16fd22382a674b1344ebcaf94e2705e503b12725..cbe099568314168c677661efa394a8a277b5b5a7 100644 --- a/src/obproxy/proxy/mysqllib/ob_session_field_mgr.h +++ b/src/obproxy/proxy/mysqllib/ob_session_field_mgr.h @@ -57,6 +57,7 @@ enum ObVFieldType OB_V_FIELD_LDG_LOGICAL_TENANT_NAME, OB_V_FIELD_LDG_REAL_CLUSTER_NAME, OB_V_FIELD_LDG_REAL_TENANT_NAME, + OB_V_FIELD_VIP_ADDR_NAME, OB_V_FIELD_MAX, }; @@ -385,6 +386,7 @@ public: //set and get methord int set_cluster_name(const common::ObString &cluster_name); int set_tenant_name(const common::ObString &tenant_name); + int set_vip_addr_name(const common::ObString &vip_addr_name); int set_user_name(const common::ObString &user_name); int set_database_name(const common::ObString &database_name); int set_logic_tenant_name(const common::ObString &logic_tenant_name); @@ -393,6 +395,7 @@ public: int set_ldg_logical_tenant_name(const common::ObString &tenant_name); int get_cluster_name(common::ObString &cluster_name) const; int get_tenant_name(common::ObString &tenant_name) const; + int get_vip_addr_name(common::ObString &vip_addr_name) const; int get_user_name(common::ObString &user_name) const; int get_database_name(common::ObString &database_name) const; int get_logic_tenant_name(common::ObString &logic_tenant_name) const; diff --git a/src/obproxy/proxy/plugins/Makemodule.am b/src/obproxy/proxy/plugins/Makemodule.am index 69fd1b449380d9f7a03b880bf0c386e1ce0f3110..ee07f45cc543ad43394097236fca72e7cdf080fe 100644 --- a/src/obproxy/proxy/plugins/Makemodule.am +++ b/src/obproxy/proxy/plugins/Makemodule.am @@ -7,6 +7,8 @@ obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.cpp\ obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.h\ obproxy/proxy/plugins/ob_mysql_request_prepare_transform_plugin.cpp\ obproxy/proxy/plugins/ob_mysql_request_prepare_transform_plugin.h\ +obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.cpp\ +obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.h\ obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.cpp\ obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.h\ obproxy/proxy/plugins/ob_mysql_response_new_ps_transform_plugin.cpp\ diff --git a/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.cpp b/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0516f399d96415ff860a573534590720286619d7 --- /dev/null +++ b/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.cpp @@ -0,0 +1,331 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include "ob_mysql_request_execute_transform_plugin.h" +#include "proxy/mysqllib/ob_mysql_analyzer_utils.h" +#include "rpc/obmysql/ob_mysql_util.h" + +using namespace oceanbase::common; +using namespace oceanbase::obproxy::event; + +namespace oceanbase +{ +namespace obproxy +{ +namespace proxy +{ + +ObMysqlRequestExecuteTransformPlugin *ObMysqlRequestExecuteTransformPlugin::alloc(ObApiTransaction &transaction, int64_t req_pkt_len) +{ + return op_reclaim_alloc_args(ObMysqlRequestExecuteTransformPlugin, transaction, req_pkt_len); +} + +ObMysqlRequestExecuteTransformPlugin::ObMysqlRequestExecuteTransformPlugin(ObApiTransaction &transaction, int64_t req_pkt_len) + : ObTransformationPlugin(transaction, ObTransformationPlugin::REQUEST_TRANSFORMATION), + ps_pkt_len_(req_pkt_len), is_handle_finished_(false), param_type_pos_(-1), param_num_(-1), param_offset_(0), + new_param_bound_flag_(-1), param_type_buf_(), ps_id_entry_(NULL), local_reader_(NULL), is_first_analyze_with_flag_(true) +{ + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin born", K(this)); +} + +void ObMysqlRequestExecuteTransformPlugin::destroy() +{ + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin destroy", K(this)); + ObTransformationPlugin::destroy(); + op_reclaim_free(this); +} + +int ObMysqlRequestExecuteTransformPlugin::consume(event::ObIOBufferReader *reader) +{ + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin::consume happen"); + int ret = OB_SUCCESS; + if (OB_ISNULL(reader)) { + ret = OB_INVALID_ARGUMENT; + PROXY_API_LOG(WARN, "invalid argument", K(reader), K(ret)); + } else { + int64_t read_avail = 0; + + if (NULL == local_reader_) { + local_reader_ = reader->clone(); + ObClientSessionInfo &session_info = sm_->get_client_session()->get_session_info(); + if (OB_UNLIKELY(ps_pkt_len_ >= MYSQL_PACKET_MAX_LENGTH)) { + ret = OB_NOT_SUPPORTED; + PROXY_API_LOG(WARN, "we cannot support packet which is larger than 16MB", K_(ps_pkt_len), + K(MYSQL_PACKET_MAX_LENGTH), K(ret)); + } else if (OB_ISNULL(ps_id_entry_ = session_info.get_ps_id_entry()) || !ps_id_entry_->is_valid()) { + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "ps id entry does not exist", KPC(ps_id_entry_), K(ret)); + } else { + if (OB_LIKELY((param_num_ = ps_id_entry_->get_param_count()) > 0)) { + param_type_pos_ = MYSQL_NET_META_LENGTH + MYSQL_PS_EXECUTE_HEADER_LENGTH + ((param_num_ + 7) /8) + 1; + } + } + } + + if (OB_SUCC(ret)) { + read_avail = local_reader_->read_avail(); + if (read_avail > 0) { + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin::save_rewrite_ps_execute::consume", K(read_avail)); + // If there are no parameters, you can directly transfer + if (param_num_ == 0 || is_handle_finished_) { + if (OB_FAIL(produce_data(local_reader_, read_avail))) { + PROXY_API_LOG(WARN, "fail to consume local transfer reader", K(param_num_), K(ret)); + } + } else if (OB_FAIL(analyze_ps_execute_request(read_avail))) { + PROXY_API_LOG(WARN, "fail to analyze ps execute request", K(read_avail), K(ret)); + } + } + } + } + + if (OB_FAIL(ret)) { + sm_->trans_state_.inner_errcode_ = ret; + // if failed set to INTERNAL_ERROR + sm_->trans_state_.current_.state_ = ObMysqlTransact::INTERNAL_ERROR; + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::rewrite_pkt_length_and_flag() +{ + int ret = OB_SUCCESS; + + int32_t pkt_length = static_cast(ps_pkt_len_ - 4); // packet length without header + const ObString& param_type = ps_id_entry_->get_ps_sql_meta().get_param_type(); + if (OB_UNLIKELY(pkt_length <= 0) || OB_UNLIKELY(param_type.empty())) { + ret = OB_INVALID_ARGUMENT; + PROXY_API_LOG(WARN, "invalid pkt_length or param_type is empty", K(pkt_length), K(param_type), K(ret)); + } else { + pkt_length += param_type.length(); + + char header[MYSQL_PAYLOAD_LENGTH_LENGTH]; + int64_t pos = 0; + if (OB_FAIL(obmysql::ObMySQLUtil::store_int3(header, MYSQL_PAYLOAD_LENGTH_LENGTH, pkt_length, pos))) { + PROXY_API_LOG(WARN, "fail to store pkg meta header", K(ret)); + } else { + int64_t new_param_bound_flag_pos = param_type_pos_ - 1; + int8_t new_param_bound_flag = 1; + local_reader_->replace(header, MYSQL_PAYLOAD_LENGTH_LENGTH, 0); + local_reader_->replace(reinterpret_cast(&new_param_bound_flag), 1, new_param_bound_flag_pos); + + sm_->trans_state_.trans_info_.client_request_.get_packet_meta().pkt_len_ = static_cast(ps_pkt_len_ + (param_type.length())); + sm_->trans_state_.trans_info_.request_content_length_ = ps_pkt_len_ + (param_type.length()); + } + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::produce_param_type() +{ + int ret = OB_SUCCESS; + + const ObString& param_type = ps_id_entry_->get_ps_sql_meta().get_param_type(); + if (OB_UNLIKELY(param_type.empty())) { + ret = OB_INVALID_ARGUMENT; + PROXY_API_LOG(WARN, "invalid param_type", K(param_type), K(ret)); + } else { + ObMIOBuffer* mio_new = NULL; + int64_t written_len = 0; + + event::ObIOBufferReader* reader_tmp = NULL; + + if (OB_ISNULL(mio_new = new_miobuffer(param_type.length()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + PROXY_API_LOG(WARN, "fail to alloc miobuffer for reader_buffer_tmp", K(ret)); + } else if (OB_ISNULL(reader_tmp = mio_new->alloc_reader())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + PROXY_API_LOG(WARN, "fail to alloc reader for reader_tmp", K(ret)); + } else if (OB_FAIL(mio_new->write(param_type.ptr(), param_type.length(), written_len))) { + PROXY_API_LOG(WARN, "fail to write to reader_tmp", K(ret)); + } else if (OB_UNLIKELY(written_len != (param_type.length()))) { + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "fail to write to reader_tmp", K(ret)); + } else if (OB_FAIL(produce_data(reader_tmp, written_len))) { + PROXY_API_LOG(WARN, "fail to produce param type", K(ret)); + } + + if (NULL != mio_new) { + free_miobuffer(mio_new); + } + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::analyze_without_flag(const int64_t local_read_avail) +{ + int ret = OB_SUCCESS; + // At this point, the packet length and new_param_bound_flag_ fields are already there, + // you can directly rewrite them, and append the type content + if (OB_FAIL(rewrite_pkt_length_and_flag())) { + PROXY_API_LOG(WARN, "fail to rewrite header length", K(ret)); + // Send out the previous content first + } else if (OB_FAIL(produce_data(local_reader_, param_type_pos_))) { + PROXY_API_LOG(WARN, "fail to produce execute", K(ret)); + // Output type content + } else if (OB_FAIL(produce_param_type())) { + PROXY_API_LOG(WARN, "fail to consume reader_tmp transfer reader", K(ret)); + // Output the remaining content of this time + } else if (local_read_avail > param_type_pos_ && OB_FAIL(produce_data(local_reader_, local_read_avail - param_type_pos_))) { + PROXY_API_LOG(WARN, "fail to consume rewrite transfer reader", K(ret)); + } else { + is_handle_finished_ = true; + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::analyze_with_flag(const int64_t local_read_avail) +{ + int ret = OB_SUCCESS; + int64_t consume_len = local_read_avail; + ObIArray ¶m_types = ps_id_entry_->get_ps_sql_meta().get_param_types(); + + // At this point, the packet length and new_param_bound_flag_ fields are already there, + // and the previous content can be consumed directly + if (is_first_analyze_with_flag_) { + if (OB_FAIL(produce_data(local_reader_, param_type_pos_))) { + PROXY_API_LOG(WARN, "fail to produce local reader", K_(param_type_pos), K(ret)); + } else if (OB_FAIL(param_type_buf_.reserve(2 * param_num_))) { + PROXY_API_LOG(WARN, "fail to reserve param type buf", "param_type_len", 2 * param_num_, K(ret)); + } else { + is_first_analyze_with_flag_ = false; + consume_len -= param_type_pos_; + param_types.reset(); + } + } + + if (OB_SUCC(ret)) { + int64_t analyzed_len = 0; + int64_t param_type_len = 0; + if (OB_FAIL(ObMysqlRequestAnalyzer::parse_param_type_from_reader(param_offset_, param_num_, + param_types, + local_reader_, + analyzed_len, is_handle_finished_))) { + if (OB_LIKELY(ret == OB_SIZE_OVERFLOW)) { + ret = OB_SUCCESS; + } else { + PROXY_API_LOG(WARN, "fail to parse param type from reader", K(ret)); + } + } + + if (OB_SUCC(ret)) { + param_type_len = param_type_buf_.length() + analyzed_len; + if (OB_UNLIKELY(param_type_len > param_type_buf_.capacity())) { + if (OB_FAIL(param_type_buf_.reserve(param_type_len))) { + PROXY_API_LOG(WARN, "fail to reserve param type buf", K(param_type_len), K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + local_reader_->copy(param_type_buf_.ptr() + param_type_buf_.length(), analyzed_len, 0); + if (OB_FAIL(param_type_buf_.set_length(param_type_len))) { + PROXY_API_LOG(WARN, "fail to set param type len", K(param_type_len), K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (is_handle_finished_) { + if (OB_FAIL(ps_id_entry_->get_ps_sql_meta().set_param_type(param_type_buf_.ptr(), param_type_len))) { + PROXY_API_LOG(WARN, "fail to set param type", K(param_type_len), K(ret)); + } + } else { + consume_len = analyzed_len; + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(produce_data(local_reader_, consume_len))) { + PROXY_API_LOG(WARN, "fail to produce local reader", K(local_read_avail), K(consume_len), K(ret)); + } + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::analyze_ps_execute_request(const int64_t local_read_avail) +{ + int ret = OB_SUCCESS; + // You must wait until new_param_bound_flag_ to send data, + // otherwise it is not clear whether to rewrite the header length field + if (-1 == new_param_bound_flag_) { + int64_t new_param_bound_flag_pos = param_type_pos_ - 1; + // Because the data has not been consumed, local_read_avail here is the data from the very beginning + if (local_read_avail > new_param_bound_flag_pos) { + local_reader_->copy(reinterpret_cast(&new_param_bound_flag_), 1, new_param_bound_flag_pos); + if (1 != new_param_bound_flag_ && 0 != new_param_bound_flag_) { + ret = OB_DATA_OUT_OF_RANGE; + PROXY_API_LOG(WARN, "fail to get new_param_bound_flag", K_(new_param_bound_flag), K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + if (1 == new_param_bound_flag_) { + if (OB_FAIL(analyze_with_flag(local_read_avail))) { + PROXY_API_LOG(WARN, "fail to analyze with flag", K(ret)); + } + } else if (0 == new_param_bound_flag_) { + if (OB_FAIL(analyze_without_flag(local_read_avail))) { + PROXY_API_LOG(WARN, "fail to analyze without flag", K(ret)); + } + } + } + + return ret; +} + +int ObMysqlRequestExecuteTransformPlugin::produce_data(event::ObIOBufferReader *reader, + const int64_t produce_size) +{ + int ret = OB_SUCCESS; + + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin::produce_data", K(produce_size)); + + int64_t produce_length = 0; + if (OB_ISNULL(reader)) { + ret = OB_INVALID_ARGUMENT; + PROXY_API_LOG(WARN, "invalid argument", K(reader), K(ret)); + } else if (OB_UNLIKELY(produce_size < 0)) { + ret = OB_INVALID_ARGUMENT; + PROXY_API_LOG(WARN, "invalid argument", K(produce_size), K(ret)); + } else if (OB_UNLIKELY((produce_size > reader->read_avail()))){ + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "produce_size is larger than read_avail", K(produce_size), K(reader->read_avail()), K(ret)); + } else if (OB_UNLIKELY(produce_size != (produce_length = produce(reader, produce_size)))) { + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "fail to produce", "expected size", produce_size, + "actual size", produce_length, K(ret)); + } else if (OB_FAIL(reader->consume(produce_size))) { + PROXY_API_LOG(WARN, "fail to consume local transfer reader", K(produce_size), K(ret)); + } + + return ret; +} + +void ObMysqlRequestExecuteTransformPlugin::handle_input_complete() +{ + PROXY_API_LOG(DEBUG, "ObMysqlRequestExecuteTransformPlugin::handle_input_complete happen"); + if (NULL != local_reader_) { + local_reader_->dealloc(); + local_reader_ = NULL; + } + + set_output_complete(); +} + +} // end of namespace proxy +} // end of namespace obproxy +} // end of namespace oceanbase diff --git a/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.h b/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.h new file mode 100644 index 0000000000000000000000000000000000000000..dd078609eead59512c3cff1041f6dede8c46d9dc --- /dev/null +++ b/src/obproxy/proxy/plugins/ob_mysql_request_execute_transform_plugin.h @@ -0,0 +1,148 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBPROXY_MYSQL_REQUEST_EXECUTE_TRANSFORM_PLUGIN_H +#define OBPROXY_MYSQL_REQUEST_EXECUTE_TRANSFORM_PLUGIN_H + +#include "proxy/api/ob_global_plugin.h" +#include "proxy/api/ob_transformation_plugin.h" +#include "proxy/mysqllib/ob_proxy_session_info_handler.h" +#include "proxy/mysql/ob_mysql_client_session.h" +#include "proxy/mysql/ob_mysql_sm.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace proxy +{ + +class ObMysqlRequestExecuteTransformPlugin : public ObTransformationPlugin +{ +public: + static ObMysqlRequestExecuteTransformPlugin *alloc(ObApiTransaction &transaction, int64_t ps_pkt_len); + + ObMysqlRequestExecuteTransformPlugin(ObApiTransaction &transaction, int64_t ps_pkt_len); + + virtual void destroy(); + + // this func can not consume the reader, super class will do it + virtual int consume(event::ObIOBufferReader *reader); + + virtual void handle_input_complete(); + +private: + int produce_data(event::ObIOBufferReader *reader, const int64_t produce_size); + + int analyze_without_flag(const int64_t local_read_avail); + int analyze_with_flag(const int64_t local_read_avail); + int analyze_ps_execute_request(const int64_t local_read_avail); + + int rewrite_pkt_length_and_flag(); + int produce_param_type(); + +private: + int64_t ps_pkt_len_; + bool is_handle_finished_; + int64_t param_type_pos_; // The starting position of param type in the package + int64_t param_num_; + int64_t param_offset_; + int8_t new_param_bound_flag_; + ObSqlString param_type_buf_; + ObPsIdEntry *ps_id_entry_; + event::ObIOBufferReader *local_reader_; + bool is_first_analyze_with_flag_; + DISALLOW_COPY_AND_ASSIGN(ObMysqlRequestExecuteTransformPlugin); +}; + +class ObMysqlRequestExecuteGlobalPlugin : public ObGlobalPlugin +{ +public: + static ObMysqlRequestExecuteGlobalPlugin *alloc() + { + return op_reclaim_alloc(ObMysqlRequestExecuteGlobalPlugin); + } + + ObMysqlRequestExecuteGlobalPlugin() + { + register_hook(HOOK_READ_REQUEST); + } + + virtual void destroy() + { + ObGlobalPlugin::destroy(); + op_reclaim_free(this); + } + + virtual void handle_read_request(ObApiTransaction &transaction) + { + ObTransactionPlugin *plugin = NULL; + + if (need_enable_plugin(transaction.get_sm())) { + // Contains the size of the header + int64_t req_pkt_len = transaction.get_sm()->trans_state_.trans_info_.request_content_length_; + plugin = ObMysqlRequestExecuteTransformPlugin::alloc(transaction, req_pkt_len); + if (NULL != plugin) { + transaction.add_plugin(plugin); + PROXY_API_LOG(DEBUG, "add ObMysqlRequestExecuteTransformPlugin", K(plugin), K(req_pkt_len)); + } else { + PROXY_API_LOG(ERROR, "fail to allocate memory for ObMysqlRequestExecuteTransformPlugin"); + } + } else { + PROXY_API_LOG(DEBUG, "handle_read_request, no need setup ObMysqlRequestExecuteTransformPlugin"); + } + + transaction.resume(); + } + + inline bool need_enable_plugin(ObMysqlSM *sm) const + { + PROXY_API_LOG(DEBUG, "need_enable_plugin", + "request_content_length", sm->trans_state_.trans_info_.request_content_length_, + "mysql_cmd", get_mysql_cmd_str(sm->trans_state_.trans_info_.sql_cmd_)); + + bool bret = false; + + if (!sm->trans_state_.trans_info_.client_request_.is_internal_cmd() + && sm->trans_state_.trans_info_.request_content_length_ > 0 + && obmysql::OB_MYSQL_COM_STMT_EXECUTE == sm->trans_state_.trans_info_.sql_cmd_) { + ObClientSessionInfo &cs_info = sm->get_client_session()->get_session_info(); + ObPsIdEntry* ps_id_entry = cs_info.get_ps_id_entry(); + if (OB_ISNULL(ps_id_entry)) { + PROXY_API_LOG(WARN, "client ps id entry is null"); + } else { + int64_t param_num = ps_id_entry->get_param_count(); + if (param_num > 0) { + bret = true; + } + } + } + + return bret; + } + +private: + DISALLOW_COPY_AND_ASSIGN(ObMysqlRequestExecuteGlobalPlugin); +}; + +void init_mysql_request_execute_transform() +{ + PROXY_API_LOG(INFO, "init mysql request execute transformation plugin"); + ObMysqlRequestExecuteGlobalPlugin *execute_transform = ObMysqlRequestExecuteGlobalPlugin::alloc(); + UNUSED(execute_transform); +} + +} // end of namespace proxy +} // end of namespace obproxy +} // end of namespace oceanbase + +#endif // OBPROXY_MYSQL_REQUEST_EXECUTE_TRANSFORM_PLUGIN_H diff --git a/src/obproxy/proxy/plugins/ob_mysql_response_compress_transform_plugin.cpp b/src/obproxy/proxy/plugins/ob_mysql_response_compress_transform_plugin.cpp index 0055070d032d2435e9566e45496aee9f20acb323..397ccbbe05b0719479872d462d44423bab69618c 100644 --- a/src/obproxy/proxy/plugins/ob_mysql_response_compress_transform_plugin.cpp +++ b/src/obproxy/proxy/plugins/ob_mysql_response_compress_transform_plugin.cpp @@ -71,10 +71,11 @@ int ObMysqlResponseCompressTransformPlugin::consume(event::ObIOBufferReader *rea if (!analyzer_->is_inited()) { const obmysql::ObMySQLCmd cmd = sm_->trans_state_.trans_info_.client_request_.get_packet_meta().cmd_; + const ObMysqlProtocolMode mysql_mode = sm_->client_session_->get_session_info().is_oracle_mode() ? OCEANBASE_ORACLE_PROTOCOL_MODE : OCEANBASE_MYSQL_PROTOCOL_MODE; const bool extra_ok_packet_for_stats_enabled = sm_->is_extra_ok_packet_for_stats_enabled(); // should decomrpess the data if (OB_FAIL(analyzer_->init(req_seq_, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, - cmd, extra_ok_packet_for_stats_enabled, req_seq_, request_id_, server_sessid_))) { + cmd, mysql_mode, extra_ok_packet_for_stats_enabled, req_seq_, request_id_, server_sessid_))) { PROXY_API_LOG(WARN, "fail to init compress analyzer", K_(req_seq), K_(request_id), K_(server_sessid), K(cmd), K(extra_ok_packet_for_stats_enabled), K(ret)); } diff --git a/src/obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.cpp b/src/obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.cpp index ebdbf2dc8f67e0c2074ff69c09ccbf78cabb99ba..c0efbe4a3bb2d2f331ca1d1d18037973efd3d562 100644 --- a/src/obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.cpp +++ b/src/obproxy/proxy/plugins/ob_mysql_response_cursor_transform_plugin.cpp @@ -224,10 +224,7 @@ int ObMysqlResponseCursorTransformPlugin::handle_resultset_row(event::ObIOBuffer /* first 2 bits are reserved */ ObObj param; if (ObSMUtils::update_from_bitmap(param, bitmap, i + 2)) { - if (OB_MYSQL_TYPE_CURSOR != field_types.at(i)) { - ret = OB_ERR_UNEXPECTED; - PROXY_API_LOG(WARN, "cursor filed is null", K(i), "bitmap", *bitmap); - } + // do nothing } else { if (OB_MYSQL_TYPE_CURSOR == field_types.at(i)) { ObMysqlClientSession *client_session = sm->get_client_session(); diff --git a/src/obproxy/proxy/plugins/ob_mysql_response_new_ps_transform_plugin.cpp b/src/obproxy/proxy/plugins/ob_mysql_response_new_ps_transform_plugin.cpp index c1fa79502fa68240fd5c8a3cf6a7759c4d5c3b85..8b80bf7552db35a9f2ffa4f5b5235a7a5e498392 100644 --- a/src/obproxy/proxy/plugins/ob_mysql_response_new_ps_transform_plugin.cpp +++ b/src/obproxy/proxy/plugins/ob_mysql_response_new_ps_transform_plugin.cpp @@ -186,7 +186,6 @@ int ObMysqlResponsePrepareExecuteTransformPlugin::handle_prepare_column(event::O if (OB_FAIL(pkt_reader_.get_packet(*reader, field_packet))) { PROXY_API_LOG(ERROR, "fail to get filed packet from reader", K(ret)); } else { - pkt_count_++; field_types_.push_back(field.type_); if (OB_MYSQL_TYPE_CURSOR == field.type_) { hava_cursor_ = true; diff --git a/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.cpp b/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.cpp index 2a18b0e8546c1a35fede97ef0724fef6dba47d1b..eadfd4246eb3ea7da484a9f077b12b99fc70de0e 100644 --- a/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.cpp +++ b/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.cpp @@ -169,8 +169,14 @@ int ObMysqlResponsePrepareTransformPlugin::handle_prepare_param(event::ObIOBuffe PROXY_API_LOG(ERROR, "fail to get filed packet from reader", K(ret)); } else { ObClientSessionInfo &cs_info = sm_->get_client_session()->get_session_info(); - ObIArray ¶m_types = cs_info.get_ps_entry()->get_ps_sql_meta().get_param_types(); - param_types.push_back(field.type_); + ObPsIdEntry* ps_id_entry = cs_info.get_ps_id_entry(); + if (OB_ISNULL(ps_id_entry)) { + ret = OB_ERR_UNEXPECTED; + PROXY_API_LOG(WARN, "ps id entry is null", K(ret), KPC(ps_id_entry)); + } else { + ObIArray ¶m_types = ps_id_entry->get_ps_sql_meta().get_param_types(); + param_types.push_back(field.type_); + } } } @@ -192,29 +198,24 @@ int ObMysqlResponsePrepareTransformPlugin::handle_prepare_ok(event::ObIOBufferRe reader->replace(reinterpret_cast(&client_ps_id), sizeof(client_ps_id), MYSQL_NET_META_LENGTH); ObClientSessionInfo &cs_info = sm_->get_client_session()->get_session_info(); - int64_t origin_num_params = cs_info.get_ps_entry()->get_ps_sql_meta().get_param_count(); - int64_t origin_num_columns = cs_info.get_ps_entry()->get_ps_sql_meta().get_column_count(); - - if ((origin_num_params > 0 && OB_UNLIKELY(origin_num_params != num_params_)) - || (origin_num_columns > 0 && OB_UNLIKELY(origin_num_columns != num_columns_))) { + ObPsIdEntry* ps_id_entry = cs_info.get_ps_id_entry(); + if (OB_ISNULL(ps_id_entry)) { ret = OB_ERR_UNEXPECTED; - PROXY_API_LOG(ERROR, "prepare response ok returns different param count or param count", - K_(num_columns), K(origin_num_columns), - K_(num_params), K(origin_num_params), KPC(cs_info.get_ps_entry()), K(ret)); - } else if (num_params_ > 0) { - cs_info.get_ps_entry()->get_ps_sql_meta().set_param_count(num_params_); - prepare_state_ = PREPARE_PARAM; - ObClientSessionInfo &cs_info = sm_->get_client_session()->get_session_info(); - ObIArray ¶m_types = cs_info.get_ps_entry()->get_ps_sql_meta().get_param_types(); - param_types.reset(); - } else if (num_columns_ > 0) { - cs_info.get_ps_entry()->get_ps_sql_meta().set_column_count(num_columns_); - prepare_state_ = PREPARE_COLUMN; + PROXY_API_LOG(WARN, "ps id entry is null", K(ret), KPC(ps_id_entry)); } else { - prepare_state_ = PREPARE_END; + if (num_params_ > 0) { + ps_id_entry->get_ps_sql_meta().set_param_count(num_params_); + prepare_state_ = PREPARE_PARAM; + ObIArray ¶m_types = ps_id_entry->get_ps_sql_meta().get_param_types(); + param_types.reset(); + } else if (num_columns_ > 0) { + ps_id_entry->get_ps_sql_meta().set_column_count(num_columns_); + prepare_state_ = PREPARE_COLUMN; + } else { + prepare_state_ = PREPARE_END; + } + pkt_count_ = 0; } - - pkt_count_ = 0; } return ret; diff --git a/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.h b/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.h index 3d69e65b45988c851cfc2bef8a79f1312be870a1..fa2b8ed267e9fa9ef7b20941ae62bc0d5578f886 100644 --- a/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.h +++ b/src/obproxy/proxy/plugins/ob_mysql_response_prepare_transform_plugin.h @@ -104,6 +104,10 @@ public: inline bool need_enable_plugin(ObMysqlSM *sm) const { + PROXY_API_LOG(DEBUG, "need_enable_plugin", + "send action", sm->trans_state_.current_.send_action_, + "mysql_cmd", get_mysql_cmd_str(sm->trans_state_.trans_info_.sql_cmd_)); + return (!sm->trans_state_.trans_info_.client_request_.is_internal_cmd() && ObMysqlTransact::SERVER_SEND_REQUEST == sm->trans_state_.current_.send_action_ && obmysql::OB_MYSQL_COM_STMT_PREPARE == sm->trans_state_.trans_info_.sql_cmd_); diff --git a/src/obproxy/proxy/route/ob_ldc_location.cpp b/src/obproxy/proxy/route/ob_ldc_location.cpp index 67ef3e1dcf81d0cc156e0cf44c83bd9052ce1140..4c2540b805743fcf4e0e72a421159efc02cd8164 100644 --- a/src/obproxy/proxy/route/ob_ldc_location.cpp +++ b/src/obproxy/proxy/route/ob_ldc_location.cpp @@ -355,6 +355,7 @@ bool ObLDCLocation::is_in_primary_zone(const ObProxyReplicaLocation &replica, int ObLDCLocation::fill_strong_read_location(const ObProxyPartitionLocation *pl, ObLDCLocation &dummy_ldc, ObLDCItem &leader_item, ObLDCLocation &ldc_location, bool &entry_need_update, const bool is_only_readwrite_zone, const bool need_use_dup_replica, + const bool need_skip_leader_item, const ObIArray &ss_info, const ObIArray ®ion_names, const ObString &proxy_primary_zone_name) @@ -423,11 +424,13 @@ int ObLDCLocation::fill_strong_read_location(const ObProxyPartitionLocation *pl, if (replica.is_leader()) { LOG_WARN("fail to find leader in dummy ldc with ldc, maybe someone old, continue use it", K(replica)); - leader_item.set(replica, default_merging_status, default_idc_type, default_zone_type, - true, default_congested_status); - if (need_use_dup_replica) { - if (OB_FAIL(tmp_item_array.push_back(leader_item))) { - LOG_WARN("fail to push_back leader_item", K(leader_item), K(tmp_item_array), K(ret)); + if (!need_skip_leader_item) { + leader_item.set(replica, default_merging_status, default_idc_type, default_zone_type, + true, default_congested_status); + if (need_use_dup_replica) { + if (OB_FAIL(tmp_item_array.push_back(leader_item))) { + LOG_WARN("fail to push_back leader_item", K(leader_item), K(tmp_item_array), K(ret)); + } } } } else if (is_ldc_used) { @@ -445,10 +448,12 @@ int ObLDCLocation::fill_strong_read_location(const ObProxyPartitionLocation *pl, } else { //found it if (replica.is_leader()) { - leader_item = tmp_item; - if (need_use_dup_replica) { - if (OB_FAIL(tmp_item_array.push_back(leader_item))) { - LOG_WARN("fail to push_back leader_item", K(leader_item), K(tmp_item_array), K(ret)); + if (!need_skip_leader_item) { + leader_item = tmp_item; + if (need_use_dup_replica) { + if (OB_FAIL(tmp_item_array.push_back(leader_item))) { + LOG_WARN("fail to push_back leader_item", K(leader_item), K(tmp_item_array), K(ret)); + } } } } else if (OB_FAIL(tmp_item_array.push_back(tmp_item))) { diff --git a/src/obproxy/proxy/route/ob_ldc_location.h b/src/obproxy/proxy/route/ob_ldc_location.h index 501665d721dc8344251f2d5fb660c146395ed484..27d06b621bcdb59d8c0d128a13c6b7a3732c1232 100644 --- a/src/obproxy/proxy/route/ob_ldc_location.h +++ b/src/obproxy/proxy/route/ob_ldc_location.h @@ -134,8 +134,11 @@ public: pl_(NULL), ts_(NULL), safe_snapshot_mananger_(NULL), readonly_exist_status_(READONLY_ZONE_UNKNOWN), use_ldc_(false), idc_name_(), idc_name_buf_(), random_() - { } - ~ObLDCLocation() { reset(); } + {} + ~ObLDCLocation() + { + reset(); + } enum ObRegionMatchedType { @@ -204,6 +207,7 @@ public: bool &entry_need_update, const bool is_only_readwrite_zone, const bool need_use_dup_replica, + const bool need_skip_leader_item, const common::ObIArray &ss_info, const common::ObIArray ®ion_names, const common::ObString &proxy_primary_zone_name); diff --git a/src/obproxy/proxy/route/ob_mysql_route.cpp b/src/obproxy/proxy/route/ob_mysql_route.cpp index 11873a2a66bfe2d3793de2781731b192ca5b883d..0fc3cdb67b15d032c2cb62e382604f96c2418f67 100644 --- a/src/obproxy/proxy/route/ob_mysql_route.cpp +++ b/src/obproxy/proxy/route/ob_mysql_route.cpp @@ -578,7 +578,8 @@ inline void ObMysqlRoute::setup_partition_id_calc() } } else { result = ¶m_.client_request_->get_parse_result(); - if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == param_.client_request_->get_packet_meta().cmd_) { + if (obmysql::OB_MYSQL_COM_STMT_EXECUTE == param_.client_request_->get_packet_meta().cmd_ + || obmysql::OB_MYSQL_COM_STMT_SEND_LONG_DATA == param_.client_request_->get_packet_meta().cmd_) { if (OB_FAIL(param_.client_info_->get_ps_sql(user_sql))) { LOG_WARN("fail to get ps sql", K(ret)); ret = OB_SUCCESS; diff --git a/src/obproxy/proxy/route/ob_partition_processor.cpp b/src/obproxy/proxy/route/ob_partition_processor.cpp index c60ad4ca241adcf96b28e463db8daeac4ce8f456..e2ac35571ce6234353c4095b58fd1f4e1d67669f 100644 --- a/src/obproxy/proxy/route/ob_partition_processor.cpp +++ b/src/obproxy/proxy/route/ob_partition_processor.cpp @@ -200,6 +200,7 @@ int ObPartitionEntryCont::main_handler(int event, void *data) data = NULL; // fall through } + __attribute__ ((fallthrough)); case CLIENT_TRANSPORT_MYSQL_RESP_EVENT: { if (OB_FAIL(handle_client_resp(data))) { LOG_WARN("fail to handle client resp", K(ret)); diff --git a/src/obproxy/proxy/route/ob_route_struct.h b/src/obproxy/proxy/route/ob_route_struct.h index 4d30f20df4de651267a301cf4a6340a149e14700..4c016fbca01a8ccaa1c5b882b3cfa3b26e02dc6a 100644 --- a/src/obproxy/proxy/route/ob_route_struct.h +++ b/src/obproxy/proxy/route/ob_route_struct.h @@ -580,7 +580,7 @@ inline bool ObTableEntryName::is_oceanbase_db() const inline bool ObTableEntryName::is_all_dummy_table() const { static const common::ObString all_dummy_tname_str(share::OB_ALL_DUMMY_TNAME); - return is_valid() && all_dummy_tname_str == table_name_; + return is_valid() && table_name_[0] == '_' && all_dummy_tname_str == table_name_; } inline bool ObTableEntryName::is_ob_dummy() const diff --git a/src/obproxy/proxy/route/ob_route_utils.cpp b/src/obproxy/proxy/route/ob_route_utils.cpp index 7ddc75528914b4338a82f5e60776bd0815e50f3d..322fbdf03bd87def67864700286d892bcf1262dc 100644 --- a/src/obproxy/proxy/route/ob_route_utils.cpp +++ b/src/obproxy/proxy/route/ob_route_utils.cpp @@ -28,6 +28,7 @@ #include "opsql/func_expr_parser/ob_func_expr_parser.h" #include "opsql/func_expr_parser/ob_func_expr_parser_utils.h" #include "obutils/ob_proxy_sql_parser.h" +#include "utils/ob_proxy_utils.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -67,7 +68,7 @@ static const char *PROXY_PART_INFO_SQL = "part_interval_bin, interval_start_bin, sub_part_num, sub_part_type, sub_part_space, " "sub_part_expr, def_sub_part_interval_bin, def_sub_interval_start_bin, " "part_key_num, part_key_name, part_key_type, part_key_idx, part_key_level, part_key_extra, " - "spare1, spare2, spare4 " + "spare1, spare2, spare4, spare5 " "FROM oceanbase.%s " "WHERE table_id = %lu order by part_key_idx LIMIT %d;"; @@ -279,7 +280,7 @@ int ObRouteUtils::fetch_table_entry(ObResultSetFetcher &rs_fetcher, ObSEArray server_list; const bool is_dummy_entry = entry.is_dummy_entry(); bool use_fake_addrs = false; - bool has_dup_replica = true; + bool has_dup_replica = false; while ((OB_SUCC(ret)) && (OB_SUCC(rs_fetcher.next()))) { ip_str[0] = '\0'; @@ -560,6 +561,97 @@ int ObRouteUtils::build_part_desc(ObProxyPartInfo &part_info, return ret; } +/* + * according to rs observer, spare5 will specify the accuracy of all the part key + * set this if the part_key_type is valid, otherwise maybe invalid value + * accuracy is different between diff types, for eg: "6" for TIMESTAMP, "(2, 5)" for NUMBER + * see more info in ObProxyPartKeyAccuracy + * + * format: "length,precision/length_semantics,scale", type: "int32_t,int16_t,int16_t" + * the guide of how to handle the accuracy of part key type is reffered to yuque. + * + * return: no ret for this function, cause we need to continue our part info build procedure, + * even if the accuracy parse failed or invalid value. + */ +void ObRouteUtils::parse_part_key_accuracy(ObProxyPartKey *part_key, + ObObjType part_key_type, + ObIAllocator *allocator, + ObString &part_key_accuracy) +{ + int ret = OB_SUCCESS; + + if (part_key_accuracy.empty()) { + ret = OB_INVALID_ARGUMENT; + LOG_DEBUG("empty part key accuracy from rs observer"); + } else { + // alloc buf to maintain string + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator->alloc(part_key_accuracy.length() + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc buf", K(ret), K(part_key_accuracy.length())); + } else { + memcpy(buf, part_key_accuracy.ptr(), part_key_accuracy.length()); + buf[part_key_accuracy.length()] = '\0'; + } + + if (OB_SUCC(ret)) { + const char *delim = ","; + char *token = NULL; + char *saveptr = NULL; + int64_t nums[3] = {0}; + + token = strtok_r(buf, delim, &saveptr); + int i; + for (i = 0; OB_SUCC(ret) && token != NULL && i < 3; ++i) { + if (OB_FAIL(get_int_value(ObString::make_string(token), nums[i]))) { + LOG_WARN("fail to get int value from each token", K(ret), K(token)); + } + token = strtok_r(NULL, delim, &saveptr); + } + + // succ to parse the triple + if (i != 3 || OB_FAIL(ret)) { + LOG_WARN("invalid token or get failed from rs observer", K(i), K(ret)); + } else { + int32_t length = static_cast(nums[0]); + int16_t precision = static_cast(nums[1]); + int16_t scale = static_cast(nums[2]); + + part_key->accuracy_.length_ = -1; // init -1 means not used + part_key->accuracy_.precision_ = -1; + part_key->accuracy_.scale_ = -1; + + // use accord to obj_type + if (ob_is_otimestamp_type(part_key_type)) { + if (scale < MIN_SCALE_FOR_TEMPORAL || scale > MAX_SCALE_FOR_ORACLE_TEMPORAL) { + part_key->accuracy_.scale_ = DEFAULT_SCALE_FOR_ORACLE_TIMESTAMP; + LOG_WARN("invalid scale for timestamp in oracle, set to default:6", K(scale)); + } else { + part_key->accuracy_.scale_ = static_cast(scale); // timestamp only need scale + LOG_DEBUG("succ to set timestamp scale of accuracy", K(scale)); + } + } else if (ob_is_number_tc(part_key_type) + || ob_is_datetime_tc(part_key_type) + || ob_is_time_tc(part_key_type)) { + part_key->accuracy_.precision_ = precision; + part_key->accuracy_.scale_ = scale; + } else if (ob_is_string_tc(part_key_type)) { + part_key->accuracy_.length_ = length; + part_key->accuracy_.precision_ = precision; + } + // more obj type could be supported here. + } + } + + if (OB_NOT_NULL(buf)) { + allocator->free(buf); + buf = NULL; + } + } + + return; +} + inline int ObRouteUtils::fetch_part_key(ObResultSetFetcher &rs_fetcher, ObProxyPartInfo &part_info) { @@ -575,6 +667,7 @@ inline int ObRouteUtils::fetch_part_key(ObResultSetFetcher &rs_fetcher, ObString part_key_extra; ObString constraint_part_key; int64_t idx_in_rowid = -1; + ObString part_key_accuracy; PROXY_EXTRACT_INT_FIELD_MYSQL(rs_fetcher, "part_key_level", part_key_level, ObPartitionLevel); // part key idx is the order of part key in all columns @@ -588,15 +681,22 @@ inline int ObRouteUtils::fetch_part_key(ObResultSetFetcher &rs_fetcher, PROXY_EXTRACT_INT_FIELD_MYSQL(rs_fetcher, "spare1", part_key_cs_type, ObCollationType); // use spare2 as rowid index PROXY_EXTRACT_INT_FIELD_MYSQL(rs_fetcher, "spare2", idx_in_rowid, int64_t); + // use spare5 as the accuracy of the part key + PROXY_EXTRACT_VARCHAR_FIELD_MYSQL(rs_fetcher, "spare5", part_key_accuracy); + LOG_DEBUG("fetch part key", K(part_key_level), K(part_key_idx), K(part_key_type), K(part_key_name), + K(part_key_extra), K(constraint_part_key), K(part_key_cs_type), K(idx_in_rowid), K(part_key_accuracy)); + if (!is_obj_type_supported(part_key_type)) { part_info.set_unknown_part_key(true); } + ObProxyPartKey *part_key = &part_key_info.part_keys_[part_key_info.key_num_]; + if (PARTITION_LEVEL_ONE == part_key_level) { - part_key_info.part_keys_[part_key_info.key_num_].level_ = PART_KEY_LEVEL_ONE; + part_key->level_ = PART_KEY_LEVEL_ONE; } else if (PARTITION_LEVEL_TWO == part_key_level) { - part_key_info.part_keys_[part_key_info.key_num_].level_ = PART_KEY_LEVEL_TWO; + part_key->level_ = PART_KEY_LEVEL_TWO; } else { ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; LOG_WARN("part key level is invalid", K(part_key_level), K(ret)); @@ -611,20 +711,22 @@ inline int ObRouteUtils::fetch_part_key(ObResultSetFetcher &rs_fetcher, } if (OB_SUCC(ret)) { - part_key_info.part_keys_[part_key_info.key_num_].idx_ = part_key_idx; - part_key_info.part_keys_[part_key_info.key_num_].name_.str_len_ = part_key_name.length(); - part_key_info.part_keys_[part_key_info.key_num_].name_.str_ = buf; - part_key_info.part_keys_[part_key_info.key_num_].obj_type_ = part_key_type; - part_key_info.part_keys_[part_key_info.key_num_].idx_in_rowid_ = idx_in_rowid; + part_key->idx_ = part_key_idx; + part_key->name_.str_len_ = part_key_name.length(); + part_key->name_.str_ = buf; + part_key->obj_type_ = part_key_type; + part_key->idx_in_rowid_ = idx_in_rowid; + + parse_part_key_accuracy(part_key, part_key_type, &allocator, part_key_accuracy); + if (CS_TYPE_INVALID == part_key_cs_type) { - part_key_info.part_keys_[part_key_info.key_num_].cs_type_ = - ObCharset::get_default_collation(ObCharset::get_default_charset()); + part_key->cs_type_ = ObCharset::get_default_collation(ObCharset::get_default_charset()); } else { - part_key_info.part_keys_[part_key_info.key_num_].cs_type_ = part_key_cs_type; + part_key->cs_type_ = part_key_cs_type; } if (!part_key_extra.empty() || !constraint_part_key.empty()) { - part_key_info.part_keys_[part_key_info.key_num_].is_generated_ = true; + part_key->is_generated_ = true; part_info.set_has_generated_key(true); int64_t generated_key_idx = part_key_info.key_num_; ++part_key_info.key_num_; @@ -643,7 +745,7 @@ inline int ObRouteUtils::fetch_part_key(ObResultSetFetcher &rs_fetcher, LOG_WARN("fail to add generated key", K(constraint_part_key), K(ret)); } } else { - part_key_info.part_keys_[part_key_info.key_num_].is_generated_ = false; + part_key->is_generated_ = false; ++part_key_info.key_num_; } } @@ -1240,7 +1342,7 @@ int ObRouteUtils::get_routine_entry_sql(char *sql_buf, const int64_t buf_len, } else { len = static_cast(snprintf(sql_buf, OB_SHORT_SQL_LENGTH, PROXY_ROUTINE_SCHEMA_SQL, OB_ALL_VIRTUAL_PROXY_SCHEMA_TNAME, - name.table_name_.length(), name.table_name_.ptr(), + new_tenant_name.length(), new_tenant_name.ptr(), name.database_name_.length(), name.database_name_.ptr(), name.table_name_.length(), name.table_name_.ptr())); } diff --git a/src/obproxy/proxy/route/ob_route_utils.h b/src/obproxy/proxy/route/ob_route_utils.h index fd39189aa8ae8825bfea5eb5fcd11ed11e533265..3aef1be171be674c224a10d7a8b0646324cde11d 100644 --- a/src/obproxy/proxy/route/ob_route_utils.h +++ b/src/obproxy/proxy/route/ob_route_utils.h @@ -23,6 +23,7 @@ namespace common { class ObString; class ObAddr; +class ObIAllocator; } namespace obproxy { @@ -92,6 +93,8 @@ public: private: static int fetch_part_key(obproxy::ObResultSetFetcher &rs_fetcher, ObProxyPartInfo &part_info); + static void parse_part_key_accuracy(ObProxyPartKey *part_key, common::ObObjType part_key_type, + common::ObIAllocator *allocator, ObString &part_key_accuracy); static int add_generated_part_key(const common::ObString &func_expr, const int64_t generated_key_idx, ObProxyPartInfo &part_info); diff --git a/src/obproxy/proxy/route/ob_routine_processor.cpp b/src/obproxy/proxy/route/ob_routine_processor.cpp index 2f00b093f9a2334eaac6a6c978ee2587f8b37738..6fd4deaad3328ba8666f43eaedf601cd04012ef2 100644 --- a/src/obproxy/proxy/route/ob_routine_processor.cpp +++ b/src/obproxy/proxy/route/ob_routine_processor.cpp @@ -192,11 +192,13 @@ int ObRoutineEntryCont::main_handler(int event, void *data) } break; } + __attribute__ ((fallthrough)); case ROUTINE_ENTRY_FAIL_SCHEDULE_LOOKUP_REMOTE_EVENT: { // fail to schedule, data must be NULL data = NULL; // fall through } + __attribute__ ((fallthrough)); case CLIENT_TRANSPORT_MYSQL_RESP_EVENT: { if (OB_FAIL(handle_client_resp(data))) { LOG_WARN("fail to handle client resp", K(ret)); diff --git a/src/obproxy/proxy/route/ob_server_route.cpp b/src/obproxy/proxy/route/ob_server_route.cpp index 75e215c5413fb560454342e41591c6f74e044e4a..8561a67d3e77b8d5e80b78ece188f4879a8c07a2 100644 --- a/src/obproxy/proxy/route/ob_server_route.cpp +++ b/src/obproxy/proxy/route/ob_server_route.cpp @@ -36,7 +36,8 @@ int64_t ObServerRoute::to_string(char *buf, const int64_t buf_len) const K_(need_use_dup_replica), KP_(table_entry), KP_(part_entry), - KP_(dummy_entry)); + KP_(dummy_entry), + K_(skip_leader_item)); J_OBJ_END(); return pos; diff --git a/src/obproxy/proxy/route/ob_server_route.h b/src/obproxy/proxy/route/ob_server_route.h index 32c963873050f3919069cb7285f067d0fd847dc7..7b738156280ace2234380529e380da3129b4fd18 100644 --- a/src/obproxy/proxy/route/ob_server_route.h +++ b/src/obproxy/proxy/route/ob_server_route.h @@ -36,7 +36,7 @@ public: has_dup_replica_(false), need_use_dup_replica_(false), consistency_level_(common::INVALID_CONSISTENCY), leader_item_(), ldc_route_(), valid_count_(0), cur_chosen_server_(), - cur_chosen_route_type_(ROUTE_TYPE_MAX) {} + cur_chosen_route_type_(ROUTE_TYPE_MAX), skip_leader_item_(false) {} ~ObServerRoute() { reset(); }; inline void reset(); @@ -127,6 +127,7 @@ public: int64_t valid_count_; ObLDCItem cur_chosen_server_; ObRouteType cur_chosen_route_type_; + bool skip_leader_item_; private: DISALLOW_COPY_AND_ASSIGN(ObServerRoute); }; @@ -137,6 +138,7 @@ inline void ObServerRoute::reset() is_part_entry_from_remote_ = false; has_dup_replica_ = false; need_use_dup_replica_ = false; + skip_leader_item_ = false; set_dummy_entry(NULL); set_table_entry(NULL); set_part_entry(NULL); @@ -192,6 +194,7 @@ inline int ObServerRoute::fill_strong_read_replica(const ObProxyPartitionLocatio ldc_route_.location_, entry_need_update, is_only_readwrite_zone, need_use_dup_replica_, + skip_leader_item_, ss_info, region_names, proxy_primary_zone_name))) { PROXY_LOG(WARN, "fail to divide_leader_replica", K(ret)); diff --git a/src/obproxy/proxy/route/ob_table_cache.h b/src/obproxy/proxy/route/ob_table_cache.h index 0482dea88eba64ae72ce4b38e729353522bcdfca..d5556fe4d11d5bd192a4811ba08fc58876b50579 100644 --- a/src/obproxy/proxy/route/ob_table_cache.h +++ b/src/obproxy/proxy/route/ob_table_cache.h @@ -104,7 +104,7 @@ inline void ObTableCache::set_cache_expire_time(const int64_t relative_time_ms) bool ObTableCache::is_table_entry_expired(const ObTableEntry &entry) { - return is_table_entry_expired_in_time_mode(entry) || is_table_entry_expired_in_qa_mode(entry); + return is_table_entry_expired_in_time_mode(entry) || is_table_entry_expired_in_qa_mode(entry); } bool ObTableCache::is_table_entry_expired_in_qa_mode(const ObTableEntry &entry) diff --git a/src/obproxy/proxy/route/ob_table_entry_cont.cpp b/src/obproxy/proxy/route/ob_table_entry_cont.cpp index caef224527bcb223e9ee909bd748dd27e2f8254b..4fafaf1c5b2cfe030a91a7017064f5ec76dbc89e 100644 --- a/src/obproxy/proxy/route/ob_table_entry_cont.cpp +++ b/src/obproxy/proxy/route/ob_table_entry_cont.cpp @@ -280,6 +280,7 @@ inline int ObTableEntryCont::main_handler(int event, void *data) data = NULL; // fail through, do not break } + __attribute__ ((fallthrough)); case CLIENT_TRANSPORT_MYSQL_RESP_EVENT: { if (OB_FAIL(handle_client_resp(data))) { LOG_WARN("fail to handle client resp", K(ret)); diff --git a/src/obproxy/proxy/route/ob_table_processor.cpp b/src/obproxy/proxy/route/ob_table_processor.cpp index d4db746fb8f55792b97ebb8a514f161adbea0344..4ced5ae957996af3101cd2a03b2b4edaa792159c 100644 --- a/src/obproxy/proxy/route/ob_table_processor.cpp +++ b/src/obproxy/proxy/route/ob_table_processor.cpp @@ -40,6 +40,7 @@ namespace obproxy { namespace proxy { + int ObTableProcessor::init(ObTableCache *table_cache) { int ret = OB_SUCCESS; @@ -340,21 +341,21 @@ int ObTableProcessor::get_table_entry_from_thread_cache( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid input value", K(table_param), K(ret)); } else { - if (!table_param.need_fetch_remote()) { + if (OB_LIKELY(!table_param.need_fetch_remote())) { // find entry from thread cache ObTableRefHashMap &table_map = self_ethread().get_table_map(); ObTableEntry *tmp_entry = NULL; ObTableEntryKey key(table_param.name_, table_param.cr_version_, table_param.cr_id_); tmp_entry = table_map.get(key); // get will inc entry's ref - if (NULL != tmp_entry) { + if (OB_LIKELY(NULL != tmp_entry)) { bool find_succ = false; if (tmp_entry->is_deleted_state()) { LOG_DEBUG("this table entry in thread cache has deleted", KPC(tmp_entry)); - } else if (table_cache.is_table_entry_expired(*tmp_entry)) { + } else if (OB_UNLIKELY(table_cache.is_table_entry_expired(*tmp_entry))) { // table entry has expired LOG_DEBUG("the table entry in thread cache is expired", "expire_time_us", table_cache.get_cache_expire_time_us(), KPC(tmp_entry), K(table_param)); - } else if (tmp_entry->is_avail_state() || tmp_entry->is_updating_state()) { // avail + } else if (OB_LIKELY(tmp_entry->is_avail_state() || tmp_entry->is_updating_state())) { // avail find_succ = true; } else if (tmp_entry->is_building_state()) { LOG_ERROR("building state table entry can not in thread cache", KPC(tmp_entry)); @@ -516,7 +517,7 @@ int ObTableProcessor::get_table_entry(ObTableRouteParam &table_param, ObAction * if (OB_FAIL(get_table_entry_from_thread_cache(table_param, *table_cache_, tmp_entry))) { LOG_WARN("fail to get table entry in thread cache", K(table_param), K(ret)); } else { - if (NULL != tmp_entry) { + if (OB_LIKELY(NULL != tmp_entry)) { PROCESSOR_INCREMENT_DYN_STAT(GET_PL_FROM_THREAD_CACHE_HIT); ROUTE_PROMETHEUS_STAT(table_param.name_, PROMETHEUS_ENTRY_LOOKUP_COUNT, TBALE_ENTRY, true, true); tmp_entry->renew_last_access_time(); diff --git a/src/obproxy/proxy/route/obproxy_expr_calculator.cpp b/src/obproxy/proxy/route/obproxy_expr_calculator.cpp index 999ed79305705fa157ffa6e843b2389bfdbfe6c3..07338cb367a92f436bd29742beade2087c40403c 100644 --- a/src/obproxy/proxy/route/obproxy_expr_calculator.cpp +++ b/src/obproxy/proxy/route/obproxy_expr_calculator.cpp @@ -21,6 +21,10 @@ #include "proxy/mysql/ob_prepare_statement_struct.h" #include "lib/rowid/ob_urowid.h" #include "obproxy/utils/ob_proxy_utils.h" +#include "share/part/ob_part_desc.h" +#include "rpc/obmysql/ob_mysql_packet.h" +#include "lib/timezone/ob_time_convert.h" +#include "lib/timezone/ob_timezone_info.h" using namespace oceanbase::common; @@ -47,7 +51,7 @@ int ObProxyExprCalculator::calculate_partition_id(common::ObArenaAllocator &allo } } if (OB_INVALID_INDEX == partition_id && parse_result.has_simple_route_info()) { - if (OB_FAIL(calc_part_id_with_simple_route_info(allocator, parse_result, part_info, partition_id))) { + if (OB_FAIL(calc_part_id_with_simple_route_info(allocator, parse_result, client_info, part_info, partition_id))) { LOG_WARN("fail to calc part id with simple part info, will do calc in normal path", K(ret)); } } @@ -56,13 +60,17 @@ int ObProxyExprCalculator::calculate_partition_id(common::ObArenaAllocator &allo expr_parse_result.is_oracle_mode_ = client_info.is_oracle_mode(); ObExprResolverResult resolve_result; const common::ObString &print_sql = ObProxyMysqlRequest::get_print_sql(req_sql); - ObPsEntry *ps_entry = NULL; + ObPsIdEntry *ps_id_entry = NULL; ObTextPsEntry *text_ps_entry = NULL; - if (OB_MYSQL_COM_STMT_EXECUTE == client_request.get_packet_meta().cmd_) { - // parse execute param value - if (OB_ISNULL(ps_entry = client_info.get_ps_entry())) { + ObMySQLCmd cmd = client_request.get_packet_meta().cmd_; + + if (OB_MYSQL_COM_STMT_EXECUTE == cmd || OB_MYSQL_COM_STMT_SEND_LONG_DATA == cmd) { + // parse execute param value for OB_MYSQL_COM_STMT_EXECUTE + // try to get param types from OB_MYSQL_COM_STMT_EXECUTE while handling OB_MYSQL_COM_STMT_SEND_LONG_DATA + ps_id_entry = client_info.get_ps_id_entry(); + if (OB_ISNULL(ps_id_entry)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("client ps entry is null", K(ret)); + LOG_WARN("client ps id entry is null", K(ret)); } } else if (parse_result.is_text_ps_execute_stmt()) { if (OB_ISNULL(text_ps_entry = client_info.get_text_ps_entry())) { @@ -71,21 +79,24 @@ int ObProxyExprCalculator::calculate_partition_id(common::ObArenaAllocator &allo } } - if (FAILEDx(do_expr_parse(req_sql, parse_result, part_info, allocator, expr_parse_result, - static_cast(client_info.get_collation_connection())))) { - LOG_INFO("fail to do expr parse", K(print_sql), - K(part_info), "expr_parse_result", ObExprParseResultPrintWrapper(expr_parse_result)); - } else if (OB_FAIL(do_expr_resolve(expr_parse_result, client_request, &client_info, ps_entry, - text_ps_entry, part_info, allocator, resolve_result))) { - LOG_INFO("fail to do expr resolve", K(print_sql), - "expr_parse_result", ObExprParseResultPrintWrapper(expr_parse_result), - K(part_info), KPC(ps_entry), KPC(text_ps_entry), K(resolve_result)); - } else if (OB_FAIL(do_partition_id_calc(resolve_result, part_info, allocator, partition_id))) { - if (OB_MYSQL_COM_STMT_PREPARE != client_request.get_packet_meta().cmd_) { - LOG_INFO("fail to do expr resolve", K(print_sql), K(resolve_result), K(part_info)); + if (OB_SUCC(ret)) { + if (OB_FAIL(do_expr_parse(req_sql, parse_result, part_info, allocator, expr_parse_result, + static_cast(client_info.get_collation_connection())))) { + LOG_INFO("fail to do expr parse", K(print_sql), K(part_info), "expr_parse_result", + ObExprParseResultPrintWrapper(expr_parse_result)); + } else if (OB_FAIL(do_expr_resolve(expr_parse_result, client_request, &client_info, ps_id_entry, + text_ps_entry, part_info, allocator, resolve_result))) { + LOG_INFO("fail to do expr resolve", K(print_sql), "expr_parse_result", + ObExprParseResultPrintWrapper(expr_parse_result), + K(part_info), KPC(ps_id_entry), KPC(text_ps_entry), K(resolve_result)); + } else if (OB_FAIL(do_partition_id_calc(resolve_result, client_info, part_info, parse_result, + allocator, partition_id))) { + if (OB_MYSQL_COM_STMT_PREPARE != cmd) { + LOG_INFO("fail to do expr resolve", K(print_sql), K(resolve_result), K(part_info)); + } + } else { + /* do nothing */ } - } else { - // do nothing here } if (OB_FAIL(ret)) { @@ -107,6 +118,7 @@ int ObProxyExprCalculator::calculate_partition_id(common::ObArenaAllocator &allo int ObProxyExprCalculator::calc_part_id_with_simple_route_info(ObArenaAllocator &allocator, const ObSqlParseResult &parse_result, + ObClientSessionInfo &client_info, ObProxyPartInfo &part_info, int64_t &part_id) { @@ -121,7 +133,8 @@ int ObProxyExprCalculator::calc_part_id_with_simple_route_info(ObArenaAllocator ObExprResolverResult resolve_result; if (OB_FAIL(do_resolve_with_part_key(parse_result, allocator, resolve_result))) { LOG_WARN("fail to do_resolve_with_part_key", K(ret)); - } else if (OB_FAIL(do_partition_id_calc(resolve_result, part_info, allocator, part_id))) { + } else if (OB_FAIL(do_partition_id_calc(resolve_result, client_info, part_info, + parse_result, allocator, part_id))) { LOG_INFO("fail to do_partition_id_calc", K(resolve_result), K(part_info)); } } @@ -138,7 +151,7 @@ int ObProxyExprCalculator::do_expr_parse(const common::ObString &req_sql, int ret = OB_SUCCESS; // do parse - ObExprParseMode parse_mode = INVLIAD_PARSE_MODE; + ObExprParseMode parse_mode = INVALID_PARSE_MODE; if (parse_result.is_select_stmt() || parse_result.is_delete_stmt() || parse_result.is_text_ps_select_stmt() || parse_result.is_text_ps_delete_stmt()) { @@ -215,7 +228,7 @@ int ObProxyExprCalculator::do_resolve_with_part_key(const ObSqlParseResult &pars int ObProxyExprCalculator::do_expr_resolve(ObExprParseResult &parse_result, const ObProxyMysqlRequest &client_request, ObClientSessionInfo *client_info, - ObPsEntry *ps_entry, + ObPsIdEntry *ps_id_entry, ObTextPsEntry *text_ps_entry, ObProxyPartInfo &part_info, ObIAllocator &allocator, @@ -226,7 +239,7 @@ int ObProxyExprCalculator::do_expr_resolve(ObExprParseResult &parse_result, ctx.relation_info_ = &parse_result.relation_info_; ctx.part_info_ = &part_info; ctx.client_request_ = const_cast(&client_request); - ctx.ps_entry_ = ps_entry; + ctx.ps_id_entry_ = ps_id_entry; ctx.text_ps_entry_ = text_ps_entry; ctx.client_info_ = client_info; @@ -246,7 +259,9 @@ int ObProxyExprCalculator::do_expr_resolve(ObExprParseResult &parse_result, } int ObProxyExprCalculator::do_partition_id_calc(ObExprResolverResult &resolve_result, + ObClientSessionInfo &session_info, ObProxyPartInfo &part_info, + const ObSqlParseResult &parse_result, ObIAllocator &allocator, int64_t &partition_id) { @@ -255,10 +270,12 @@ int ObProxyExprCalculator::do_partition_id_calc(ObExprResolverResult &resolve_re int64_t first_part_id = OB_INVALID_INDEX; int64_t sub_part_id = OB_INVALID_INDEX; if (part_info.has_first_part()) { + ObPartDescCtx ctx(&session_info, parse_result.is_insert_stmt()); ObSEArray part_ids; if (OB_FAIL(part_mgr.get_first_part(resolve_result.ranges_[PARTITION_LEVEL_ONE - 1], allocator, - part_ids))) { + part_ids, + ctx))) { LOG_DEBUG("fail to get first part", K(ret)); } else if (part_ids.count() >= 1) { first_part_id = part_ids[0]; @@ -276,7 +293,8 @@ int ObProxyExprCalculator::do_partition_id_calc(ObExprResolverResult &resolve_re } else if (OB_FAIL(part_mgr.get_sub_part(resolve_result.ranges_[PARTITION_LEVEL_TWO - 1], allocator, sub_part_desc_ptr, - sub_part_ids))) { + sub_part_ids, + ctx))) { LOG_WARN("fail to get sub part", K(ret)); } else if (sub_part_ids.count() >= 1) { sub_part_id = sub_part_ids[0]; @@ -419,3 +437,98 @@ int ObProxyExprCalculator::calc_partition_id_using_rowid(const ObExprParseResult return ret; } + +int ObExprCalcTool::build_dtc_params_with_tz_info(ObClientSessionInfo *session_info, + ObObjType obj_type, + ObTimeZoneInfo &tz_info, + ObDataTypeCastParams &dtc_params) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(build_tz_info(session_info, obj_type, tz_info))) { + LOG_WARN("fail to build tz info", K(ret)); + } else if (OB_FAIL(build_dtc_params(session_info, obj_type, dtc_params))) { + LOG_WARN("fail to build dtc params", K(ret)); + } else { + dtc_params.tz_info_ = &tz_info; + } + + return ret; +} + +/* + * for ObTimestampLTZType, input timestamp string, and we also need time_zone from session + * in order to decide the absolutely time + */ +int ObExprCalcTool::build_tz_info(ObClientSessionInfo *session_info, + ObObjType obj_type, + ObTimeZoneInfo &tz_info) +{ + int ret = OB_SUCCESS; + + if (ObTimestampLTZType == obj_type) { + ObObj value_obj; + ObString sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_TIME_ZONE); + if (OB_FAIL(session_info->get_sys_variable_value(sys_key_name, value_obj))) { + LOG_WARN("fail to get sys var from session", K(ret), K(sys_key_name)); + } else { + ObString value_str = value_obj.get_string(); + if (OB_FAIL(tz_info.set_timezone(value_str))) { + LOG_WARN("fail to set time zone for tz_info", K(ret), K(value_str)); + } else { + LOG_DEBUG("succ to set time zone for tz_info", K(value_str)); + } + } + } + + return ret; +} + +int ObExprCalcTool::build_dtc_params(ObClientSessionInfo *session_info, + ObObjType obj_type, + ObDataTypeCastParams &dtc_params) +{ + int ret = OB_SUCCESS; + + if (OB_NOT_NULL(session_info)) { + ObString sys_key_name; + switch (obj_type) { + case ObDateTimeType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_DATE_FORMAT); + break; + case ObTimestampNanoType: + case ObTimestampLTZType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_TIMESTAMP_FORMAT); + break; + case ObTimestampTZType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_TIMESTAMP_TZ_FORMAT); + break; + default: + break; + } + + if (!sys_key_name.empty()) { + ObObj value_obj; + int sub_ret = OB_SUCCESS; + if (OB_SUCCESS != (sub_ret = session_info->get_sys_variable_value(sys_key_name, value_obj))) { + LOG_WARN("fail to get sys var from session, use standard nls format", K(sub_ret), K(sys_key_name)); + } else { + ObString value_str = value_obj.get_string(); + if (OB_FAIL(dtc_params.set_nls_format_by_type(obj_type, value_str))) { + LOG_WARN("fail to set nls format by type", K(ret), K(obj_type), K(value_str)); + } else { + LOG_DEBUG("succ to set nls format by type", K(obj_type), K(value_str)); + } + } + } else { + /* other types do not need nls format from session, do nothing here */ + LOG_DEBUG("no need to set nls format", K(obj_type)); + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("fail to build dtc params due to null session", K(ret)); + } + + return ret; +} + diff --git a/src/obproxy/proxy/route/obproxy_expr_calculator.h b/src/obproxy/proxy/route/obproxy_expr_calculator.h index 6dfb1aed8d4b56b9d6ee4a604591f4ab0ce0eba9..4e962c6e5b5e4ef7414884e770543b87d5840a58 100644 --- a/src/obproxy/proxy/route/obproxy_expr_calculator.h +++ b/src/obproxy/proxy/route/obproxy_expr_calculator.h @@ -14,6 +14,7 @@ #define OBPROXY_EXPR_CALCULATOR_H #include "opsql/expr_parser/ob_expr_parse_result.h" #include "lib/charset/ob_charset.h" +#include "common/ob_obj_type.h" namespace oceanbase { @@ -22,6 +23,8 @@ namespace common class ObIAllocator; class ObArenaAllocator; class ObString; +class ObTimeZoneInfo; +class ObDataTypeCastParams; } namespace obproxy { @@ -38,7 +41,7 @@ namespace proxy class ObProxyMysqlRequest; class ObProxyPartInfo; class ObClientSessionInfo; -class ObPsEntry; +class ObPsIdEntry; class ObTextPsEntry; class ObProxyExprCalculator @@ -64,17 +67,20 @@ private: int do_expr_resolve(ObExprParseResult &expr_result, const ObProxyMysqlRequest &client_request, ObClientSessionInfo *client_info, - ObPsEntry *ps_entry, + ObPsIdEntry *ps_id_entry, ObTextPsEntry *text_ps_entry, ObProxyPartInfo &part_info, common::ObIAllocator &allocator, opsql::ObExprResolverResult &resolve_result); int do_partition_id_calc(opsql::ObExprResolverResult &resolve_result, + ObClientSessionInfo &client_info, ObProxyPartInfo &part_info, + const obutils::ObSqlParseResult &parse_result, common::ObIAllocator &allocator, int64_t &partition_id); int calc_part_id_with_simple_route_info(common::ObArenaAllocator &allocator, const obutils::ObSqlParseResult &parse_result, + ObClientSessionInfo &client_info, ObProxyPartInfo &part_info, int64_t &part_id); int do_resolve_with_part_key(const obutils::ObSqlParseResult &parse_result, @@ -89,6 +95,22 @@ private: int64_t &sub_part_id, int64_t &phy_part_id); }; + +class ObExprCalcTool { +public: + static int build_dtc_params_with_tz_info(ObClientSessionInfo *session_info, + common::ObObjType obj_type, + common::ObTimeZoneInfo &tz_info, + common::ObDataTypeCastParams &dtc_params); + static int build_tz_info(ObClientSessionInfo *session_info, + common::ObObjType obj_type, + common::ObTimeZoneInfo &tz_info); + static int build_dtc_params(ObClientSessionInfo *session_info, + common::ObObjType obj_type, + common::ObDataTypeCastParams &dtc_params); +}; + + } // end of namespace proxy } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/proxy/route/obproxy_part_mgr.cpp b/src/obproxy/proxy/route/obproxy_part_mgr.cpp index 70150dd01ec39c519a32b01d0b04e107bf3fc37a..8365c653bfae2de59df0a567c5edb54028f6b7b6 100644 --- a/src/obproxy/proxy/route/obproxy_part_mgr.cpp +++ b/src/obproxy/proxy/route/obproxy_part_mgr.cpp @@ -78,12 +78,13 @@ int ObProxyPartMgr::get_first_part_id_by_idx(const int64_t idx, int64_t &part_id int ObProxyPartMgr::get_first_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; if (OB_NOT_NULL(first_part_desc_)) { - ret = first_part_desc_->get_part(range, allocator, part_ids); + ret = first_part_desc_->get_part(range, allocator, part_ids, ctx); } else { ret = OB_INVALID_ARGUMENT; } @@ -159,7 +160,8 @@ int ObProxyPartMgr::get_sub_part_desc_by_first_part_id(const bool is_template_ta int ObProxyPartMgr::get_sub_part(ObNewRange &range, ObIAllocator &allocator, ObPartDesc *sub_part_desc_ptr, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; @@ -167,7 +169,7 @@ int ObProxyPartMgr::get_sub_part(ObNewRange &range, ret = OB_INVALID_ARGUMENT; LOG_WARN("fail to get sub part, null ptr", K(ret)); } else { - if (OB_FAIL(sub_part_desc_ptr->get_part(range, allocator, part_ids))) { + if (OB_FAIL(sub_part_desc_ptr->get_part(range, allocator, part_ids, ctx))) { LOG_WARN("fail to get sub part", K(sub_part_desc_ptr), K(ret)); } } @@ -261,6 +263,7 @@ int ObProxyPartMgr::build_hash_part(const bool is_oracle_mode, if (static_cast(key_info.part_keys_[i].level_) == part_level) { desc_hash->set_part_key_type(static_cast(key_info.part_keys_[i].obj_type_)); desc_hash->set_part_key_cs_type(static_cast(key_info.part_keys_[i].cs_type_)); + desc_hash->set_accuracy(key_info.part_keys_[i].accuracy_); break; } } @@ -320,6 +323,7 @@ int ObProxyPartMgr::build_sub_hash_part_with_non_template(const bool is_oracle_m if (static_cast(key_info.part_keys_[i].level_) == PARTITION_LEVEL_TWO) { desc_hash->set_part_key_type(static_cast(key_info.part_keys_[i].obj_type_)); desc_hash->set_part_key_cs_type(static_cast(key_info.part_keys_[i].cs_type_)); + desc_hash->set_accuracy(key_info.part_keys_[i].accuracy_); break; } } @@ -398,6 +402,7 @@ int ObProxyPartMgr::build_key_part(const ObPartitionLevel part_level, if (static_cast(key_info.part_keys_[i].level_) == part_level) { desc_key->set_part_key_type(static_cast(key_info.part_keys_[i].obj_type_)); desc_key->set_part_key_cs_type(static_cast(key_info.part_keys_[i].cs_type_)); + desc_key->set_accuracy(key_info.part_keys_[i].accuracy_); break; } } @@ -455,6 +460,7 @@ int ObProxyPartMgr::build_sub_key_part_with_non_template(const ObPartitionFuncTy if (static_cast(key_info.part_keys_[i].level_) == PARTITION_LEVEL_TWO) { desc_key->set_part_key_type(static_cast(key_info.part_keys_[i].obj_type_)); desc_key->set_part_key_cs_type(static_cast(key_info.part_keys_[i].cs_type_)); + desc_key->set_accuracy(key_info.part_keys_[i].accuracy_); break; } } @@ -562,10 +568,8 @@ int ObProxyPartMgr::build_range_part(const ObPartitionLevel part_level, desc_range->set_part_func_type(part_func_type); for (int k = 0; k < key_info.key_num_; ++k) { if (static_cast(key_info.part_keys_[k].level_) == part_level) { - if (static_cast(key_info.part_keys_[k].cs_type_ != CS_TYPE_INVALID)) { - desc_range->set_collation_type(static_cast(key_info.part_keys_[k].cs_type_)); - break; - } + desc_range->set_accuracy(key_info.part_keys_[k].accuracy_); + break; } } if (PARTITION_LEVEL_ONE == part_level) { @@ -640,10 +644,8 @@ int ObProxyPartMgr::build_sub_range_part_with_non_template(const ObPartitionFunc desc_range->set_part_func_type(part_func_type); for (int k = 0; k < key_info.key_num_; ++k) { if (static_cast(key_info.part_keys_[k].level_) == PARTITION_LEVEL_TWO) { - if (static_cast(key_info.part_keys_[k].cs_type_ != CS_TYPE_INVALID)) { - desc_range->set_collation_type(static_cast(key_info.part_keys_[k].cs_type_)); - break; - } + desc_range->set_accuracy(key_info.part_keys_[k].accuracy_); + break; } } if (OB_FAIL(desc_range->set_part_array(part_array, sub_part_num_[i]))) { @@ -764,10 +766,8 @@ int ObProxyPartMgr::build_list_part(const ObPartitionLevel part_level, desc_list->set_part_func_type(part_func_type); for (int k = 0; k < key_info.key_num_; ++k) { if (static_cast(key_info.part_keys_[k].level_) == part_level) { - if (static_cast(key_info.part_keys_[k].cs_type_ != CS_TYPE_INVALID)) { - desc_list->set_collation_type(static_cast(key_info.part_keys_[k].cs_type_)); - break; - } + desc_list->set_accuracy(key_info.part_keys_[k].accuracy_); + break; } } if (PARTITION_LEVEL_ONE == part_level) { @@ -857,10 +857,8 @@ int ObProxyPartMgr::build_sub_list_part_with_non_template(const ObPartitionFuncT desc_list->set_part_func_type(part_func_type); for (int k = 0; k < key_info.key_num_; ++k) { if (static_cast(key_info.part_keys_[k].level_) == PARTITION_LEVEL_TWO) { - if (static_cast(key_info.part_keys_[k].cs_type_ != CS_TYPE_INVALID)) { - desc_list->set_collation_type(static_cast(key_info.part_keys_[k].cs_type_)); - break; - } + desc_list->set_accuracy(key_info.part_keys_[k].accuracy_); + break; } } if (OB_FAIL(desc_list->set_part_array(part_array, sub_part_num_[i]))) { diff --git a/src/obproxy/proxy/route/obproxy_part_mgr.h b/src/obproxy/proxy/route/obproxy_part_mgr.h index 89e6650789a448e71abdb04638f8ecd80a26b3cd..98798152125c297dad347533cd5363b6a4cdab0e 100644 --- a/src/obproxy/proxy/route/obproxy_part_mgr.h +++ b/src/obproxy/proxy/route/obproxy_part_mgr.h @@ -22,6 +22,7 @@ namespace oceanbase namespace common { class ObPartDesc; +class ObPartDescCtx; } namespace obproxy { @@ -87,15 +88,17 @@ public: int get_first_part_id_by_idx(const int64_t idx, int64_t &part_id); int get_first_part(common::ObNewRange &range, common::ObIAllocator &allocator, - common::ObIArray &part_ids); - int get_sub_part(ObNewRange &range, - ObIAllocator &allocator, - ObPartDesc *sub_part_desc_ptr, - ObIArray &part_ids); + common::ObIArray &part_ids, + common::ObPartDescCtx &ctx); + int get_sub_part(common::ObNewRange &range, + common::ObIAllocator &allocator, + common::ObPartDesc *sub_part_desc_ptr, + common::ObIArray &part_ids, + common::ObPartDescCtx &ctx); int get_sub_part_by_random(const int64_t rand_num, - ObPartDesc *sub_part_desc_ptr, - ObIArray &part_ids); + common::ObPartDesc *sub_part_desc_ptr, + common::ObIArray &part_ids); int build_hash_part(const bool is_oracle_mode, const share::schema::ObPartitionLevel part_level, diff --git a/src/obproxy/proxy/shard/Makemodule.am b/src/obproxy/proxy/shard/Makemodule.am index 8fef1d77df550f2b93e00d2df7696f955b997283..44c8dde94d4a82ee29ed0fcdd0524b9b60a3e8bf 100644 --- a/src/obproxy/proxy/shard/Makemodule.am +++ b/src/obproxy/proxy/shard/Makemodule.am @@ -1,3 +1,5 @@ proxy_mysql_shard_sources:=\ obproxy/proxy/shard/obproxy_shard_utils.h\ -obproxy/proxy/shard/obproxy_shard_utils.cpp +obproxy/proxy/shard/obproxy_shard_utils.cpp\ +obproxy/proxy/shard/obproxy_shard_ddl_cont.h\ +obproxy/proxy/shard/obproxy_shard_ddl_cont.cpp diff --git a/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.cpp b/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d22f2567efaa5f5f1595badfdca5d9f3f1e924c6 --- /dev/null +++ b/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.cpp @@ -0,0 +1,484 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX PROXY + +#include +#include "proxy/shard/obproxy_shard_utils.h" +#include "obutils/ob_proxy_config.h" + +using namespace oceanbase::json; +using namespace oceanbase::obproxy::obutils; + +namespace oceanbase +{ +namespace obproxy +{ +namespace proxy +{ +//-------ObShardDDLStatus +int ObShardDDLStatus::parse_id_list_from_json(const Value *json) +{ + int ret = OB_SUCCESS; + + if (JT_ARRAY == json->get_type()) { + DLIST_FOREACH(it, json->get_array()) { + ddl_task_id_list_.push_back(it->get_number()); + } + } else { + ret = OB_INVALID_CONFIG; + LOG_WARN("invalid json config type", "expected type", JT_ARRAY, + "actual type", json->get_type(), K(ret)); + } + + return ret; +} + +int ObShardDDLStatus::parse_data_from_json(const Value *json, const bool is_first) +{ + int ret = OB_SUCCESS; + + if (JT_OBJECT == json->get_type()) { + DLIST_FOREACH(it, json->get_object()) { + if (it->name_ == SHARD_DDL_STATUS) { + if (OB_FAIL(set_ddl_status(it->value_->get_string()))) { + LOG_WARN("fail to set ddl status", K(ret)); + } + } else if (it->name_ == SHARD_DDL_ERROR_CODE) { + if (OB_FAIL(set_error_code(it->value_->get_string()))) { + LOG_WARN("fail to set error code", K(ret)); + } + } else if (it->name_ == SHARD_DDL_ERROR_MSG) { + if (OB_FAIL(set_error_message(it->value_->get_string()))) { + LOG_WARN("fail to set error msg", K(ret)); + } + } else if (it->name_ == SHARD_DDL_TASK_ID_LIST) { + if (is_first && OB_FAIL(parse_id_list_from_json(it->value_))) { + LOG_WARN("fail to parse id list", K(ret)); + } + } + } + } else { + ret = OB_INVALID_CONFIG; + LOG_WARN("invalid json config type", "expected type", JT_OBJECT, + "actual type", json->get_type(), K(ret)); + } + + return ret; +} + +int ObShardDDLStatus::parse_from_json(ObString &json, const bool is_first) +{ + int ret = OB_SUCCESS; + + ObArenaAllocator json_allocator(ObModIds::OB_JSON_PARSER); + Value *json_root = NULL; + Parser parser; + if (OB_FAIL(parser.init(&json_allocator))) { + LOG_WARN("json parser init failed", K(ret)); + } else if (OB_FAIL(parser.parse(json.ptr(), json.length(), json_root))) { + LOG_WARN("parse json failed", K(ret), "json", get_print_json(json)); + } else if (OB_ISNULL(json_root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("json root is null", K(ret)); + } else { + if (JT_OBJECT == json_root->get_type()) { + DLIST_FOREACH(it, json_root->get_object()) { + if (it->name_ == SHARD_DDL_DATA) { + if (OB_FAIL(parse_data_from_json(it->value_, is_first))) { + LOG_WARN("fail to parse data", K(ret)); + } + } + } + } else { + ret = OB_INVALID_CONFIG; + LOG_WARN("invalid json config type", "expected type", JT_OBJECT, + "actual type", json_root->get_type(), K(ret)); + } + } + + return ret; +} + +int ObShardDDLStatus::to_json(ObSqlString &buf) +{ + int ret = OB_SUCCESS; + ret = buf.append_fmt("{\"%s\":\"%.*s\", \"%s\":\"%.*s\", \"%s\":\"%.*s\", \"%s\":[", + SHARD_DDL_STATUS, ddl_status_.length(), ddl_status_.ptr(), + SHARD_DDL_ERROR_CODE, error_code_.length(), error_code_.ptr(), + SHARD_DDL_ERROR_MSG, error_message_.length(), error_message_.ptr(), + SHARD_DDL_TASK_ID_LIST); + + if (OB_SUCC(ret)) { + int64_t count = ddl_task_id_list_.count(); + for (int64_t j = 0; OB_SUCC(ret) && j < count; j++) { + if (OB_FAIL(buf.append_fmt("%ld", ddl_task_id_list_.at(j)))) { + LOG_WARN("fail to append config", K(ret)); + } else if (j < count - 1) { + ret = buf.append(","); + } + j++; + } + + if (OB_SUCC(ret)) { + ret = buf.append("]}"); + } + } + + return ret; +} + +//-------ObShardDDLTableSourceDefinition +int ObShardDDLTableSourceDefinition::init(const ObString &schema, const ObString &sql, const ObShardDDLOperation operation) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(table_options_.create(TABLE_OPTIONS_MAP_BUCKET, ObModIds::OB_PROXY_SHARDING_DDL, ObModIds::OB_PROXY_SHARDING_DDL))) { + LOG_WARN("fail to init table options rule map", K(ret)); + } else { + schema_ = schema; + sql_ = sql; + operation_ = operation; + } + + return ret; +} + +int ObShardDDLTableSourceDefinition::set_table_options(const ObString &key, const ObString value) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(table_options_.set_refactored(key, value))) { + LOG_WARN("fail to set table options", K(key), K(value), K(ret)); + } + + return ret; +} + +const char* ObShardDDLTableSourceDefinition::get_operation_name(ObShardDDLOperation operation) +{ + static const char *operation_name_array[SHARD_DDL_OPERATION_MAX] = + { + "CREATE_TABLE", + "CREATE_INDEX", + "ALTER", + "DROP", + "RENAME" + }; + const char *str_ret = ""; + if (operation >= SHARD_DDL_OPERATION_CREATE_TABLE && operation < SHARD_DDL_OPERATION_MAX) { + str_ret = operation_name_array[operation]; + } + return str_ret; +} + +int ObShardDDLTableSourceDefinition::to_json(ObSqlString &buf) +{ + int ret = OB_SUCCESS; + ret = buf.append_fmt("{\"%s\":\"%.*s\", \"%s\":\"%.*s\", \"%s\":\"%s\", \"%s\":{", + SHARD_DDL_SCHEMA, schema_.length(), schema_.ptr(), + SHARD_DDL_SQL, sql_.length(), sql_.ptr(), + SHARD_DDL_OPERATION, get_operation_name(operation_), + SHARD_DDL_TABLE_OPTIONS); + + if (OB_SUCC(ret)) { + int64_t size = table_options_.size(); + hash::ObHashMap::iterator it = table_options_.begin(); + hash::ObHashMap::iterator end = table_options_.end(); + for (int64_t j = 0; OB_SUCC(ret) && it != end; it++) { + if (OB_FAIL(buf.append_fmt("\"%.*s\": \"%.*s\"", + it->first.length(), it->first.ptr(), + it->second.length(), it->second.ptr()))) { + LOG_WARN("fail to append config", K(ret)); + } else if (j < size - 1) { + ret = buf.append(","); + } + j++; + } + + if (OB_SUCC(ret)) { + ret = buf.append("}}"); + } + } + + return ret; +} + +//-------ObShardDDLCont------ +ObShardDDLCont::ObShardDDLCont(ObContinuation *cb_cont, ObEThread *cb_thread) + : ObAsyncCommonTask(cb_cont->mutex_, "shardDDLCont", cb_cont, cb_thread), + is_first_(true), allocator_(ObModIds::OB_PROXY_SHARDING_DDL), + response_buf_(NULL), ddl_status_(allocator_) +{ +} + +void ObShardDDLCont::destroy() +{ + ObAsyncCommonTask::destroy(); +} + +int ObShardDDLCont::covert_stmt_type_to_operation(const ObProxyBasicStmtType stmt_type, + const ObProxyBasicStmtSubType sub_stmt_type, + ObShardDDLOperation &operation) +{ + int ret = OB_SUCCESS; + + if (stmt_type == OBPROXY_T_CREATE) { + if (sub_stmt_type == OBPROXY_T_SUB_CREATE_TABLE) { + operation = SHARD_DDL_OPERATION_CREATE_TABLE; + } else if (sub_stmt_type == OBPROXY_T_SUB_CREATE_INDEX) { + operation = SHARD_DDL_OPERATION_CREATE_INDEX; + } else { + ret = OB_NOT_SUPPORTED; + } + } else if (stmt_type == OBPROXY_T_ALTER) { + operation = SHARD_DDL_OPERATION_ALTER; + } else if (stmt_type == OBPROXY_T_DROP) { + operation = SHARD_DDL_OPERATION_DROP; + } else if (stmt_type == OBPROXY_T_RENAME) { + operation = SHARD_DDL_OPERATION_RENAME; + } else { + ret = OB_NOT_SUPPORTED; + } + + return ret; +} + +int ObShardDDLCont::get_ddl_url(const char *url_template, char *&buffer) +{ + int ret = OB_SUCCESS; + + ObProxyConfig &proxy_config = get_global_proxy_config(); + int64_t mng_url_len = strlen(proxy_config.mng_url.str()); + int64_t cloud_instance_id_len = strlen(proxy_config.cloud_instance_id.str()); + int64_t url_len = strlen(url_template) + mng_url_len + cloud_instance_id_len; + if (OB_ISNULL(buffer = static_cast(allocator_.alloc(url_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to alloc mem for ddl url", K(url_len), K(ret)); + } else { + // If the last character is /, remove it + if ('/' == *(proxy_config.mng_url.str() + mng_url_len - 1)) { + mng_url_len -= 1; + } + int64_t w_len = snprintf(buffer, static_cast(url_len), url_template, + mng_url_len, proxy_config.mng_url.str(), + proxy_config.cloud_instance_id.str()); + if (OB_UNLIKELY(w_len < 0) || OB_UNLIKELY(w_len > url_len)) { + ret = OB_BUF_NOT_ENOUGH; + LOG_ERROR("fail to snprintf for post ddl url", K(url_len), K(ret)); + } + } + return ret; +} + +int ObShardDDLCont::init(const ObString &instance_id, const ObString &schema, + const ObString &sql, const ObProxyBasicStmtType stmt_type, + const ObProxyBasicStmtSubType sub_stmt_type) +{ + int ret = OB_SUCCESS; + char *async_ddl_url = NULL; + char *check_ddl_url = NULL; + + if (OB_FAIL(ob_write_string(allocator_, instance_id, instance_id_))) { + LOG_WARN("fail to write string instance_id", K(instance_id), K(ret)); + } else if (OB_FAIL(ob_write_string(allocator_, schema, schema_))) { + LOG_WARN("fail to write string schema", K(schema), K(ret)); + } else if (OB_FAIL(ob_write_string(allocator_, sql, sql_))) { + LOG_WARN("fail to write string sql", K(sql), K(ret)); + } else if (OB_FAIL(covert_stmt_type_to_operation(stmt_type, sub_stmt_type, operation_))) { + LOG_WARN("fail to covert stmt type to operation", K(stmt_type), K(sub_stmt_type), K(ret)); + } else if (OB_UNLIKELY(NULL != response_buf_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("buf should be null before memory allocated", K_(response_buf), K(ret)); + } else if (OB_ISNULL(response_buf_ = static_cast(allocator_.alloc(OBPROXY_MAX_JSON_INFO_SIZE)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret)); + } else if (OB_FAIL(get_ddl_url(ASYNC_DDL_URL, async_ddl_url))) { + LOG_WARN("fail to get async ddl url", K(ret)); + } else if (OB_FAIL(get_ddl_url(CHECK_DDL_URL, check_ddl_url))) { + LOG_WARN("fail to get check ddl url", K(ret)); + } else { + async_ddl_url_.assign_ptr(async_ddl_url, static_cast(strlen(async_ddl_url))); + check_ddl_url_.assign_ptr(check_ddl_url, static_cast(strlen(check_ddl_url))); + ddl_status_.set_error_code(ObShardDDLStatus::SHARD_DDL_JOB_ERROR); + ddl_status_.set_error_message(ObShardDDLStatus::SHARD_DDL_JOB_ERROR_MSG); + } + + return ret; +} + +int ObShardDDLCont::init_task() +{ + int ret = OB_SUCCESS; + bool need_reschedule = false; + ObSqlString buf; + + response_string_.assign_buffer(response_buf_, static_cast(OBPROXY_MAX_JSON_INFO_SIZE)); + if (is_first_) { + ObShardDDLTableSourceDefinition table_source_definition; + if (OB_FAIL(table_source_definition.init(schema_, sql_, operation_))) { + LOG_WARN("fail to init table source definition", K_(schema), K_(sql), K_(operation), K(ret)); + } else if (OB_FAIL(table_source_definition.set_table_options("InstanceId", instance_id_))) { + LOG_WARN("fail to set table options", K_(instance_id), K(ret)); + } else if (OB_FAIL(table_source_definition.to_json(buf))) { + LOG_WARN("fail to get post json data", K(ret)); + } else if (OB_FAIL(post_ddl_request(async_ddl_url_.ptr(), buf.ptr(), static_cast(&response_string_), write_data))) { + LOG_WARN("fail to post ddl request", K_(async_ddl_url), K(ret)); + } else if (OB_FAIL(ddl_status_.parse_from_json(response_string_, is_first_))) { + LOG_WARN("fail to parse ddl response", K_(response_string), K(ret)); + } else { + is_first_ = false; + } + } else { + if (OB_FAIL(ddl_status_.to_json(buf))) { + LOG_WARN("fail to get post json data", K(ret)); + } else if (OB_FAIL(post_ddl_request(check_ddl_url_.ptr(), buf.ptr(), static_cast(&response_string_), write_data))) { + LOG_WARN("fail to post ddl request", K_(check_ddl_url), K(ret)); + } else if (OB_FAIL(ddl_status_.parse_from_json(response_string_, is_first_))) { + LOG_WARN("fail to parse ddl response", K_(response_string), K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (ddl_status_.is_success()) { + need_reschedule = false; + } else if (ddl_status_.is_running()) { + need_reschedule = true; + } else { + LOG_WARN("Execute ddl failed", K_(sql), "errcode", ddl_status_.get_error_code(), + "errmsg", ddl_status_.get_error_message()); + ddl_status_.set_error_code(ObShardDDLStatus::SHARD_DDL_JOB_ERROR); + need_reschedule = false; + } + + if (need_reschedule) { + // reschedule + if (OB_ISNULL(self_ethread().schedule_in(this, HRTIME_MSECONDS(RETRY_INTERVAL_MS), EVENT_IMMEDIATE))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to reschedule shard ddl cont", K(ret)); + } else { + ret = OB_SUCCESS; + LOG_DEBUG("succ to reschedule shard ddl cont", K(ret)); + } + } else { + if (NULL != cb_cont_) { + need_callback_ = true; + } else { + terminate_ = true; + } + } + } + + if (OB_FAIL(ret)) { + ddl_status_.set_error_code(ObShardDDLStatus::SHARD_DDL_JOB_ERROR); + ddl_status_.set_error_message(ObShardDDLStatus::SHARD_DDL_JOB_ERROR_MSG); + } + + return ret; +} + +int ObShardDDLCont::post_ddl_request(const char *url, void *postrequest, void *response, write_func write_func_callback /*NULL*/) +{ + int ret = OB_SUCCESS; + CURL *curl = NULL; + if (OB_ISNULL(url) || OB_ISNULL(postrequest) || OB_ISNULL(response)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("NULL pointer, invalid fetch by curl", KP(url), KP(postrequest), KP(response), K(ret)); + } else if (OB_ISNULL(curl = curl_easy_init())) { + ret = OB_CURL_ERROR; + LOG_WARN("init curl failed", K(ret)); + } else { + CURLcode cc = CURLE_OK; + curl_slist *plist = NULL; + int64_t http_code = 0; + //set curl options + if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_URL, url))) { + LOG_WARN("set url failed", K(cc), "url", url); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L))) { + LOG_WARN("set no signal failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1L))) { + LOG_WARN("set tcp_nodelay failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3))) {//set max redirect + LOG_WARN("set max redirect failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1))) {//for http redirect 301 302 + LOG_WARN("set follow location failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, CURL_CONNECTION_TIMEOUT))) { + LOG_WARN("set connect timeout failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_TIMEOUT, CURL_TRANSFER_TIMEOUT))) { + LOG_WARN("set transfer timeout failed", K(cc)); + } else if (FALSE_IT(plist = curl_slist_append(NULL, JSON_HTTP_HEADER))) { + // impossiable + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, plist))) { + LOG_WARN("set http header failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postrequest))) { + LOG_WARN("set postfields failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_func_callback))) { + LOG_WARN("set write callback failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_setopt(curl, CURLOPT_WRITEDATA, response))) { + LOG_WARN("set write data failed", K(cc)); + } else if (CURLE_OK != (cc = curl_easy_perform(curl))) { + LOG_WARN("curl easy perform failed", K(cc)); + } else if(CURLE_OK != (cc = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code))) { + LOG_WARN("curl getinfo failed", K(cc)); + } else { + // http status code 2xx means success + if (http_code / 100 != 2) { + ret = OB_CURL_ERROR; + LOG_WARN("unexpected http status code", K(http_code), K(postrequest), K(url), K(ret)); + } + } + + if (CURLE_OK != cc) { + if (CURLE_WRITE_ERROR == cc) { + ret = OB_SIZE_OVERFLOW; + } else { + ret = OB_CURL_ERROR; + } + LOG_WARN("curl error", "curl_error_code", cc, "curl_error_message", + curl_easy_strerror(cc), K(ret), K(url)); + } + curl_easy_cleanup(curl); + } + return ret; +} + +int64_t ObShardDDLCont::write_data(void *ptr, int64_t size, int64_t nmemb, void *stream) +{ + int ret = OB_SUCCESS; + int64_t real_size = 0; + ObString *content = NULL; + + if (OB_ISNULL(stream) || OB_ISNULL(ptr) || OB_UNLIKELY(size < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(stream), K(ptr), K(size), K(ret)); + } else { + real_size = size * nmemb; + if (real_size > 0) { + content = static_cast(stream); + if (real_size + content->length() > content->size()) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("unexpected long content", + "new_byte", real_size, + "recved_byte", content->length(), + "content_size", content->size(), + K(ret)); + } else if (content->write(static_cast(ptr), static_cast(real_size)) <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("append data failed", K(ret)); + } + } + } + return OB_SUCCESS == ret ? real_size : 0; +} + +} // end of namespace proxy +} // end of namespace obproxy +} // end of namespace oceanbase diff --git a/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.h b/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.h new file mode 100644 index 0000000000000000000000000000000000000000..6a4a1261b0a493e5ceb612f0457fb604ac7ddcfc --- /dev/null +++ b/src/obproxy/proxy/shard/obproxy_shard_ddl_cont.h @@ -0,0 +1,171 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBPROXY_SHARD_DDL_CONT_H +#define OBPROXY_SHARD_DDL_CONT_H + +#include "lib/json/ob_json.h" + +namespace oceanbase +{ +namespace obproxy +{ +namespace proxy +{ + +#define LIMIT_RULE_MAP_BUCKET 8 + +typedef int64_t (* write_func)(void *ptr, int64_t size, int64_t nmemb, void *stream); + +enum ObShardDDLOperation { + SHARD_DDL_OPERATION_CREATE_TABLE, + SHARD_DDL_OPERATION_CREATE_INDEX, + SHARD_DDL_OPERATION_ALTER, + SHARD_DDL_OPERATION_DROP, + SHARD_DDL_OPERATION_RENAME, + SHARD_DDL_OPERATION_MAX +}; + +class ObShardDDLStatus +{ +public: + ObShardDDLStatus(common::ObArenaAllocator &allocator) : allocator_(&allocator), + ddl_task_id_list_(ObModIds::OB_PROXY_SHARDING_DDL, OB_MALLOC_NORMAL_BLOCK_SIZE) {} + virtual ~ObShardDDLStatus() {} + + int parse_id_list_from_json(const json::Value *json); + int parse_data_from_json(const json::Value *json, const bool is_first); + int parse_from_json(ObString &buf, const bool is_first); + int to_json(ObSqlString &buf); + + int set_ddl_status(const ObString &ddl_status) { + if (!ddl_status_.empty()) { + allocator_->free(ddl_status_.ptr()); + } + return ob_write_string(*allocator_, ddl_status, ddl_status_); + } + int set_error_code(const ObString &error_code) { + if (!error_code_.empty()) { + allocator_->free(error_code_.ptr()); + } + return ob_write_string(*allocator_, error_code, error_code_); + } + int set_error_message(const ObString &error_message) { + if (!error_message_.empty()) { + allocator_->free(error_message_.ptr()); + } + return ob_write_string(*allocator_, error_message, error_message_); + } + const ObString &get_error_code() { return error_code_; } + const ObString &get_error_message() { return error_message_; } + + bool is_success() { return 0 == error_code_.case_compare(SHARD_DDL_SUCCESS); } + bool is_running() { return 0 == error_code_.case_compare(SHARD_DDL_RUNNING); } + +public: + static constexpr char const *SHARD_DDL_SUCCESS = "0000"; + static constexpr char const *SHARD_DDL_RUNNING = "0004"; + static constexpr char const *SHARD_DDL_JOB_ERROR = "8204"; + static constexpr char const *SHARD_DDL_JOB_ERROR_MSG = "Execute DDL job error"; + static constexpr char const *SHARD_DDL_DATA = "data"; + static constexpr char const *SHARD_DDL_STATUS = "ddlStatus"; + static constexpr char const *SHARD_DDL_ERROR_CODE = "errorCode"; + static constexpr char const *SHARD_DDL_ERROR_MSG = "errorMessage"; + static constexpr char const *SHARD_DDL_TASK_ID_LIST = "ddlTaskIdList"; + +private: + common::ObArenaAllocator *allocator_; + common::ObSEArray ddl_task_id_list_; + ObString ddl_status_; + ObString error_code_; + ObString error_message_; +}; + +#define TABLE_OPTIONS_MAP_BUCKET 1 + +class ObShardDDLTableSourceDefinition +{ +public: + ObShardDDLTableSourceDefinition() {} + virtual ~ObShardDDLTableSourceDefinition() {} + + int init(const ObString &schema, const ObString &sql, const ObShardDDLOperation operation); + int set_table_options(const ObString &key, const ObString value); + + int to_json(ObSqlString &buf); + +private: + const char* get_operation_name(ObShardDDLOperation operation); + +private: + static constexpr char const *SHARD_DDL_SCHEMA = "schema"; + static constexpr char const *SHARD_DDL_SQL = "sql"; + static constexpr char const *SHARD_DDL_OPERATION = "operation"; + static constexpr char const *SHARD_DDL_TABLE_OPTIONS = "tableOptions"; + +private: + ObString schema_; + ObString sql_; + ObShardDDLOperation operation_; + hash::ObHashMap table_options_; +}; + +class ObShardDDLCont : public obutils::ObAsyncCommonTask +{ +public: + ObShardDDLCont(event::ObContinuation *cb_cont, event::ObEThread *cb_thread); + virtual ~ObShardDDLCont() {} + + virtual void destroy(); + virtual int init_task(); + virtual void *get_callback_data() { return static_cast(&ddl_status_); } + + int init(const ObString &instance_id, const ObString &schema, + const ObString &sql, const ObProxyBasicStmtType stmt_type, + const ObProxyBasicStmtSubType sub_stmt_type); + +private: + int covert_stmt_type_to_operation(const ObProxyBasicStmtType stmt_type, + const ObProxyBasicStmtSubType sub_stmt_type, + ObShardDDLOperation &operation); + int get_ddl_url(const char *url_template, char *&buffer); + int post_ddl_request(const char *url, void *postrequest, void *response, write_func write_func_callback /*NULL*/); + static int64_t write_data(void *ptr, int64_t size, int64_t nmemb, void *stream); + +private: + static const int64_t OBPROXY_MAX_JSON_INFO_SIZE = 64 * 1024; // 64K + static const int64_t RETRY_INTERVAL_MS = 1000; + static const int64_t CURL_CONNECTION_TIMEOUT = 10; + static const int64_t CURL_TRANSFER_TIMEOUT = 5; + static constexpr char const *JSON_HTTP_HEADER = "Content-Type:application/json;charset=UTF-8"; + static constexpr char const *ASYNC_DDL_URL = "%.*s/privateapi/v1/%s/zdalproxy/ddl/ddlAsyncByODP"; + static constexpr char const *CHECK_DDL_URL = "%.*s/privateapi/v1/%s/zdalproxy/ddl/checkDDLTaskStatus"; + +private: + bool is_first_; + common::ObArenaAllocator allocator_; + ObString instance_id_; + ObString schema_; + ObString sql_; + ObShardDDLOperation operation_; + ObString async_ddl_url_; + ObString check_ddl_url_; + ObString response_string_; + char *response_buf_; + ObShardDDLStatus ddl_status_; +}; + +} // end of namespace proxy +} // end of namespace obproxy +} // end of namespace oceanbase + +#endif //OBPROXY_SHARD_DDL_CONT_H diff --git a/src/obproxy/proxy/shard/obproxy_shard_utils.cpp b/src/obproxy/proxy/shard/obproxy_shard_utils.cpp index 67bb1187f9890d8c61356279b747b6ff9b8ec679..d369e4acfa0b0478d1ff901c68d276ee1e2f3119 100644 --- a/src/obproxy/proxy/shard/obproxy_shard_utils.cpp +++ b/src/obproxy/proxy/shard/obproxy_shard_utils.cpp @@ -25,8 +25,10 @@ #include "optimizer/ob_sharding_select_log_plan.h" #include "obutils/ob_proxy_stmt.h" #include "optimizer/ob_proxy_optimizer_processor.h" +#include "lib/container/ob_se_array_iterator.h" using namespace oceanbase::common; +using namespace oceanbase::common::hash; using namespace oceanbase::obmysql; using namespace oceanbase::obproxy::dbconfig; using namespace oceanbase::obproxy::obutils; @@ -133,8 +135,18 @@ int ObProxyShardUtils::change_connector(ObDbConfigLogicDb &logic_db_info, } } - if (OB_SUCC(ret) && OB_FAIL(handle_sys_read_consitency_prop(logic_db_info, *shard_conn, session_info))) { - LOG_WARN("fail to handle_sys_read_consitency_prop", KPC(shard_conn)); + if (OB_SUCC(ret)) { + if (OB_FAIL(handle_sys_read_consitency_prop(logic_db_info, *shard_conn, session_info))) { + LOG_WARN("fail to handle_sys_read_consitency_prop", KPC(shard_conn)); + } else { + ObShardProp* shard_prop = NULL; + if (OB_FAIL(logic_db_info.get_shard_prop(shard_conn->shard_name_, shard_prop))) { + LOG_DEBUG("fail to get shard prop", "shard name", shard_conn->shard_name_, K(ret)); + ret = OB_SUCCESS; + } else { + session_info.set_shard_prop(shard_prop); + } + } } if (OB_SUCC(ret)) { @@ -219,6 +231,146 @@ void ObProxyShardUtils::replace_oracle_table(ObSqlString &new_sql, const ObStrin hava_quoto = false; } +int ObProxyShardUtils::do_rewrite_shard_select_request(const ObString &sql, + ObSqlParseResult &parse_result, + bool is_oracle_mode, + const ObHashMap &table_name_map, + const ObString &real_database_name, + bool is_single_shard_db_table, + ObSqlString &new_sql) +{ + int ret = OB_SUCCESS; + + const uint32_t PARSE_EXTRA_CHAR_NUM = 2; + const char *sql_ptr = sql.ptr(); + int64_t sql_len = sql.length(); + int64_t copy_pos = 0; + + ObProxySelectStmt *select_stmt = static_cast(parse_result.get_proxy_stmt()); + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null select stmt", K(ret)); + } else { + ObProxySelectStmt::TablePosArray &table_pos_array = select_stmt->get_table_pos_array(); + std::sort(table_pos_array.begin(), table_pos_array.end()); + + for (int64_t i = 0; OB_SUCC(ret) && i < table_pos_array.count(); i++) { + ObProxyExprTablePos &expr_table_pos = table_pos_array.at(i); + ObProxyExprTable *expr_table = expr_table_pos.get_table_expr(); + + if (OB_ISNULL(expr_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null expr table", K(ret)); + } else { + bool database_hava_quoto = false; + bool table_hava_quoto = false; + int64_t table_pos = expr_table_pos.get_table_pos(); + int64_t database_pos = expr_table_pos.get_database_pos(); + + ObString &database_name = expr_table->get_database_name(); + ObString &table_name = expr_table->get_table_name(); + uint64_t database_len = database_name.length(); + uint64_t table_len = table_name.length(); + + ObString real_table_name; + if (OB_FAIL(table_name_map.get_refactored(table_name, real_table_name))) { + LOG_WARN("fail to get real table name", K(table_name), K(ret)); + } else { + if (*(sql_ptr + table_pos - 1) == '`' || *(sql_ptr + table_pos - 1) == '"') { + table_hava_quoto = true; + table_pos -= 1; + table_len += 2; + } + + // replace database + if (database_pos > 0) { + // If there is database in SQL + if (*(sql_ptr + database_pos - 1) == '`' || *(sql_ptr + database_pos - 1) == '"') { + database_hava_quoto = true; + database_pos -= 1; + database_len += 2; + } + new_sql.append(sql_ptr + copy_pos, database_pos - copy_pos); + + if (is_oracle_mode) { + replace_oracle_table(new_sql, real_database_name, database_hava_quoto, is_single_shard_db_table, true); + } else { + new_sql.append(real_database_name); + } + + copy_pos = database_pos + database_len; + new_sql.append(sql_ptr + copy_pos, table_pos - copy_pos); + } else { + // If there is no database in SQL, single database and single table will not be added. + // add real database name before logic table name + new_sql.append(sql_ptr + copy_pos, table_pos - copy_pos); + if (!is_single_shard_db_table && !real_database_name.empty()) { + if (is_oracle_mode) { + replace_oracle_table(new_sql, real_database_name, database_hava_quoto, is_single_shard_db_table, true); + } else { + new_sql.append(real_database_name); + } + new_sql.append(".", 1); + } + } + + // replace table name + if (is_oracle_mode) { + replace_oracle_table(new_sql, real_table_name, table_hava_quoto, is_single_shard_db_table, false); + } else { + new_sql.append(real_table_name); + } + copy_pos = table_pos + table_len; + } + } + } + } + + + if (OB_SUCC(ret)) { + new_sql.append(sql_ptr + copy_pos, sql_len - copy_pos - PARSE_EXTRA_CHAR_NUM); + } + + return ret; +} + +int ObProxyShardUtils::rewrite_shard_select_request(ObClientSessionInfo &session_info, + ObProxyMysqlRequest &client_request, + ObIOBufferReader &client_buffer_reader, + const ObHashMap &table_name_map, + const ObString &real_database_name, + bool is_single_shard_db_table) +{ + int ret = OB_SUCCESS; + + ObSqlParseResult &parse_result = client_request.get_parse_result(); + ObSqlString new_sql; + + if (OB_FAIL(do_rewrite_shard_select_request(client_request.get_parse_sql(), parse_result, + session_info.is_oracle_mode(), table_name_map, + real_database_name, is_single_shard_db_table, + new_sql))) { + } else { + // 4. push reader forward by consuming old buffer and write new sql into buffer + if (OB_FAIL(client_buffer_reader.consume_all())) { + LOG_WARN("fail to consume all", K(ret)); + } else { + ObMIOBuffer *writer = client_buffer_reader.mbuf_; + if (OB_ISNULL(writer)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null values ", K(writer), K(ret)); + // no need compress here, if server session support compress, it will compress later + } else if (OB_FAIL(ObMysqlRequestBuilder::build_mysql_request(*writer, obmysql::OB_MYSQL_COM_QUERY, new_sql.string(), false, false))) { + LOG_WARN("fail to build_mysql_request", K(new_sql), K(ret)); + } else if (OB_FAIL(ObProxySessionInfoHandler::rewrite_query_req_by_sharding(session_info, client_request, client_buffer_reader))) { + LOG_WARN("fail to rewrite_query_req_by_sharding", K(ret)); + } + } + } + + return ret; +} + // MySQL use single quoto to table, case-insensitive // Oracle use double quoto, case-sensitive, The default is uppercase // if single schema table mode, table from SQL: @@ -594,10 +746,10 @@ int ObProxyShardUtils::testload_check_and_rewrite_testload_request(ObClientSessi common::hash::ObHashMap alias_table_map; if (OB_FAIL(alias_table_map.create(OB_ALIAS_TABLE_MAP_MAX_BUCKET_NUM, - ObModIds::ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { + ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { LOG_WARN("failed to create alias_table_map map"); } else if (OB_FAIL(all_table_map.create(OB_ALIAS_TABLE_MAP_MAX_BUCKET_NUM, - ObModIds::ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { + ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { LOG_WARN("failed to create all_table_map map"); } else if (OB_ISNULL(parse_result.get_ob_parser_result())) { ret = OB_ERR_UNEXPECTED; @@ -722,7 +874,7 @@ int ObProxyShardUtils::testload_check_and_rewrite_testload_request(ObClientSessi if (OB_ISNULL(writer)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null values ", K(writer), K(ret)); - // no need compress here, if server session support comrpess, it will compress later + // no need compress here, if server session support compress, it will compress later } else if (OB_FAIL(ObMysqlRequestBuilder::build_mysql_request(*writer, obmysql::OB_MYSQL_COM_QUERY, new_sql.string(), false, false))) { LOG_WARN("fail to build_mysql_request", K(new_sql), K(ret)); } else if (OB_FAIL(ObProxySessionInfoHandler::rewrite_query_req_by_sharding(session_info, client_request, client_buffer_reader))) { @@ -861,7 +1013,7 @@ bool ObProxyShardUtils::is_read_stmt(ObClientSessionInfo &session_info, ObMysqlT return (!is_sharding_in_trans(session_info, trans_state) && (parse_result.is_show_stmt() || parse_result.is_desc_stmt() - || parse_result.is_select_stmt() + || (parse_result.is_select_stmt() && !parse_result.has_for_update()) || parse_result.is_set_stmt() //[set ac = 0] is internal request, will not be here || parse_result.has_explain())); } @@ -882,10 +1034,10 @@ bool ObProxyShardUtils::is_read_stmt(ObClientSessionInfo &session_info, ObMysqlT * if have es_id, route based on es_id. otherwise based on weight * ignore other hint */ -int ObProxyShardUtils::handle_single_shard_request(ObMysqlClientSession &client_session, - ObMysqlTransact::ObTransState &trans_state, - ObIOBufferReader &client_buffer_reader, - ObDbConfigLogicDb &logic_db_info) +int ObProxyShardUtils::do_handle_single_shard_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + ObDbConfigLogicDb &logic_db_info) { int ret = OB_SUCCESS; @@ -961,7 +1113,7 @@ int ObProxyShardUtils::handle_single_shard_request(ObMysqlClientSession &client_ hint_table, testload_type, is_read_stmt))) { LOG_WARN("shard tpo info is null", K(ret)); } else if (OB_ISNULL(shard_conn) || OB_ISNULL(prev_shard_conn)) { - ret = OB_ERR_COULUMN_VALUE_NOT_MATCH; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("shard connector info or prev shard connector info is null", KP(shard_conn), KP(prev_shard_conn), K(ret)); } else if (*prev_shard_conn != *shard_conn) { @@ -991,7 +1143,12 @@ int ObProxyShardUtils::handle_single_shard_request(ObMysqlClientSession &client_ ret = OB_ERR_TESTLOAD_ALIPAY_COMPATIBLE; LOG_WARN("not have table_name's hint for 'testload=8' (TESTLOAD_ALIPAY_COMPATIBLE)", K(ret)); } else if (OB_SUCC(ret) && TESTLOAD_NON != testload_type) { //rewrite table name for testload - if (OB_FAIL(testload_check_and_rewrite_testload_request(session_info, client_request, client_buffer_reader, + ObProxySqlParser sql_parser; + ObSqlParseResult &sql_parse_result = client_request.get_parse_result(); + ObString sql = client_request.get_parse_sql(); + if (OB_FAIL(sql_parser.parse_sql_by_obparser(sql, NORMAL_PARSE_MODE, sql_parse_result, false))) { + LOG_WARN("parse_sql_by_obparser failed", K(ret), K(sql)); + } else if (OB_FAIL(testload_check_and_rewrite_testload_request(session_info, client_request, client_buffer_reader, false, hint_table, ObString::make_string(real_database_name), logic_db_info))) { LOG_WARN("fail to check and rewrite testload request"); } else { @@ -1016,10 +1173,36 @@ int ObProxyShardUtils::handle_single_shard_request(ObMysqlClientSession &client_ return ret; } -int ObProxyShardUtils::handle_shard_request(ObMysqlClientSession &client_session, +int ObProxyShardUtils::handle_single_shard_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + ObDbConfigLogicDb &logic_db_info, + bool &need_wait_callback) +{ + int ret = OB_SUCCESS; + + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + ObSqlParseResult &parse_result = client_request.get_parse_result(); + const ObString runtime_env = get_global_proxy_config().runtime_env.str(); + if (parse_result.is_ddl_stmt() && 0 == runtime_env.case_compare(OB_PROXY_DBP_RUNTIME_ENV)) { + if (OB_FAIL(handle_ddl_request(sm, client_session, trans_state, logic_db_info, need_wait_callback))) { + LOG_WARN("fail to handle ddl request", K(ret)); + } + } else if (OB_FAIL(do_handle_single_shard_request(client_session, trans_state, + client_buffer_reader, logic_db_info))) { + LOG_WARN("fail to handle single shard request", K(ret)); + } + + return ret; +} + +int ObProxyShardUtils::handle_shard_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, ObMysqlTransact::ObTransState &trans_state, ObIOBufferReader &client_buffer_reader, - ObDbConfigLogicDb &db_info) + ObDbConfigLogicDb &db_info, + bool &need_wait_callback) { int ret = OB_SUCCESS; ObClientSessionInfo &session_info = client_session.get_session_info(); @@ -1028,16 +1211,158 @@ int ObProxyShardUtils::handle_shard_request(ObMysqlClientSession &client_session ObSqlParseResult &parse_result = client_request.get_parse_result(); ObString table_name = client_request.get_parse_result().get_origin_table_name(); if (parse_result.is_ddl_stmt()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("ddl stmt is unsupported for sharding table", K(ret)); + const ObString runtime_env = get_global_proxy_config().runtime_env.str(); + if (0 == runtime_env.case_compare(OB_PROXY_DBP_RUNTIME_ENV)) { + if (OB_FAIL(handle_ddl_request(sm, client_session, trans_state, db_info, need_wait_callback))) { + LOG_WARN("fail to handle ddl request", K(ret)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("ddl stmt is unsupported for sharding table", K(ret)); + } } else if (parse_result.is_show_tables_stmt() || (table_name.empty() && !parse_result.is_dml_stmt())) { //do nothing } else if (table_name.empty()) { // keep compatible, skip - } else if (OB_FAIL(handle_dml_request(client_session, trans_state, - client_buffer_reader, table_name, db_info))) { - LOG_WARN("fail to handle dml request", K(table_name), K(ret)); + } else if (parse_result.is_multi_stmt()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("multi stmt is unsupported for sharding table", "stmt type", parse_result.get_stmt_type(), K(ret)); + } else if (parse_result.is_show_create_table_stmt() || parse_result.is_desc_table_stmt()) { + if (OB_FAIL(handle_other_request(client_session, trans_state, + client_buffer_reader, table_name, db_info))) { + LOG_WARN("fail to handle other request", K(ret), K(session_info), K(table_name)); + } + } else if (parse_result.is_select_stmt()) { + if (OB_FAIL(handle_select_request(client_session, trans_state, + client_buffer_reader, table_name, db_info))) { + LOG_WARN("fail to handle select request", K(ret), K(session_info), K(table_name)); + } + } else if (parse_result.is_dml_stmt()) { + if (OB_FAIL(handle_dml_request(client_session, trans_state, + client_buffer_reader, table_name, db_info))) { + LOG_WARN("fail to handle dml request", K(table_name), K(ret)); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("stmt is unsupported for sharding table", "stmt type", parse_result.get_stmt_type(), K(ret)); +} + return ret; +} + +int ObProxyShardUtils::handle_ddl_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObDbConfigLogicDb &db_info, + bool &need_wait_callback) +{ + int ret = OB_SUCCESS; + + ObShardDDLCont *cont = NULL; + + ObClientSessionInfo &session_info = client_session.get_session_info(); + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + ObString sql = client_request.get_parse_sql(); + ObSqlParseResult &parse_result = client_request.get_parse_result(); + ObString logic_tenant_name; + ObEThread *cb_thread = &self_ethread(); + if (OB_FAIL(session_info.get_logic_tenant_name(logic_tenant_name))) { + ret = OB_ERR_UNEXPECTED; // no need response, just return ret and disconnect + LOG_ERROR("fail to get logic tenant name", K(ret)); + } else if (OB_ISNULL(cont = new(std::nothrow) ObShardDDLCont(sm, cb_thread))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory for ObShardDDLCont", K(ret)); + } else if (OB_FAIL(cont->init(logic_tenant_name, db_info.db_name_.config_string_, sql, + parse_result.get_stmt_type(), parse_result.get_cmd_sub_type()))) { + LOG_WARN("fail to init ObShardDDLCont", K(ret)); + } else if (OB_ISNULL(g_event_processor.schedule_imm(cont, ET_TASK))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to schedule ob shard ddl task", K(ret)); + } else if (OB_FAIL(sm->setup_handle_shard_ddl(&cont->get_action()))) { + LOG_WARN("fail to setup handle shard ddl", K(ret)); + } else { + need_wait_callback = true; + LOG_INFO("succ to schedule ob shard ddl task"); + } + + if (OB_FAIL(ret)) { + if (NULL != cont) { + cont->destroy(); + cont = NULL; + } + } + + return ret; +} + +int ObProxyShardUtils::handle_other_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + const ObString &table_name, + ObDbConfigLogicDb &db_info) +{ + int ret = OB_SUCCESS; + + ObClientSessionInfo &session_info = client_session.get_session_info(); + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + char real_table_name[OB_MAX_TABLE_NAME_LENGTH]; + char real_database_name[OB_MAX_DATABASE_NAME_LENGTH]; + + if (OB_FAIL(handle_other_real_info(db_info, client_session, trans_state, table_name, + real_database_name, OB_MAX_DATABASE_NAME_LENGTH, + real_table_name, OB_MAX_TABLE_NAME_LENGTH))) { + LOG_WARN("fail to handle other real info", K(ret), K(session_info), K(table_name)); + } else if (OB_FAIL(rewrite_shard_request(session_info, client_request, client_buffer_reader, + table_name, db_info.db_name_.config_string_, + ObString::make_string(real_table_name), + ObString::make_string(real_database_name), false))) { + LOG_WARN("fail to rewrite shard request", K(ret), K(table_name), + K(db_info.db_name_), K(real_table_name), K(real_database_name)); + } + + return ret; +} + +int ObProxyShardUtils::handle_select_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + const ObString &table_name, + ObDbConfigLogicDb &db_info) +{ + int ret = OB_SUCCESS; + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + ObSqlParseResult &parse_result = client_request.get_parse_result(); + SqlFieldResult& sql_result = parse_result.get_sql_filed_result(); + + bool is_scan_all = false; + ObProxySqlParser sql_parser; + ObString sql = client_request.get_parse_sql(); + if (OB_FAIL(sql_parser.parse_sql_by_obparser(sql, NORMAL_PARSE_MODE, parse_result, true))) { + LOG_WARN("parse_sql_by_obparser failed", K(ret), K(sql)); + } else if (OB_FAIL(check_topology(parse_result, db_info))) { + if (OB_ERR_UNSUPPORT_DIFF_TOPOLOGY != ret) { + LOG_WARN("fail to check topology", K(ret)); + } + } else if (FALSE_IT(is_scan_all = need_scan_all(parse_result))) { + // impossible + } else if (is_scan_all) { + if (OB_FAIL(need_scan_all_by_index(table_name, db_info, sql_result, is_scan_all))) { + LOG_WARN("fail to exec scan all by index", K(table_name), K(ret)); + } } + + if (OB_SUCC(ret)) { + if (is_scan_all) { + if (OB_FAIL(handle_scan_all_real_info(db_info, client_session, trans_state, table_name))) { + LOG_WARN("fail to handle scan all real info", K(table_name), K(ret)); + } + } else { + if (OB_FAIL(handle_select_real_info(db_info, client_session, trans_state, + table_name, client_buffer_reader))) { + LOG_WARN("fail to handle dml real info", K(table_name), K(ret)); + } + } + } + return ret; } @@ -1053,83 +1378,136 @@ int ObProxyShardUtils::handle_dml_request(ObMysqlClientSession &client_session, ObSqlParseResult &parse_result = client_request.get_parse_result(); char real_table_name[OB_MAX_TABLE_NAME_LENGTH]; char real_database_name[OB_MAX_DATABASE_NAME_LENGTH]; - bool is_scan_all = parse_result.get_dbp_route_info().scan_all_; - ObSEArray shard_connector_array; - ObSEArray physical_table_name_array; - ObIAllocator *allocator = NULL; - if (parse_result.is_show_create_table_stmt() || parse_result.is_desc_table_stmt()) { - if (OB_FAIL(handle_other_real_info(db_info, client_session, trans_state, table_name, - real_database_name, OB_MAX_DATABASE_NAME_LENGTH, - real_table_name, OB_MAX_TABLE_NAME_LENGTH))) { - LOG_WARN("fail to handle other real info", K(ret), K(session_info), K(table_name)); - } + ObCollationType connection_collation = static_cast(client_session.get_session_info().get_collation_connection()); + if (OB_FAIL(ObMysqlRequestAnalyzer::parse_sql_fileds(client_request, connection_collation))) { + LOG_WARN("fail to extract_fileds", K(connection_collation), K(ret)); } else if ((parse_result.is_insert_stmt() || parse_result.is_replace_stmt() || parse_result.is_update_stmt()) && parse_result.get_batch_insert_values_count() > 1) { ret = OB_ERR_BATCH_INSERT_FOUND; LOG_WARN("batch insert not supported in sharding sql", K(ret), K(parse_result.get_batch_insert_values_count())); - } else if (is_scan_all && parse_result.is_select_stmt()) { - if (OB_FAIL(get_global_optimizer_processor().alloc_allocator(allocator))) { - LOG_WARN("alloc allocator failed", K(ret)); - } else if (OB_FAIL(handle_sharding_select_real_info(db_info, client_session, trans_state, - table_name, *allocator, - shard_connector_array, - physical_table_name_array))) { - LOG_WARN("fail to handle sharding select real info", K(ret), K(session_info), K(table_name)); - } - } else if (parse_result.is_dml_stmt()) { - if (OB_FAIL(handle_dml_real_info(db_info, client_session, trans_state, table_name, - real_database_name, OB_MAX_DATABASE_NAME_LENGTH, - real_table_name, OB_MAX_TABLE_NAME_LENGTH))) { - LOG_WARN("fail to handle dml real info", K(ret), K(session_info), K(table_name)); - } + } else if (OB_FAIL(handle_dml_real_info(db_info, client_session, trans_state, table_name, + real_database_name, OB_MAX_DATABASE_NAME_LENGTH, + real_table_name, OB_MAX_TABLE_NAME_LENGTH))) { + LOG_WARN("fail to handle dml real info", K(ret), K(session_info), K(table_name)); + } else if (OB_FAIL(rewrite_shard_request(session_info, client_request, client_buffer_reader, + table_name, db_info.db_name_.config_string_, ObString::make_string(real_table_name), + ObString::make_string(real_database_name), false))) { + LOG_WARN("fail to rewrite shard request", K(ret), K(table_name), + K(db_info.db_name_), K(real_table_name), K(real_database_name)); } - if (OB_SUCC(ret)) { - if (is_scan_all && parse_result.is_select_stmt()) { - ObProxySelectStmt *select_stmt = static_cast(parse_result.get_proxy_stmt()); - if (OB_ISNULL(select_stmt)) { + return ret; +} + +int ObProxyShardUtils::check_topology(ObSqlParseResult &parse_result, + ObDbConfigLogicDb &db_info) +{ + int ret = OB_SUCCESS; + + ObProxySelectStmt *select_stmt = static_cast(parse_result.get_proxy_stmt()); + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select stmt is null, unexpected", K(ret)); + } else { + int64_t last_db_size = -1; + int64_t last_tb_size = -1; + ObProxySelectStmt::ExprMap &table_exprs_map = select_stmt->get_table_exprs_map(); + ObProxySelectStmt::ExprMap::iterator iter = table_exprs_map.begin(); + ObProxySelectStmt::ExprMap::iterator end = table_exprs_map.end(); + + for (; OB_SUCC(ret) && iter != end; iter++) { + ObProxyExpr *expr = iter->second; + ObProxyExprTable *table_expr = NULL; + ObShardRule *logic_tb_info = NULL; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null, unexpected", K(ret)); + } else if (OB_ISNULL(table_expr = dynamic_cast(expr))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("select stmt is null, unexpected", K(ret)); + LOG_WARN("fail to cast to table expr", K(expr), K(ret)); } else { - LOG_DEBUG("proxy stmt", K(select_stmt->get_stmt_type()), K(select_stmt->limit_start_), K(select_stmt->limit_offset_), K(select_stmt->hint_string_)); - if (select_stmt->condition_exprs_.count() > 0) { - LOG_DEBUG("condition expr"); - ObProxyExpr::print_proxy_expr(select_stmt->condition_exprs_.at(0)); + ObString &table_name = table_expr->get_table_name(); + if (OB_FAIL(db_info.get_shard_rule(logic_tb_info, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); } - LOG_DEBUG("select expr"); - for (int64_t i = 0; i < select_stmt->select_exprs_.count(); i++) { - ObProxyExpr::print_proxy_expr(select_stmt->select_exprs_.at(i)); - } - LOG_DEBUG("group by expr"); - for (int64_t i = 0; i < select_stmt->group_by_exprs_.count(); i++) { - ObProxyExpr::print_proxy_expr(select_stmt->group_by_exprs_.at(i)); - } - for (int64_t i = 0; i < select_stmt->order_by_exprs_.count(); i++) { - ObProxyOrderItem *order_expr = select_stmt->order_by_exprs_.at(i); - LOG_DEBUG("order by expr", K(order_expr->order_direction_)); - ObProxyExpr::print_proxy_expr(order_expr->expr_); + } + + if (OB_SUCC(ret)) { + int64_t current_db_size = logic_tb_info->db_size_; + if (-1 == last_db_size) { + last_db_size = current_db_size; + } else if (last_db_size != current_db_size) { + ret = OB_ERR_UNSUPPORT_DIFF_TOPOLOGY; } - // handle distributed select - ObShardingSelectLogPlan* select_plan = NULL; - void *ptr = NULL; - if (OB_ISNULL(ptr = allocator->alloc(sizeof(ObShardingSelectLogPlan)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; + } + + if (OB_SUCC(ret)) { + int64_t current_tb_size = 0; + if (1 == logic_tb_info->tb_size_) { + current_tb_size = logic_tb_info->db_size_; } else { - select_plan = new(ptr) ObShardingSelectLogPlan(parse_result, allocator, physical_table_name_array); - client_session.set_sharding_select_log_plan(select_plan); - if (OB_FAIL(select_plan->set_shard_connector_array(shard_connector_array))) { - LOG_WARN("set shard connector array failed", K(ret)); - } else if (OB_FAIL(select_plan->generate_plan())) { - LOG_WARN("fail to generate plan", K(ret)); - } + current_tb_size = logic_tb_info->tb_size_; + } + + if (-1 == last_tb_size) { + last_tb_size = current_tb_size; + } else if (last_tb_size != current_tb_size) { + ret = OB_ERR_UNSUPPORT_DIFF_TOPOLOGY; } } - } else if (OB_FAIL(rewrite_shard_request(session_info, client_request, client_buffer_reader, - table_name, db_info.db_name_.config_string_, ObString::make_string(real_table_name), - ObString::make_string(real_database_name), false))) { - LOG_WARN("fail to rewrite shard request", K(ret), K(table_name), - K(db_info.db_name_), K(real_table_name), K(real_database_name)); + } + } + + return ret; +} + +bool ObProxyShardUtils::need_scan_all(ObSqlParseResult &parse_result) +{ + if (parse_result.is_select_stmt() && !parse_result.has_for_update() + && (parse_result.get_dbp_route_info().scan_all_ + || (!parse_result.is_use_dbp_hint() && get_global_proxy_config().auto_scan_all))) { + return true; + } + + return false; +} + +int ObProxyShardUtils::need_scan_all_by_index(const ObString &table_name, + ObDbConfigLogicDb &db_info, + SqlFieldResult& sql_result, + bool &is_scan_all) +{ + int ret = OB_SUCCESS; + + ObShardRule *logic_tb_info = NULL; + ObSEArray index_array; + if (OB_FAIL(db_info.get_shard_rule(logic_tb_info, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); + // It is possible to have no where condition, but only one database and one table + } else if (logic_tb_info->tb_size_ == 1 && logic_tb_info->db_size_ == 1) { + is_scan_all = false; + } else if (sql_result.field_num_ > 0) { + if (logic_tb_info->tb_size_ == 1) { + // Sub-library without table and sub-library single table, calculated according to db rule + if (OB_FAIL(ObShardRule::get_physic_index_array(sql_result, logic_tb_info->db_rules_, + logic_tb_info->db_size_, + TESTLOAD_NON, index_array))) { + LOG_WARN("fail to get physic tb index", K(table_name), KPC(logic_tb_info), K(ret)); + } + } else { + // Sub-library and sub-table, calculated according to tb rule + if (OB_FAIL(ObShardRule::get_physic_index_array(sql_result, logic_tb_info->tb_rules_, + logic_tb_info->tb_size_, + TESTLOAD_NON, index_array))) { + LOG_WARN("fail to get physic tb index", K(table_name), KPC(logic_tb_info), K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (!index_array.empty() && index_array.count() == 1) { + is_scan_all = false; + } } } @@ -1344,6 +1722,16 @@ int ObProxyShardUtils::handle_shard_auth(ObMysqlClientSession &client_session, c } } + if (OB_SUCC(ret)) { + ObShardProp* shard_prop = NULL; + if (OB_FAIL(db_info->get_shard_prop(shard_conn->shard_name_, shard_prop))) { + LOG_DEBUG("fail to get shard prop", "shard name", shard_conn->shard_name_, K(ret)); + ret = OB_SUCCESS; + } else { + session_info.set_shard_prop(shard_prop); + } + } + if (OB_SUCC(ret)) { session_info.set_shard_connector(shard_conn); if (client_session.is_session_pool_client()) { @@ -1437,6 +1825,27 @@ int ObProxyShardUtils::get_all_schema_table(const ObString &logic_tenant_name, c return ret; } +int ObProxyShardUtils::add_table_name_to_map(ObIAllocator &allocator, + ObHashMap &table_name_map, + const ObString &table_name, const ObString &real_table_name) +{ + int ret = OB_SUCCESS; + + char *buf = NULL; + if (OB_ISNULL(buf = (char*) allocator.alloc(real_table_name.length()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc table name buf", K(real_table_name), K(ret)); + } else { + memcpy(buf, real_table_name.ptr(), real_table_name.length()); + ObString real_table_name_buf(real_table_name.length(), buf); + if (OB_FAIL(table_name_map.set_refactored(table_name, real_table_name_buf))) { + LOG_WARN("fail to set table name buf", K(table_name), K(ret)); + } + } + + return ret; +} + int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, ObMysqlClientSession &client_session, ObMysqlTransact::ObTransState &trans_state, @@ -1450,7 +1859,6 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, ObShardConnector *prev_shard_conn = cs_info.get_shard_connector(); ObSqlParseResult &parse_result = trans_state.trans_info_.client_request_.get_parse_result(); - ObShardRouter *shard_router = NULL; ObShardRule *logic_tb_info = NULL; int64_t group_id = OBPROXY_MAX_DBMESH_ID; @@ -1459,25 +1867,15 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, ObTestLoadType testload_type = TESTLOAD_NON; ObString hint_table; - // table_name_len must be less than OB_MAX_TABLE_NAME_LENGTH - char table_name_str[OB_MAX_TABLE_NAME_LENGTH] = "\0"; - memcpy(table_name_str, table_name.ptr(), table_name.length()); - string_to_upper_case(table_name_str, table_name.length()); - ObString upper_table_name(table_name.length(), table_name_str); - if (OB_ISNULL(prev_shard_conn)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("shard connector info is null", K(ret)); } else if (OB_FAIL(get_shard_hint(logic_db_info, parse_result, group_id, table_id, es_id, hint_table, testload_type))) { LOG_WARN("fail to get shard hint", K(ret)); - } else if (OB_FAIL(logic_db_info.get_shard_router(upper_table_name, shard_router))) { - LOG_WARN("fail to get shard router", K(upper_table_name), K(ret)); - } else if (OB_FAIL(shard_router->get_shard_rule(upper_table_name, logic_tb_info))) { - LOG_WARN("fail to get logic table info", K(upper_table_name), KPC(shard_router), K(ret)); - } else if (OB_ISNULL(logic_tb_info)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("logic tb info is null", K(ret)); + // get group_id + } else if (OB_FAIL(logic_db_info.get_shard_rule(logic_tb_info, table_name))) { + LOG_WARN("fail to get shard rule", K(table_name), K(ret)); } else { int64_t last_group_id = cs_info.get_group_id(); @@ -1489,9 +1887,12 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, // 1. SQL comes with database, and it is different from the current logic library // 2. The group id is specified in the hint, and it is different from the last time // 3. No group_id is specified, nor did the last one (for example, switch the library) + // 4. If the last group_id is greater than the db_size of the table of this SQL + // (for example, the table of the previous SQL is group_{00-99}, the table of this SQL is group_00) if (is_not_saved_database || (group_id != OBPROXY_MAX_DBMESH_ID && group_id != last_group_id) - || last_group_id == OBPROXY_MAX_DBMESH_ID) { + || last_group_id == OBPROXY_MAX_DBMESH_ID + || last_group_id >= logic_tb_info->db_size_) { ObShardTpo *shard_tpo = NULL; ObGroupCluster *gc_info = NULL; if (OB_FAIL(logic_db_info.get_shard_tpo(shard_tpo))) { @@ -1524,7 +1925,7 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, LOG_DEBUG("succ to get eid by weight", K(es_id)); } } else if (OB_UNLIKELY(es_id >= es_size)) { - ret = OB_INVALID_ARGUMENT_FOR_EXTRACT; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("es index is larger than elastic array", K(es_id), K(es_size), K(ret)); } @@ -1562,7 +1963,11 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, snprintf(real_table_name, tb_name_len, "%.*s", static_cast(hint_table.length()), hint_table.ptr()); } else { if (table_id == OBPROXY_MAX_DBMESH_ID) { - table_id = group_id * (logic_tb_info->tb_size_ / logic_tb_info->db_size_); + if (1 == logic_tb_info->tb_size_) { + table_id = group_id; + } else { + table_id = group_id * (logic_tb_info->tb_size_ / logic_tb_info->db_size_); + } } if (OB_FAIL(logic_tb_info->get_real_name_by_index(logic_tb_info->tb_size_, logic_tb_info->tb_suffix_len_, table_id, logic_tb_info->tb_prefix_.config_string_, @@ -1575,16 +1980,193 @@ int ObProxyShardUtils::handle_other_real_info(ObDbConfigLogicDb &logic_db_info, } } - if (NULL != shard_router) { - shard_router->dec_ref(); - shard_router = NULL; + if (NULL != shard_conn) { + shard_conn->dec_ref(); + shard_conn = NULL; + } + + return ret; +} + +int ObProxyShardUtils::handle_scan_all_real_info(ObDbConfigLogicDb &logic_db_info, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + const ObString &table_name) +{ + int ret = OB_SUCCESS; + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + ObSqlParseResult &parse_result = client_request.get_parse_result(); + + ObSEArray shard_connector_array; + ObSEArray shard_prop_array; + ObSEArray, 4> table_name_map_array; + ObIAllocator *allocator = NULL; + + ObProxySelectStmt *select_stmt = static_cast(parse_result.get_proxy_stmt()); + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select stmt is null, unexpected", K(ret)); + } else if (OB_FAIL(get_global_optimizer_processor().alloc_allocator(allocator))) { + LOG_WARN("alloc allocator failed", K(ret)); + } else if (OB_FAIL(handle_sharding_select_real_info(logic_db_info, client_session, trans_state, + table_name, *allocator, + shard_connector_array, + shard_prop_array, + table_name_map_array))) { + LOG_WARN("fail to handle sharding select real info", K(ret), K(table_name)); + } else { + LOG_DEBUG("proxy stmt", K(select_stmt->get_stmt_type()), K(select_stmt->limit_offset_), K(select_stmt->limit_size_)); + LOG_DEBUG("select expr"); + for (int64_t i = 0; i < select_stmt->select_exprs_.count(); i++) { + ObProxyExpr::print_proxy_expr(select_stmt->select_exprs_.at(i)); + } + LOG_DEBUG("group by expr"); + for (int64_t i = 0; i < select_stmt->group_by_exprs_.count(); i++) { + ObProxyExpr::print_proxy_expr(select_stmt->group_by_exprs_.at(i)); + } + for (int64_t i = 0; i < select_stmt->order_by_exprs_.count(); i++) { + ObProxyOrderItem *order_expr = select_stmt->order_by_exprs_.at(i); + LOG_DEBUG("order by expr", K(order_expr->order_direction_)); + ObProxyExpr::print_proxy_expr(order_expr); + } + // Handling distributed selects + ObShardingSelectLogPlan* select_plan = NULL; + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator->alloc(sizeof(ObShardingSelectLogPlan)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc select plan buf", "size", sizeof(ObShardingSelectLogPlan), K(ret)); + } else { + select_plan = new(ptr) ObShardingSelectLogPlan(client_request, allocator); + client_session.set_sharding_select_log_plan(select_plan); + if (OB_FAIL(select_plan->generate_plan(shard_connector_array, shard_prop_array, table_name_map_array))) { + LOG_WARN("fail to generate plan", K(ret)); + } + } + } + return ret; +} + +int ObProxyShardUtils::handle_select_real_info(ObDbConfigLogicDb &logic_db_info, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + const ObString &table_name, + ObIOBufferReader &client_buffer_reader) +{ + int ret = OB_SUCCESS; + + ObShardConnector *shard_conn = NULL; + ObClientSessionInfo &session_info = client_session.get_session_info(); + ObShardConnector *prev_shard_conn = session_info.get_shard_connector(); + + ObProxyMysqlRequest &client_request = trans_state.trans_info_.client_request_; + ObSqlParseResult &parse_result = client_request.get_parse_result(); + SqlFieldResult &sql_result = parse_result.get_sql_filed_result(); + ObProxySelectStmt *select_stmt = dynamic_cast(parse_result.get_proxy_stmt()); + + int64_t group_index = OBPROXY_MAX_DBMESH_ID; + int64_t tb_index = OBPROXY_MAX_DBMESH_ID; + int64_t es_index = OBPROXY_MAX_DBMESH_ID; + ObTestLoadType testload_type = TESTLOAD_NON; + ObString hint_table; + ObArenaAllocator allocator; + + ObHashMap table_name_map; + char real_table_name[OB_MAX_TABLE_NAME_LENGTH]; + char real_database_name[OB_MAX_DATABASE_NAME_LENGTH]; + + // The write request walks the write weight, and requests within the transaction are considered to be write + // read request weight + bool is_read_stmt = ObProxyShardUtils::is_read_stmt(session_info, trans_state, parse_result); + + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select stmt is null, unexpected", K(ret)); + } else if (OB_FAIL(table_name_map.create(OB_ALIAS_TABLE_MAP_MAX_BUCKET_NUM, + ObModIds::OB_HASH_ALIAS_TABLE_MAP))) { + LOG_WARN("fail to create table name map", K(ret)); + } else if (OB_FAIL(get_shard_hint(logic_db_info, parse_result, + group_index, tb_index, es_index, + hint_table, testload_type))) { + LOG_WARN("fail to get shard hint", K(ret)); + } + + if (OB_SUCC(ret)) { + ObProxySelectStmt::ExprMap &table_exprs_map = select_stmt->get_table_exprs_map(); + ObProxySelectStmt::ExprMap::iterator iter = table_exprs_map.begin(); + ObProxySelectStmt::ExprMap::iterator end = table_exprs_map.end(); + + if (!hint_table.empty() && table_exprs_map.size() > 1) { + ret = OB_ERR_MORE_TABLES_WITH_TABLE_HINT; + LOG_WARN("more table with table_name hint", "table size", table_exprs_map.size(), K(hint_table), K(ret)); + } else if (OB_FAIL(logic_db_info.get_shard_table_info(table_name, sql_result, shard_conn, + real_database_name, OB_MAX_DATABASE_NAME_LENGTH, + real_table_name, OB_MAX_TABLE_NAME_LENGTH, + group_index, tb_index, es_index, + hint_table, testload_type, is_read_stmt))) { + LOG_WARN("fail to get real info", K(table_name), K(group_index), K(tb_index), + K(es_index), K(hint_table), K(testload_type), K(is_read_stmt), K(ret)); + } else if (OB_ISNULL(shard_conn) || OB_ISNULL(prev_shard_conn)) { + ret = OB_EXPR_CALC_ERROR; + LOG_WARN("shard connector info or prev shard connector info is null", KP(shard_conn), + KP(prev_shard_conn), K(ret)); + } else if (OB_FAIL(add_table_name_to_map(allocator, table_name_map, table_name, real_table_name))) { + LOG_WARN("fail to add table name to map", K(table_name), K(real_table_name), K(ret)); + } + + for (; OB_SUCC(ret) && iter != end; iter++) { + ObProxyExpr *expr = iter->second; + ObProxyExprTable *table_expr = NULL; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null, unexpected", K(ret)); + } else if (OB_ISNULL(table_expr = dynamic_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to cast to table expr", K(expr), K(ret)); + } else { + ObString &sql_table_name = table_expr->get_table_name(); + if (sql_table_name == table_name) { + continue; + } + + if (OB_FAIL(logic_db_info.get_real_table_name(sql_table_name, sql_result, + real_table_name, OB_MAX_TABLE_NAME_LENGTH, + tb_index, hint_table, testload_type))) { + LOG_WARN("fail to get real table name", K(sql_table_name), K(tb_index), + K(hint_table), K(testload_type), K(ret)); + } else if (OB_FAIL(add_table_name_to_map(allocator, table_name_map, sql_table_name, real_table_name))) { + LOG_WARN("fail to add table name to map", K(sql_table_name), K(real_table_name), K(ret)); + } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(rewrite_shard_select_request(session_info, client_request, client_buffer_reader, + table_name_map, ObString::make_string(real_database_name), false))) { + LOG_WARN("fail to rewrite shard request", K(real_database_name), K(ret)); + } + } + + if (OB_SUCC(ret) && *prev_shard_conn != *shard_conn) { + if (OB_FAIL(change_connector(logic_db_info, client_session, trans_state, prev_shard_conn, shard_conn))) { + LOG_WARN("fail to change connector", KPC(prev_shard_conn), KPC(shard_conn), K(ret)); + } + } + + //It is possible that there is a database in SQL. Only the logical library saved on the session can save the group_id and reuse it next time. + if (OB_SUCC(ret)) { + ObString saved_database_name; + session_info.get_logic_database_name(saved_database_name); + bool is_not_saved_database = saved_database_name != logic_db_info.db_name_.config_string_; + if (!is_not_saved_database) { + session_info.set_group_id(group_index); + } } if (NULL != shard_conn) { shard_conn->dec_ref(); shard_conn = NULL; } - return ret; } @@ -1611,7 +2193,7 @@ int ObProxyShardUtils::handle_dml_real_info(ObDbConfigLogicDb &logic_db_info, &group_id, NULL, NULL, is_read_stmt))) { LOG_WARN("fail to get real db and tb", K(table_name), K(ret)); } else if (OB_ISNULL(shard_conn) || OB_ISNULL(prev_shard_conn)) { - ret = OB_ERR_COULUMN_VALUE_NOT_MATCH; + ret = OB_EXPR_CALC_ERROR; LOG_WARN("shard connector info or prev shard connector info is null", KP(shard_conn), KP(prev_shard_conn), K(ret)); } @@ -1735,11 +2317,9 @@ int ObProxyShardUtils::build_error_packet(int err_code, bool &need_response_for_ { int ret = OB_SUCCESS; switch (err_code) { - case OB_ERR_COULUMN_VALUE_NOT_MATCH: case OB_ERR_WRONG_TYPE_COLUMN_VALUE_ERROR: case OB_NOT_SUPPORTED: case OB_INVALID_ARGUMENT_FOR_SUBSTR: - case OB_INVALID_ARGUMENT_FOR_EXTRACT: case OB_ERR_NO_DB_SELECTED: case OB_ERR_NO_PRIVILEGE: case OB_EXPR_CALC_ERROR: @@ -1759,6 +2339,9 @@ int ObProxyShardUtils::build_error_packet(int err_code, bool &need_response_for_ case OB_ERR_TESTLOAD_ALIPAY_COMPATIBLE: case OB_ERR_NULL_DB_VAL_TESTLOAD_TABLE_MAP: case OB_ERR_BATCH_INSERT_FOUND: + case OB_ERROR_UNSUPPORT_EXPR_TYPE: + case OB_ERR_UNSUPPORT_DIFF_TOPOLOGY: + case OB_ERR_BAD_FIELD_ERROR: need_response_for_dml = true; break; case OB_ENTRY_NOT_EXIST: @@ -1784,43 +2367,39 @@ int ObProxyShardUtils::build_error_packet(int err_code, bool &need_response_for_ return ret; } -int ObProxyShardUtils::handle_sharding_select_real_info( - ObDbConfigLogicDb &logic_db_info, - ObMysqlClientSession &client_session, - ObMysqlTransact::ObTransState &trans_state, - const ObString table_name, - ObIAllocator &allocator, - ObIArray &shard_connector_array, - ObIArray &physical_table_name_array) +int ObProxyShardUtils::handle_sharding_select_real_info(ObDbConfigLogicDb &logic_db_info, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + const ObString table_name, + ObIAllocator &allocator, + ObIArray &shard_connector_array, + ObIArray &shard_prop_array, + ObIArray > &table_name_map_array) { int ret = OB_SUCCESS; ObClientSessionInfo &session_info = client_session.get_session_info(); ObSqlParseResult &parse_result = trans_state.trans_info_.client_request_.get_parse_result(); - bool is_read_stmt = ObProxyShardUtils::is_read_stmt(session_info, trans_state, parse_result); - - DbMeshRouteInfo &odp_route_info = parse_result.get_dbmesh_route_info(); - ObTestLoadType testload_type = TESTLOAD_NON; + int64_t group_index = OBPROXY_MAX_DBMESH_ID; + int64_t tb_index = OBPROXY_MAX_DBMESH_ID; int64_t es_index = OBPROXY_MAX_DBMESH_ID; - if (odp_route_info.is_valid()) { - testload_type = get_testload_type(parse_result.get_dbmesh_route_info().testload_); - es_index = parse_result.get_dbmesh_route_info().es_idx_; - const ObString &disaster_status = parse_result.get_dbmesh_route_info().disaster_status_; - // if disaster status exist and valid, use disaster_eid - // if failed, ignore - if (!disaster_status.empty() && OB_FAIL(logic_db_info.get_disaster_eid(disaster_status, es_index))) { - LOG_DEBUG("fail to get disaster elastic id", K(disaster_status), K(ret)); - ret = OB_SUCCESS; - } - } + ObTestLoadType testload_type = TESTLOAD_NON; + ObString hint_table; + + bool is_read_stmt = ObProxyShardUtils::is_read_stmt(session_info, trans_state, parse_result); - if (OB_FAIL(logic_db_info.get_sharding_select_info(table_name, parse_result, - testload_type, is_read_stmt, es_index, - allocator, - shard_connector_array, - physical_table_name_array))) { - LOG_WARN("fail to get sharding select info", K(ret), K(table_name), - K(testload_type), K(is_read_stmt)); + if (OB_FAIL(get_shard_hint(logic_db_info, parse_result, + group_index, tb_index, es_index, + hint_table, testload_type))) { + LOG_WARN("fail to get shard hint", K(ret)); + } else if (OB_FAIL(logic_db_info.get_sharding_select_info(table_name, parse_result, + testload_type, is_read_stmt, es_index, + allocator, + shard_connector_array, + shard_prop_array, + table_name_map_array))) { + LOG_WARN("fail to get sharding select info", K(table_name), + K(testload_type), K(is_read_stmt), K(ret)); } return ret; diff --git a/src/obproxy/proxy/shard/obproxy_shard_utils.h b/src/obproxy/proxy/shard/obproxy_shard_utils.h index e15bf4666b95378fcf4b11523f15341bf2efc822..3d9ca5494faf571096498629d550eecbb32bbc82 100644 --- a/src/obproxy/proxy/shard/obproxy_shard_utils.h +++ b/src/obproxy/proxy/shard/obproxy_shard_utils.h @@ -73,14 +73,22 @@ public: static int check_shard_request(ObMysqlClientSession &client_session, obutils::ObSqlParseResult &parse_result, dbconfig::ObDbConfigLogicDb &logic_db_info); - static int handle_shard_request(ObMysqlClientSession &client_session, + static int handle_shard_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, ObMysqlTransact::ObTransState &trans_state, ObIOBufferReader &client_buffer_reader, - dbconfig::ObDbConfigLogicDb &db_info); - static int handle_single_shard_request(ObMysqlClientSession &client_session, - ObMysqlTransact::ObTransState &trans_state, - ObIOBufferReader &client_buffer_reader, - dbconfig::ObDbConfigLogicDb &db_info); + dbconfig::ObDbConfigLogicDb &db_info, + bool &need_wait_callback); + static int do_handle_single_shard_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + dbconfig::ObDbConfigLogicDb &logic_db_info); + static int handle_single_shard_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + dbconfig::ObDbConfigLogicDb &logic_db_info, + bool &need_wait_callback); static int build_error_packet(int err_code, bool &need_response_for_dml, ObMysqlTransact::ObTransState &trans_state); @@ -90,7 +98,26 @@ public: const ObString table_name, common::ObIAllocator &allocator, common::ObIArray &shard_connector_array, - common::ObIArray &physcial_table_name_array); + common::ObIArray &shard_prop_array, + common::ObIArray > &table_name_map_array); + static bool need_scan_all(obutils::ObSqlParseResult &parse_result); + static int need_scan_all_by_index(const common::ObString &table_name, + dbconfig::ObDbConfigLogicDb &db_info, + obutils::SqlFieldResult& sql_result, + bool &is_scan_all); + static int check_topology(obutils::ObSqlParseResult &parse_result, + dbconfig::ObDbConfigLogicDb &db_info); + + static int do_rewrite_shard_select_request(const ObString &sql, + obutils::ObSqlParseResult &parse_result, + bool is_oracle_mode, + const common::hash::ObHashMap &table_name_map, + const ObString &real_database_name, + bool is_single_shard_db_table, + common::ObSqlString &new_sql); + static int add_table_name_to_map(ObIAllocator &allocator, + hash::ObHashMap &table_name_map, + const ObString &table_name, const ObString &real_table_name); private: static bool is_read_stmt(ObClientSessionInfo &session_info, @@ -102,11 +129,26 @@ private: const dbconfig::ObShardConnector *prev_shard_conn, dbconfig::ObShardConnector *shard_conn); + static int handle_ddl_request(ObMysqlSM *sm, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + dbconfig::ObDbConfigLogicDb &db_info, + bool &need_wait_callback); static int handle_dml_request(ObMysqlClientSession &client_session, ObMysqlTransact::ObTransState &trans_state, ObIOBufferReader &client_buffer_reader, const common::ObString &table_name, dbconfig::ObDbConfigLogicDb &db_info); + static int handle_other_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + const common::ObString &table_name, + dbconfig::ObDbConfigLogicDb &db_info); + static int handle_select_request(ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + ObIOBufferReader &client_buffer_reader, + const common::ObString &table_name, + dbconfig::ObDbConfigLogicDb &db_info); static int check_logic_database(ObMysqlTransact::ObTransState &trans_state, ObMysqlClientSession &client_session, const ObString &db_name); static void replace_oracle_table(ObSqlString &new_sql, const ObString &real_name, @@ -118,6 +160,12 @@ private: const ObString &table_name, const ObString &database_name, const ObString &real_table_name, const ObString &real_database_name, bool is_single_shard_db_table); + static int rewrite_shard_select_request(ObClientSessionInfo &session_info, + ObProxyMysqlRequest &client_request, + ObIOBufferReader &client_buffer_reader, + const common::hash::ObHashMap &table_name_map, + const ObString &real_database_name, + bool is_single_shard_db_table); static int testload_check_obparser_node_is_valid(const ParseNode *root, const ObItemType &type); static int testload_rewrite_name_base_on_parser_node(common::ObSqlString &new_sql, const char *new_name, @@ -161,6 +209,15 @@ private: const ObString &table_name, char *real_database_name, int64_t db_name_len, char *real_table_name, int64_t tb_name_len); + static int handle_scan_all_real_info(dbconfig::ObDbConfigLogicDb &logic_db_info, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + const ObString &table_name); + static int handle_select_real_info(dbconfig::ObDbConfigLogicDb &logic_db_info, + ObMysqlClientSession &client_session, + ObMysqlTransact::ObTransState &trans_state, + const ObString &table_name, + ObIOBufferReader &client_buffer_reader); static int handle_sys_read_consitency_prop(dbconfig::ObDbConfigLogicDb &logic_db_info, dbconfig::ObShardConnector& shard_conn, ObClientSessionInfo &session_info); diff --git a/src/obproxy/qos/ob_proxy_qos_stat_processor.cpp b/src/obproxy/qos/ob_proxy_qos_stat_processor.cpp index a20511a6f657f88543d8f24f86bd454aa0cef1e8..7b301e3f24a3b9aa284c91c182a925b250fd5f7d 100644 --- a/src/obproxy/qos/ob_proxy_qos_stat_processor.cpp +++ b/src/obproxy/qos/ob_proxy_qos_stat_processor.cpp @@ -96,7 +96,7 @@ int ObProxyQosStatProcessor::store_stat(const ObString &cluster_name, const ObSt ObProxyQosStatNode *node = NULL; if (cluster_name.empty() || tenant_name.empty() || database_name.empty() || user_name.empty()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(cluster_name), K(tenant_name), K(database_name), K(user_name), K(ret)); + LOG_DEBUG("invalid argument", K(cluster_name), K(tenant_name), K(database_name), K(user_name), K(ret)); } else if (OB_FAIL(get_or_create_node(&root_node_, cluster_name, QOS_NODE_TYPE_MIDDLE, node))) { LOG_WARN("fail to get or create node", K(cluster_name), K(ret)); } else if (OB_NOT_NULL(node) && OB_FAIL(store_stat_and_next(reinterpret_cast(node), tenant_name, QOS_NODE_TYPE_MIDDLE, node, cost))) { diff --git a/src/obproxy/utils/ob_proxy_hot_upgrader.cpp b/src/obproxy/utils/ob_proxy_hot_upgrader.cpp index 310b396134ad1b791796796507fdb2073b5a13ad..98fb1e9443df6d2c1e83bc4bc3f106de6ee5472b 100644 --- a/src/obproxy/utils/ob_proxy_hot_upgrader.cpp +++ b/src/obproxy/utils/ob_proxy_hot_upgrader.cpp @@ -13,15 +13,16 @@ #define USING_LOG_PREFIX PROXY #include "utils/ob_proxy_hot_upgrader.h" +#include "iocore/eventsystem/ob_event_processor.h" using namespace oceanbase::common; +using namespace oceanbase::obproxy::event; namespace oceanbase { namespace obproxy { volatile int g_proxy_fatal_errcode = OB_SUCCESS; -volatile int64_t g_client_active_close_count = 0; ObHotUpgraderInfo g_hot_upgrade_info; @@ -51,6 +52,7 @@ void ObHotUpgraderInfo::reset() } memset(upgrade_version_buf_, 0, sizeof(upgrade_version_buf_)); is_inherited_ = false; + parent_hot_upgrade_flag_ = false; } void ObHotUpgraderInfo::set_main_arg(const int32_t argc, char *const *argv) @@ -84,7 +86,29 @@ DEF_TO_STRING(ObHotUpgraderInfo) K_(argc)); for (int32_t i = 0; i < argc_; ++i) { - databuff_printf(buf, buf_len, pos, ", argv[%d]=\"%s\"", i, argv_[i]); + char *sub_str = argv_[i]; + char *end_str = argv_[i] + strlen(argv_[i]); + int32_t skip_len = strlen("password="); + databuff_printf(buf, buf_len, pos, ", argv[%d]=\"", i); + + while (sub_str < end_str) { + char *start_str = sub_str; + if (NULL == (sub_str = strstr(start_str, "password="))) { + databuff_printf(buf, buf_len, pos, "%s", start_str); + break; + } else { + sub_str += skip_len; + databuff_printf(buf, buf_len, pos, "%.*s", (int)(sub_str - start_str), start_str); + databuff_printf(buf, buf_len, pos, "***"); + for (; sub_str < end_str; sub_str++) { + if (*sub_str == ',') { + break; + } + } + } + } + + databuff_printf(buf, buf_len, pos, "\""); } for (int64_t i = 0; i < OB_MAX_INHERITED_ARGC; ++i) { @@ -109,8 +133,6 @@ ObString ObHotUpgraderInfo::get_cmd_string(const ObHotUpgradeCmd cmd) ObString::make_string("local_exit"), ObString::make_string("local_restart"), - ObString::make_string("local_commit"), - ObString::make_string("local_rollback"), }; ObString string; @@ -248,7 +270,8 @@ ObString ObHotUpgraderInfo::get_state_string(const ObHotUpgradeState state) ObString::make_string("HU_STATE_WAIT_HU_CMD"), ObString::make_string("HU_STATE_FORK_NEW_PROXY"), ObString::make_string("HU_STATE_WAIT_CR_CMD"), - ObString::make_string("HU_STATE_WAIT_CR_FINISH") + ObString::make_string("HU_STATE_WAIT_CR_FINISH"), + ObString::make_string("HU_STATE_WAIT_LOCAL_CR_FINISH") }; ObString string; @@ -336,5 +359,21 @@ int ObHotUpgraderInfo::fill_inherited_info(const bool is_server_service_mode, co return ret; } +void ObHotUpgraderInfo::disable_net_accept() +{ + int ret = OB_SUCCESS; + ObThreadId tid = 0; + need_conn_accept_ = false; + for (int64_t i = 0; i < g_event_processor.dedicate_thread_count_; ++i) { + if (DEDICATE_THREAD_ACCEPT == g_event_processor.all_dedicate_threads_[i]->get_dedicate_type()) { + tid = g_event_processor.all_dedicate_threads_[i]->tid_; + if (OB_FAIL(thread_kill(tid, 43))) { + LOG_WARN("fail to do thread_kill", K(tid), K(ret)); + } + } + ret = OB_SUCCESS;//ignore error + } +} + } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/utils/ob_proxy_hot_upgrader.h b/src/obproxy/utils/ob_proxy_hot_upgrader.h index 0582778008c115a865d37e4f77396d592c4ff5aa..95398a390bb1e9003d10e29967cc96255f88f925 100644 --- a/src/obproxy/utils/ob_proxy_hot_upgrader.h +++ b/src/obproxy/utils/ob_proxy_hot_upgrader.h @@ -21,6 +21,7 @@ namespace oceanbase namespace obproxy { #define OBPROXY_INHERITED_FD "OBPROXY_INHERITED_FD" +extern volatile int g_proxy_fatal_errcode; enum ObReloadConfigStatus { @@ -55,8 +56,6 @@ enum ObHotUpgradeCmd HUC_LOCAL_EXIT, //used for a config cmd exit HUC_LOCAL_RESTART, //used for a config cmd restart - HUC_LOCAL_COMMIT, //used for a config cmd commit - HUC_LOCAL_ROLLBACK,//used for a config cmd rollback HUC_MAX, }; @@ -66,6 +65,7 @@ enum ObHotUpgradeState HU_STATE_FORK_NEW_PROXY, //fork new proxy HU_STATE_WAIT_CR_CMD, //wait commit and rollback cmd HU_STATE_WAIT_CR_FINISH, //wait commit and rollback finish + HU_STATE_WAIT_LOCAL_CR_FINISH, //wait commit and rollback finish HU_STATE_MAX, }; @@ -149,7 +149,8 @@ public: void reset(); void set_main_arg(const int32_t argc, char *const *argv); - void disable_net_accept() { need_conn_accept_ = false; } + void disable_net_accept(); + void enable_net_accept() { need_conn_accept_ = true; } bool is_parent() const { return is_parent_; } bool need_reject_metadb() const { return USER_TYPE_METADB == user_rejected_;} bool need_reject_proxyro() const { return USER_TYPE_PROXYRO == user_rejected_;} @@ -174,9 +175,7 @@ public: bool is_local_restart() const { return HUC_LOCAL_RESTART == cmd_; }; bool is_exit() const { return HUC_EXIT == cmd_; }; bool is_local_exit() const { return HUC_LOCAL_EXIT == cmd_; }; - bool is_local_commit() const { return HUC_LOCAL_COMMIT == cmd_; }; - bool is_local_rollback() const { return HUC_LOCAL_ROLLBACK == cmd_; }; - bool is_local_cmd() const { return HUC_LOCAL_EXIT <= cmd_ && cmd_ <= HUC_LOCAL_ROLLBACK; }; + bool is_local_cmd() const { return HUC_LOCAL_EXIT <= cmd_ && cmd_ < HUC_MAX; }; bool is_in_idle_state() const { return HU_STATE_WAIT_HU_CMD == state_; }; bool is_in_wait_cr_state() const { return HU_STATE_WAIT_CR_CMD == state_; }; @@ -223,11 +222,12 @@ public: int64_t upgrade_version_; // mainly used for connection id during server service mode char upgrade_version_buf_[MAX_UPGRADE_VERSION_BUF_SIZE];// used to pass upgrade version to sub process - char restart_buf_[MAX_RESTART_BUF_SIZE];// used to pass upgrade version to sub process int32_t argc_; // main's argc, used to be passed to sub process char *const *argv_; // main's argv char *inherited_argv_[OB_MAX_INHERITED_ARGC];// argv used for inherited sub process + volatile bool parent_hot_upgrade_flag_; + lib::ObMutex hot_upgrade_mutex_; common::ObAddr local_addr_; private: @@ -347,7 +347,9 @@ inline bool ObHotUpgraderInfo::is_graceful_exit_timeout(const ObHRTime cur_time) { return (OB_UNLIKELY(!need_conn_accept_) && OB_LIKELY(graceful_exit_end_time_ > 0) - && graceful_exit_end_time_ < cur_time); + && OB_LIKELY(graceful_exit_end_time_ >= graceful_exit_start_time_) + && graceful_exit_end_time_ < cur_time + && common::OB_SUCCESS != g_proxy_fatal_errcode); } ObHotUpgraderInfo &get_global_hot_upgrade_info(); @@ -375,9 +377,6 @@ private: DISALLOW_COPY_AND_ASSIGN(ObExecCtx); }; -extern volatile int g_proxy_fatal_errcode; -extern volatile int64_t g_client_active_close_count; - extern ObHotUpgraderInfo g_hot_upgrade_info; inline ObHotUpgraderInfo &get_global_hot_upgrade_info() { diff --git a/src/obproxy/utils/ob_proxy_lib.h b/src/obproxy/utils/ob_proxy_lib.h index 3e8ccf8b00971e92e6f1630f8405ac5d62ac3a95..61931fd55d837195b2f64543b0ce55643233a474 100644 --- a/src/obproxy/utils/ob_proxy_lib.h +++ b/src/obproxy/utils/ob_proxy_lib.h @@ -140,6 +140,7 @@ static const int64_t OBPROXY_MAX_TNT_ID_LENGTH = 128; static const int64_t OB_PROXY_MAX_CLUSTER_NAME_LENGTH = 256; static const int64_t OB_PROXY_FULL_USER_NAME_MAX_LEN = oceanbase::common::OB_MYSQL_FULL_USER_NAME_MAX_LEN + OB_PROXY_MAX_CLUSTER_NAME_LENGTH;//username@tenantname#clustername +static const int64_t OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH = OB_PROXY_MAX_CLUSTER_NAME_LENGTH + oceanbase::common::OB_MAX_TENANT_NAME_LENGTH + 1; //tenantname#clustername static const int64_t OB_PROXY_CLUSTER_RESOURCE_ITEM_COUNT = 12; static const int64_t OB_PROXY_MAX_IDC_NAME_LENGTH = oceanbase::common::MAX_ZONE_LENGTH; @@ -164,6 +165,8 @@ static const int64_t OB_NORMAL_MYSQL_CLIENT_COUNT = 4; static const int64_t OB_PROXY_WARN_LOG_BUF_LENGTH = (1 << 20) * 1; static const int64_t OB_PROXY_WARN_LOG_AVG_LENGTH = 512; +static const int64_t OB_PROXY_MAX_INNER_TABLE_COLUMN_NUM = 64; + // errno define #define OPS_START_ERRNO 20000 #define NET_ERRNO OPS_START_ERRNO+100 diff --git a/src/obproxy/utils/ob_proxy_utils.cpp b/src/obproxy/utils/ob_proxy_utils.cpp index 2d8bb4e1f22ea74f51b8845d67bc58bd9ffab508..b4e624e1da9128f2b16b46d1de2fa73116fc6f98 100644 --- a/src/obproxy/utils/ob_proxy_utils.cpp +++ b/src/obproxy/utils/ob_proxy_utils.cpp @@ -326,5 +326,23 @@ int convert_timestamp_to_version(int64_t time_us, char *buf, int64_t len) return ret; } +int paste_tenant_and_cluster_name(const ObString &tenant_name, const ObString &cluster_name, + ObFixedLengthString &key_string) +{ + int ret = OB_SUCCESS; + char buf[OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH]; + int64_t len = 0; + len = static_cast(snprintf(buf, OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH, + "%.*s#%.*s", tenant_name.length(), tenant_name.ptr(), cluster_name.length(), cluster_name.ptr())); + if (OB_UNLIKELY(len <= 0) || OB_UNLIKELY(len >= OB_PROXY_MAX_TENANT_CLUSTER_NAME_LENGTH)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to fill buf", K(ret), K(tenant_name), K(cluster_name)); + } else if (OB_FAIL(key_string.assign(buf))) { + LOG_WARN("assign failed", K(ret)); + } + + return ret; +} + } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/utils/ob_proxy_utils.h b/src/obproxy/utils/ob_proxy_utils.h index 4234f7044dd55a5581c507ad3a675d4940fd742f..300d7c9c327e33e30e4b646b0457459025a2972c 100644 --- a/src/obproxy/utils/ob_proxy_utils.h +++ b/src/obproxy/utils/ob_proxy_utils.h @@ -14,6 +14,8 @@ #define OBPROXY_UTILS_H #include "lib/ob_define.h" #include "common/ob_object.h" +#include "lib/string/ob_fixed_length_string.h" +#include "utils/ob_proxy_lib.h" namespace oceanbase { @@ -74,6 +76,8 @@ int str_replace(char *input_buf, const int32_t input_size, const common::ObString &target_value, int32_t &output_pos); int convert_timestamp_to_version(int64_t time_us, char *buf, int64_t len); +int paste_tenant_and_cluster_name(const common::ObString &tenant_name, const common::ObString &cluster_name, + common::ObFixedLengthString &key_string); } // end of namespace obproxy } // end of namespace oceanbase diff --git a/src/obproxy/utils/ob_ref_hash_map.h b/src/obproxy/utils/ob_ref_hash_map.h index 1c6a4d2406b66ca2d8060e5f65a028f16bc6b073..b31a4c2c4554a01c018499c3f8a9908f266df3dd 100644 --- a/src/obproxy/utils/ob_ref_hash_map.h +++ b/src/obproxy/utils/ob_ref_hash_map.h @@ -21,7 +21,7 @@ namespace oceanbase namespace obproxy { -template class GetKey, +template class GetKey, int64_t default_size = common::OB_MALLOC_NORMAL_BLOCK_SIZE, class Allocator = common::ModulePageAllocator> class ObRefHashMap diff --git a/src/obproxy/utils/ob_zlib_stream_compressor.cpp b/src/obproxy/utils/ob_zlib_stream_compressor.cpp index ad0f663515819512d3e08a33554f22a34e37d808..28bb0c4cc6bfd3da235fb737c2b276ccfcc4d38b 100644 --- a/src/obproxy/utils/ob_zlib_stream_compressor.cpp +++ b/src/obproxy/utils/ob_zlib_stream_compressor.cpp @@ -12,6 +12,7 @@ #define USING_LOG_PREFIX PROXY #include "utils/ob_zlib_stream_compressor.h" +#include "lib/alloc/malloc_hook.h" namespace oceanbase { @@ -126,6 +127,7 @@ int ObZlibStreamCompressor::decompress_init() stream_.opaque = Z_NULL; stream_.avail_in = 0; stream_.next_in = Z_NULL; + lib::glibc_hook_opt = lib::GHO_HOOK; if (Z_OK != (zlib_ret = ::inflateInit(&stream_))) { ret = OB_INIT_FAIL; LOG_WARN("fail to init zlib", K(zlib_ret), K(ret)); @@ -133,6 +135,7 @@ int ObZlibStreamCompressor::decompress_init() type_ = DECOMPRESS_TYPE; is_finished_ = false; } + lib::glibc_hook_opt = lib::GHO_NOHOOK; return ret; } @@ -145,6 +148,7 @@ int ObZlibStreamCompressor::compress_init() stream_.zalloc = Z_NULL; stream_.zfree = Z_NULL; stream_.opaque = Z_NULL; + lib::glibc_hook_opt = lib::GHO_HOOK; if (Z_OK != (zlib_ret = ::deflateInit(&stream_, static_cast(compress_level_)))) { ret = OB_INIT_FAIL; LOG_WARN("fail to init zlib", K(zlib_ret), K(ret)); @@ -152,6 +156,7 @@ int ObZlibStreamCompressor::compress_init() type_ = COMPRESS_TYPE; is_finished_ = false; } + lib::glibc_hook_opt = lib::GHO_NOHOOK; return ret; } @@ -175,7 +180,9 @@ int ObZlibStreamCompressor::decompress(char *dest_start, const int64_t len, int6 stream_.avail_out = static_cast(len); stream_.next_out = reinterpret_cast(dest_start); + lib::glibc_hook_opt = lib::GHO_HOOK; zlib_ret = ::inflate(&stream_, Z_NO_FLUSH); + lib::glibc_hook_opt = lib::GHO_NOHOOK; LOG_DEBUG("zlib decomress complete", K(zlib_ret)); if (Z_STREAM_END == zlib_ret) { // this means the total decompres is finished @@ -240,7 +247,9 @@ int ObZlibStreamCompressor::compress(char *dest_start, const int64_t len, int64_ stream_.avail_out = static_cast(len); stream_.next_out = reinterpret_cast(dest_start); + lib::glibc_hook_opt = lib::GHO_HOOK; zlib_ret = ::deflate(&stream_, static_cast(compress_flush_type_)); + lib::glibc_hook_opt = lib::GHO_NOHOOK; LOG_DEBUG("zlib compress complete", K(zlib_ret), K_(compress_flush_type)); if (Z_STREAM_END == zlib_ret) { // this means the total decompres is finished LOG_DEBUG("zlib comress stream end"); @@ -284,7 +293,9 @@ int ObZlibStreamCompressor::decompress_close() { int ret = OB_SUCCESS; if (DECOMPRESS_TYPE == type_) { + lib::glibc_hook_opt = lib::GHO_HOOK; int zlib_ret = ::inflateEnd(&stream_); + lib::glibc_hook_opt = lib::GHO_NOHOOK; if (Z_OK != zlib_ret) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("fail to decompress close", K(zlib_ret), K(ret)); @@ -298,7 +309,9 @@ int ObZlibStreamCompressor::compress_close() { int ret = OB_SUCCESS; if (COMPRESS_TYPE == type_) { + lib::glibc_hook_opt = lib::GHO_HOOK; int zlib_ret = ::deflateEnd(&stream_); + lib::glibc_hook_opt = lib::GHO_NOHOOK; if (Z_OK != zlib_ret) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("fail to compress_close", K(zlib_ret), K(ret)); diff --git a/src/rpc/obmysql/ob_mysql_global.h b/src/rpc/obmysql/ob_mysql_global.h index 2fcba4382de1fbb0cdd4c2cd6b877db4921aa918..0c6c1c36e2dba9d3cc2fb67e56cb4b6b7180223b 100644 --- a/src/rpc/obmysql/ob_mysql_global.h +++ b/src/rpc/obmysql/ob_mysql_global.h @@ -204,7 +204,7 @@ inline const char *get_emysql_field_type_str(const obmysql::EMySQLFieldType &typ str = "OB_MYSQL_TYPE_TINYBLOB"; break; case obmysql::OB_MYSQL_TYPE_MEDIUM_BLOB: - str = "MYSQL_TYPE_MEDIUMBLOB"; + str = "OB_MYSQL_TYPE_MEDIUMBLOB"; break; case obmysql::OB_MYSQL_TYPE_LONG_BLOB: str = "OB_MYSQL_TYPE_LONGBLOB"; @@ -222,7 +222,7 @@ inline const char *get_emysql_field_type_str(const obmysql::EMySQLFieldType &typ str = "OB_MYSQL_TYPE_GEOMETRY"; break; case obmysql::OB_MYSQL_TYPE_NOT_DEFINED: - str = "MYSQL_TYPE_NOTDEFINED"; + str = "OB_MYSQL_TYPE_NOTDEFINED"; break; default: str = "UNKNOWN_TYPE"; diff --git a/src/rpc/obmysql/ob_mysql_packet.cpp b/src/rpc/obmysql/ob_mysql_packet.cpp index a648f27ce81a0f0b818aa1b4af228be242d3000f..05fb8a8c654748ac7577e7a00bcc8b0224dbb269 100644 --- a/src/rpc/obmysql/ob_mysql_packet.cpp +++ b/src/rpc/obmysql/ob_mysql_packet.cpp @@ -223,7 +223,7 @@ char const *get_mysql_cmd_str(ObMySQLCmd mysql_cmd) "Daemno", // OB_MYSQL_COM_DAEMON, "Binlog dump gtid", // OB_MYSQL_COM_BINLOG_DUMP_GTID, - // "Reset connection", // COM_RESET_CONNECTION, + // "Reset connection", // OB_MYSQL_COM_RESET_CONNECTION, "Prepare Execute", // OB_MYSQL_COM_STMT_PREPARE_EXECUTE, "SEND PIECE DATA", "GET PIECE DATA", diff --git a/src/rpc/obmysql/ob_mysql_packet.h b/src/rpc/obmysql/ob_mysql_packet.h index bdb44a9252d79fe2f3a5e7e3b99430b573fb812b..f7a1df19ea515a550b7b282ba606dea0404e3498 100644 --- a/src/rpc/obmysql/ob_mysql_packet.h +++ b/src/rpc/obmysql/ob_mysql_packet.h @@ -64,7 +64,7 @@ enum ObMySQLCmd OB_MYSQL_COM_BINLOG_DUMP_GTID, - // COM_RESET_CONNECTION, + // OB_MYSQL_COM_RESET_CONNECTION, OB_MYSQL_COM_STMT_PREPARE_EXECUTE = OBPROXY_NEW_MYSQL_CMD_START, OB_MYSQL_COM_STMT_SEND_PIECE_DATA, OB_MYSQL_COM_STMT_GET_PIECE_DATA, diff --git a/src/share/config/ob_common_config.cpp b/src/share/config/ob_common_config.cpp index 0873dce830ceedc923df666a04da98b8d37974a7..f02b5d6d8e066e65778cfaf150bb9c77c2fd02d0 100644 --- a/src/share/config/ob_common_config.cpp +++ b/src/share/config/ob_common_config.cpp @@ -73,15 +73,20 @@ int ObCommonConfig::add_extra_config(const char *config_str, while (OB_SUCC(ret) && OB_LIKELY(NULL != token)) { char *saveptr_one = NULL; const char *name = NULL; + const char *value_tmp = ""; const char *value = NULL; ObConfigItem *const *pp_item = NULL; if (OB_ISNULL(name = STRTOK_R(token, "=", &saveptr_one))) { ret = OB_INVALID_CONFIG; LOG_ERROR("Invalid config string", K(token), K(ret)); - } else if (OB_ISNULL(saveptr_one) || OB_UNLIKELY('\0' == *(value = saveptr_one))) { + } else if (OB_ISNULL(saveptr_one) || OB_UNLIKELY('\0' == *(saveptr_one))) { LOG_INFO("Empty config string", K(token), K(name)); - // ret = OB_INVALID_CONFIG; - name = ""; + value = value_tmp; + } else { + value = saveptr_one; + } + + if (OB_FAIL(ret)) { } else if (OB_ISNULL(pp_item = container_.get(ObConfigStringKey(name)))) { /* make compatible with previous configuration */ ret = check_name ? OB_INVALID_CONFIG : OB_SUCCESS; diff --git a/src/share/part/ob_part_desc.cpp b/src/share/part/ob_part_desc.cpp index 6d2f33ac472341bb7c7eaeea836901b80a7ece32..f59bd5e7eaba42d136c4745513d17f07d1760d0c 100644 --- a/src/share/part/ob_part_desc.cpp +++ b/src/share/part/ob_part_desc.cpp @@ -11,18 +11,24 @@ */ #include "share/part/ob_part_desc.h" +#include "common/ob_obj_cast.h" +#include "obproxy/proxy/mysqllib/ob_proxy_session_info.h" + namespace oceanbase { namespace common { + int ObPartDesc::get_part(common::ObNewRange &range, common::ObIAllocator &allocator, - common::ObIArray &part_ids) + common::ObIArray &part_ids, + ObPartDescCtx &ctx) { UNUSED(range); UNUSED(allocator); UNUSED(part_ids); + UNUSED(ctx); return OB_NOT_IMPLEMENT; } @@ -33,6 +39,58 @@ int ObPartDesc::get_part_by_num(const int64_t num, common::ObIArray &pa return OB_NOT_IMPLEMENT; } +/* + * in order to build ObDataTypeCastParams, get sys var value from session, according to obj type + */ +int ObPartDesc::build_dtc_params(obproxy::proxy::ObClientSessionInfo *session_info, + ObObjType obj_type, + ObDataTypeCastParams &dtc_params) +{ + int ret = OB_SUCCESS; + + if (OB_NOT_NULL(session_info)) { + ObString sys_key_name; + switch (obj_type) { + case ObDateTimeType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_DATE_FORMAT); + break; + case ObTimestampNanoType: + case ObTimestampLTZType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_TIMESTAMP_FORMAT); + break; + case ObTimestampTZType: + sys_key_name = ObString::make_string(oceanbase::sql::OB_SV_NLS_TIMESTAMP_TZ_FORMAT); + break; + default: + break; + } + + if (!sys_key_name.empty()) { + ObObj value_obj; + int sub_ret = OB_SUCCESS; + if (OB_SUCCESS != (sub_ret = session_info->get_sys_variable_value(sys_key_name, value_obj))) { + COMMON_LOG(WARN, "fail to get sys var from session, use standard nls format", K(sub_ret), K(sys_key_name)); + } else { + ObString value_str = value_obj.get_string(); + if (OB_FAIL(dtc_params.set_nls_format_by_type(obj_type, value_str))) { + COMMON_LOG(WARN, "fail to set nls format by type", K(ret), K(obj_type), K(value_str)); + } else { + COMMON_LOG(DEBUG, "succ to set nls format by type", K(obj_type), K(value_str)); + } + } + } + } else { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "fail to build dtc params due to null session", K(ret)); + } + + return ret; +} + +void ObPartDesc::set_accuracy(const ObProxyPartKeyAccuracy &accuracy) +{ + accuracy_ = accuracy; +} } // end of common } // end of oceanbase diff --git a/src/share/part/ob_part_desc.h b/src/share/part/ob_part_desc.h index 629822ba173ab7525af57d4927dff25f8d0179c6..fac0180a134437b6609f473cc0407ebdc4c4d0ee 100644 --- a/src/share/part/ob_part_desc.h +++ b/src/share/part/ob_part_desc.h @@ -15,12 +15,43 @@ #include "common/ob_range2.h" #include "lib/container/ob_iarray.h" +#include "lib/timezone/ob_time_convert.h" #include "share/part/ob_part_mgr_util.h" +#include "obproxy/opsql/expr_parser/ob_expr_parse_result.h" namespace oceanbase { + +namespace obproxy +{ +namespace proxy +{ +class ObClientSessionInfo; +} +} + namespace common { + +class ObPartDescCtx { +public: + ObPartDescCtx() : session_info_(NULL), need_accurate_(false) {} + ObPartDescCtx(obproxy::proxy::ObClientSessionInfo *session_info) + : session_info_(session_info), need_accurate_(false) {} + ObPartDescCtx(obproxy::proxy::ObClientSessionInfo *session_info, bool need_accurate) + : session_info_(session_info), need_accurate_(need_accurate) {} + ~ObPartDescCtx() {} + + obproxy::proxy::ObClientSessionInfo *get_session_info() { return session_info_; } + bool need_accurate() { return need_accurate_; } + +private: + obproxy::proxy::ObClientSessionInfo *session_info_; + + /* need accurate the part key result or not, currently only support insert stmt, need to support update set=x stmt */ + bool need_accurate_; +}; + class ObPartDesc { public: @@ -34,12 +65,17 @@ public: */ virtual int get_part(common::ObNewRange &range, common::ObIAllocator &allocator, - common::ObIArray &part_ids); + common::ObIArray &part_ids, + ObPartDescCtx &ctx); virtual int get_part_by_num(const int64_t num, common::ObIArray &part_ids); void set_part_level(share::schema::ObPartitionLevel part_level) { part_level_ = part_level; } share::schema::ObPartitionLevel get_part_level() { return part_level_; } void set_part_func_type(share::schema::ObPartitionFuncType part_func_type) { part_func_type_ = part_func_type; } share::schema::ObPartitionFuncType get_part_func_type() { return part_func_type_; } + int build_dtc_params(obproxy::proxy::ObClientSessionInfo *session_info, + ObObjType obj_type, + ObDataTypeCastParams &dtc_params); + void set_accuracy(const ObProxyPartKeyAccuracy &accuracy); ObPartDesc() : part_level_(share::schema::PARTITION_LEVEL_ZERO) {}; virtual ~ObPartDesc() {}; @@ -47,7 +83,9 @@ public: DECLARE_VIRTUAL_TO_STRING = 0; share::schema::ObPartitionLevel part_level_; share::schema::ObPartitionFuncType part_func_type_; + ObProxyPartKeyAccuracy accuracy_; }; + } } diff --git a/src/share/part/ob_part_desc_hash.cpp b/src/share/part/ob_part_desc_hash.cpp index 1480eb97cdeac6daf9cfe251b5ba1f1c2dbd70d7..d8a6467a1d82cff25027b6617188868f6ceff31a 100644 --- a/src/share/part/ob_part_desc_hash.cpp +++ b/src/share/part/ob_part_desc_hash.cpp @@ -14,6 +14,7 @@ #include "common/ob_obj_cast.h" #include "share/part/ob_part_desc_hash.h" #include "lib/charset/ob_mysql_global.h" +#include "obproxy/proxy/route/obproxy_expr_calculator.h" namespace oceanbase { @@ -33,7 +34,8 @@ ObPartDescHash::ObPartDescHash() : is_oracle_mode_(false) */ int ObPartDescHash::get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; if (1 != range.get_start_key().get_obj_cnt()) { // single value @@ -44,7 +46,7 @@ int ObPartDescHash::get_part(ObNewRange &range, int64_t part_idx = -1; ObObj &src_obj = const_cast(range.get_start_key().get_obj_ptr()[0]); if (is_oracle_mode_) { - ret = calc_value_for_oracle(src_obj, allocator, part_idx); + ret = calc_value_for_oracle(src_obj, allocator, part_idx, ctx); } else { ret = calc_value_for_mysql(src_obj, allocator, part_idx); } @@ -86,13 +88,11 @@ bool ObPartDescHash::is_oracle_supported_type(const ObObjType type) case ObDateTimeType: case ObCharType: case ObVarcharType: - /* case ObTimestampTZType: case ObTimestampLTZType: case ObTimestampNanoType: - */ case ObRawType: - /* + /* case ObIntervalYMType: case ObIntervalDSType: */ @@ -115,9 +115,6 @@ uint64_t ObPartDescHash::calc_hash_value_with_seed(const ObObj &obj, int64_t see uint64 hval = 0; ObObjType type = obj.get_type(); - // set thread context before call hash func - lib::set_oracle_mode(true); - if (ObCharType == type || ObNCharType == type) { ObObj obj_trimmed; int32_t val_len = obj.get_val_len(); @@ -144,58 +141,76 @@ uint64_t ObPartDescHash::calc_hash_value_with_seed(const ObObj &obj, int64_t see } } - // clear context - lib::set_oracle_mode(false); - return hval; } -int ObPartDescHash::calc_value_for_oracle(ObObj &src_obj, ObIAllocator &allocator, int64_t &part_idx) +int ObPartDescHash::calc_value_for_oracle(ObObj &src_obj, + ObIAllocator &allocator, + int64_t &part_idx, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; - uint64_t hash_val = 0; + ObTimeZoneInfo tz_info; + ObDataTypeCastParams dtc_params; - ObCastCtx cast_ctx(&allocator, NULL, CM_NULL_ON_WARN, cs_type_); - // use src_obj as buf_obj - COMMON_LOG(DEBUG, "begin to cast value for hash oracle", K(src_obj), K(cs_type_)); - if (OB_FAIL(ObObjCasterV2::to_type(obj_type_, cs_type_, cast_ctx, src_obj, src_obj))) { - COMMON_LOG(WARN, "failed to cast obj", K(src_obj), K(obj_type_), K(cs_type_), K(ret)); + if (OB_FAIL(obproxy::proxy::ObExprCalcTool::build_dtc_params_with_tz_info(ctx.get_session_info(), + obj_type_, tz_info, dtc_params))) { + COMMON_LOG(WARN, "fail to build dtc params with tz info", K(ret)); } else { - //1. calc hash value - COMMON_LOG(DEBUG, "end to cast values for hash oracle", K(src_obj), K(cs_type_)); - const ObObjType type = src_obj.get_type(); - if (ObNullType == type) { - //do nothing, hash_code not changed - } else if (!is_oracle_supported_type(type)) { - ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "type is wrong", K(ret), K(src_obj), K(type)); + lib::set_oracle_mode(true); + + uint64_t hash_val = 0; + ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NULL_ON_WARN, cs_type_); + ObAccuracy accuracy(accuracy_.length_, accuracy_.precision_, accuracy_.scale_); + const ObObj *res_obj = &src_obj; + + // use src_obj as buf_obj + COMMON_LOG(DEBUG, "begin to cast value for hash oracle", K(src_obj), K(cs_type_)); + if (OB_FAIL(ObObjCasterV2::to_type(obj_type_, cs_type_, cast_ctx, src_obj, src_obj))) { + COMMON_LOG(WARN, "failed to cast obj", K(ret), K(src_obj), K(obj_type_), K(cs_type_)); + } else if (ctx.need_accurate() + && OB_FAIL(obj_accuracy_check(cast_ctx, accuracy, cs_type_, *res_obj, src_obj, res_obj))) { + COMMON_LOG(WARN, "fail to obj accuracy check", K(ret), K(src_obj), K(obj_type_)); } else { - hash_val = calc_hash_value_with_seed(src_obj, hash_val); + COMMON_LOG(DEBUG, "finish to cast and accuracy values for hash oracle", K(src_obj), K(obj_type_), K(cs_type_)); + // calculate hash value + const ObObjType type = src_obj.get_type(); + if (ObNullType == type) { + //do nothing, hash_code not changed + } else if (!is_oracle_supported_type(type)) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "type is wrong", K(ret), K(src_obj), K(type)); + } else { + hash_val = calc_hash_value_with_seed(src_obj, hash_val); + } } - } - // calc logic part - if (OB_SUCC(ret)) { - int64_t N = 0; - int64_t powN = 0; - const static int64_t max_part_num_log2 = 64; + lib::set_oracle_mode(false); + + // calculate logic partition + if (OB_SUCC(ret)) { + int64_t N = 0; + int64_t powN = 0; + const static int64_t max_part_num_log2 = 64; - int64_t result_num = static_cast(hash_val); - result_num = result_num < 0 ? -result_num : result_num; + int64_t result_num = static_cast(hash_val); + result_num = result_num < 0 ? -result_num : result_num; - N = static_cast(std::log(part_num_) / std::log(2)); - if (N >= max_part_num_log2) { - ret = OB_ERR_UNEXPECTED; - COMMON_LOG(WARN, "result is too big", K(N), K(part_num_), K(result_num)); - } else { - powN = (1ULL << N); - part_idx = result_num % powN; - if (part_idx + powN < part_num_ && (result_num & powN) == powN) { - part_idx += powN; + N = static_cast(std::log(part_num_) / std::log(2)); + if (N >= max_part_num_log2) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "result is too big", K(N), K(part_num_), K(result_num)); + } else { + powN = (1ULL << N); + part_idx = result_num % powN; + if (part_idx + powN < part_num_ && (result_num & powN) == powN) { + part_idx += powN; + } } + COMMON_LOG(DEBUG, "get hash part idx for oracle mode", + K(ret), K(result_num), K(part_num_), K(N), K(powN), K(part_idx)); } - COMMON_LOG(DEBUG, "get hash part idx for oracle mode", K(ret), K(result_num), K(part_num_), K(N), K(powN), K(part_idx)); } return ret; @@ -214,6 +229,11 @@ int ObPartDescHash::calc_value_for_mysql(ObObj &src_obj, ObIAllocator &allocator if (OB_FAIL(src_obj.get_int(val))) { COMMON_LOG(WARN, "fail to get int", K(src_obj), K(ret)); } else { + if (OB_UNLIKELY(INT64_MIN == val)) { + val = INT64_MAX; + } else { + val = val < 0 ? -val : val; + } part_idx = val % part_num_; COMMON_LOG(DEBUG, "get hash part idx for mysql mode", K(ret), K(val), K(part_num_), K(part_idx)); } diff --git a/src/share/part/ob_part_desc_hash.h b/src/share/part/ob_part_desc_hash.h index dc872421f1705a693e2d7244a05c2a31dcee9f4c..67f9b025eb09360fd38ea7bb6527ea65020fb4a8 100644 --- a/src/share/part/ob_part_desc_hash.h +++ b/src/share/part/ob_part_desc_hash.h @@ -28,10 +28,11 @@ public: ObPartDescHash(); virtual ~ObPartDescHash() {} - virtual int get_part(common::ObNewRange &range, - common::ObIAllocator &allocator, - ObIArray &part_ids); - virtual int get_part_by_num(const int64_t num, common::ObIArray &part_ids); + virtual int get_part(ObNewRange &range, + ObIAllocator &allocator, + ObIArray &part_ids, + ObPartDescCtx &ctx); + virtual int get_part_by_num(const int64_t num, ObIArray &part_ids); void set_part_num(int64_t part_num) { part_num_ = part_num; } void set_part_space(int64_t part_space) { part_space_ = part_space; } void set_first_part_id(int64_t first_part_id) { first_part_id_ = first_part_id; } @@ -50,10 +51,9 @@ public: private: uint64_t calc_hash_value_with_seed(const ObObj &obj, int64_t seed); bool is_oracle_supported_type(const ObObjType type); - int calc_value_for_oracle(ObObj &src_obj, common::ObIAllocator &allocator, int64_t &part_idx); - int calc_value_for_mysql(ObObj &src_obj, common::ObIAllocator &allocator, int64_t &part_idx); - int get_part_hash_idx(const int64_t part_idx, - int64_t &part_id); + int calc_value_for_oracle(ObObj &src_obj, ObIAllocator &allocator, int64_t &part_idx, ObPartDescCtx &ctx); + int calc_value_for_mysql(ObObj &src_obj, ObIAllocator &allocator, int64_t &part_idx); + int get_part_hash_idx(const int64_t part_idx, int64_t &part_id); private: bool is_oracle_mode_; diff --git a/src/share/part/ob_part_desc_key.cpp b/src/share/part/ob_part_desc_key.cpp index d24a3243f023b77c7c156a312e1d6918e761ce73..7743213c2f016de9ff0076a82c519aca06400415 100644 --- a/src/share/part/ob_part_desc_key.cpp +++ b/src/share/part/ob_part_desc_key.cpp @@ -12,6 +12,8 @@ #include "common/ob_obj_cast.h" #include "share/part/ob_part_desc_key.h" +#include "obproxy/proxy/route/obproxy_expr_calculator.h" + namespace oceanbase { @@ -33,37 +35,52 @@ ObPartDescKey::ObPartDescKey() : part_num_(0) */ int ObPartDescKey::get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; if (1 != range.get_start_key().get_obj_cnt()) { // single value ret = OB_INVALID_ARGUMENT; - COMMON_LOG(DEBUG, "key part should be single key", + COMMON_LOG(WARN, "key part should be single key", "obj_cnt", range.get_start_key().get_obj_cnt(), K(ret)); } else { - ObObj &src_obj = const_cast(range.get_start_key().get_obj_ptr()[0]); - ObCastCtx cast_ctx(&allocator, NULL, CM_NULL_ON_WARN, cs_type_); - // use src_obj as buf_obj - if (OB_FAIL(ObObjCasterV2::to_type(obj_type_, cs_type_, cast_ctx, src_obj, src_obj))) { - COMMON_LOG(INFO, "failed to cast obj", K(src_obj), K(obj_type_), K(cs_type_), K(ret)); + ObTimeZoneInfo tz_info; + ObDataTypeCastParams dtc_params; + + if (OB_FAIL(obproxy::proxy::ObExprCalcTool::build_dtc_params_with_tz_info(ctx.get_session_info(), + obj_type_, tz_info, dtc_params))) { + COMMON_LOG(WARN, "fail to build dtc params with ctx session", K(ret), K(obj_type_)); } else { - int64_t result_num = 0; - if (share::schema::PARTITION_FUNC_TYPE_KEY_V3 == part_func_type_ - || share::schema::PARTITION_FUNC_TYPE_KEY_IMPLICIT_V2 == part_func_type_) { - result_num = static_cast(src_obj.hash_murmur()); + ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NULL_ON_WARN, cs_type_); + ObAccuracy accuracy(accuracy_.length_, accuracy_.precision_, accuracy_.scale_); + ObObj &src_obj = const_cast(range.get_start_key().get_obj_ptr()[0]); + const ObObj *res_obj = &src_obj; + + // use src_obj as buf_obj + if (OB_FAIL(ObObjCasterV2::to_type(obj_type_, cs_type_, cast_ctx, src_obj, src_obj))) { + COMMON_LOG(WARN, "fail to cast obj", K(ret), K(src_obj), K(obj_type_), K(cs_type_)); + } else if (ctx.need_accurate() + && OB_FAIL(obj_accuracy_check(cast_ctx, accuracy, cs_type_, *res_obj, src_obj, res_obj))) { + COMMON_LOG(WARN, "fail to obj accuracy check", K(ret), K(src_obj), K(obj_type_)); } else { - result_num = static_cast(src_obj.hash()); - } - result_num = result_num < 0 ? -result_num : result_num; - int64_t part_idx = result_num % part_num_; - int64_t part_id = part_idx; - if (share::schema::PARTITION_LEVEL_ONE == part_level_ && NULL != part_array_) { - part_id = part_array_[part_idx]; - } - part_id = part_space_ << OB_PART_IDS_BITNUM | part_id; - if (OB_FAIL(part_ids.push_back(part_id))) { - COMMON_LOG(WARN, "fail to push part_id", K(ret)); + int64_t result_num = 0; + if (share::schema::PARTITION_FUNC_TYPE_KEY_V3 == part_func_type_ + || share::schema::PARTITION_FUNC_TYPE_KEY_IMPLICIT_V2 == part_func_type_) { + result_num = static_cast(src_obj.hash_murmur()); + } else { + result_num = static_cast(src_obj.hash()); + } + result_num = result_num < 0 ? -result_num : result_num; + int64_t part_idx = result_num % part_num_; + int64_t part_id = part_idx; + if (share::schema::PARTITION_LEVEL_ONE == part_level_ && NULL != part_array_) { + part_id = part_array_[part_idx]; + } + part_id = part_space_ << OB_PART_IDS_BITNUM | part_id; + if (OB_FAIL(part_ids.push_back(part_id))) { + COMMON_LOG(WARN, "fail to push part_id", K(ret)); + } } } } diff --git a/src/share/part/ob_part_desc_key.h b/src/share/part/ob_part_desc_key.h index 1815e788a09ab02dc74de742504eefbd02dc4618..7ddd342fcc61938d09a353e68ae1b1e279478d52 100644 --- a/src/share/part/ob_part_desc_key.h +++ b/src/share/part/ob_part_desc_key.h @@ -28,7 +28,8 @@ public: virtual int get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids); + ObIArray &part_ids, + ObPartDescCtx &ctx); virtual int get_part_by_num(const int64_t num, common::ObIArray &part_ids); void set_part_num(int64_t part_num) { part_num_ = part_num; } void set_part_space(int64_t part_space) { part_space_ = part_space; } diff --git a/src/share/part/ob_part_desc_list.cpp b/src/share/part/ob_part_desc_list.cpp index a19782258c9be878232c69da50bfe3ba7acacb37..68cb0342a2ca14e755691fec79131fc94f7f4e80 100644 --- a/src/share/part/ob_part_desc_list.cpp +++ b/src/share/part/ob_part_desc_list.cpp @@ -12,6 +12,8 @@ #include "common/ob_obj_cast.h" #include "share/part/ob_part_desc_list.h" +#include "obproxy/proxy/route/obproxy_expr_calculator.h" + namespace oceanbase { @@ -21,7 +23,6 @@ namespace common ObPartDescList::ObPartDescList() : part_array_ (NULL) , part_array_size_(0) , default_part_array_idx_(OB_INVALID_INDEX) - , collation_type_(CS_TYPE_UTF8MB4_BIN) { } @@ -47,7 +48,8 @@ ListPartition::ListPartition() : part_id_(0), rows_() int ObPartDescList::get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; @@ -64,7 +66,7 @@ int ObPartDescList::get_part(ObNewRange &range, ObObj &src_obj = const_cast(range.get_start_key().get_obj_ptr()[0]); // use the first row cell as target obj ObObj &target_obj = const_cast(part_array_[0].rows_[0].get_cell(0)); - if (OB_FAIL(cast_obj(src_obj, target_obj, allocator))) { + if (OB_FAIL(cast_obj(src_obj, target_obj, allocator, ctx))) { COMMON_LOG(INFO, "fail to cast obj", K(src_obj), K(target_obj), K(ret)); } else { bool found = false; @@ -108,16 +110,35 @@ int ObPartDescList::get_part_by_num(const int64_t num, common::ObIArray inline int ObPartDescList::cast_obj(ObObj &src_obj, ObObj &target_obj, - ObIAllocator &allocator) + ObIAllocator &allocator, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; - COMMON_LOG(DEBUG, "begin to cast obj for list", K(src_obj), K(target_obj), K_(collation_type)); - ObCastCtx cast_ctx(&allocator, NULL, CM_NULL_ON_WARN, target_obj.get_collation_type()); - // use src_obj as buf_obj - if (OB_FAIL(ObObjCasterV2::to_type(target_obj.get_type(), collation_type_, cast_ctx, src_obj, src_obj))) { - COMMON_LOG(WARN, "failed to cast obj", K(src_obj), K(target_obj), K_(collation_type), K(ret)); + COMMON_LOG(DEBUG, "begin to cast obj for list", K(src_obj), K(target_obj)); + + ObTimeZoneInfo tz_info; + ObDataTypeCastParams dtc_params; + ObObjType obj_type = target_obj.get_type(); + + if (OB_FAIL(obproxy::proxy::ObExprCalcTool::build_dtc_params_with_tz_info(ctx.get_session_info(), + obj_type, tz_info, dtc_params))) { + COMMON_LOG(WARN, "fail to build dtc params with ctx session", K(ret), K(obj_type)); + } else { + ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NULL_ON_WARN, target_obj.get_collation_type()); + ObAccuracy accuracy(accuracy_.length_, accuracy_.precision_, accuracy_.scale_); + const ObObj *res_obj = &src_obj; + + // use src_obj as buf_obj + if (OB_FAIL(ObObjCasterV2::to_type(obj_type, target_obj.get_collation_type(), cast_ctx, src_obj, src_obj))) { + COMMON_LOG(WARN, "failed to cast obj", K(ret), K(src_obj), K(target_obj)); + } else if (ctx.need_accurate() + && OB_FAIL(obj_accuracy_check(cast_ctx, accuracy, target_obj.get_collation_type(), *res_obj, src_obj, res_obj))) { + COMMON_LOG(WARN, "fail to obj accuracy check", K(ret), K(src_obj), K(obj_type)); + } else { + COMMON_LOG(DEBUG, "succ to cast obj for list", K(src_obj), K(target_obj)); + } } - COMMON_LOG(DEBUG, "end to cast obj for list", K(src_obj), K(target_obj), K_(collation_type)); + return ret; } diff --git a/src/share/part/ob_part_desc_list.h b/src/share/part/ob_part_desc_list.h index 07ed11483296eac2c7c9a7cac4febae883d8ab9e..6d58340e0bd027b967eab303874561321ad026a3 100644 --- a/src/share/part/ob_part_desc_list.h +++ b/src/share/part/ob_part_desc_list.h @@ -40,7 +40,8 @@ public: virtual int get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids); + ObIArray &part_ids, + ObPartDescCtx &ctx); virtual int get_part_by_num(const int64_t num, common::ObIArray &part_ids); void set_default_part_array_idx(int64_t idx) { default_part_array_idx_ = idx; } int64_t get_default_part_array_idx() const { return default_part_array_idx_; } @@ -51,21 +52,16 @@ public: return OB_SUCCESS; } - void set_collation_type(const ObCollationType collation_type) - { - collation_type_ = collation_type; - } - DECLARE_VIRTUAL_TO_STRING; private: int cast_obj(ObObj &src_obj, ObObj &target_obj, - ObIAllocator &allocator); + ObIAllocator &allocator, + ObPartDescCtx &ctx); private: ListPartition *part_array_; int64_t part_array_size_; int64_t default_part_array_idx_; - ObCollationType collation_type_; }; } // end common diff --git a/src/share/part/ob_part_desc_range.cpp b/src/share/part/ob_part_desc_range.cpp index 306e63c4315b064e5e5a76c2c965a0fd0b2d8fa0..313d5d64915daa31023ddc1e896aff632c4222dc 100644 --- a/src/share/part/ob_part_desc_range.cpp +++ b/src/share/part/ob_part_desc_range.cpp @@ -12,6 +12,8 @@ #include "common/ob_obj_cast.h" #include "share/part/ob_part_desc_range.h" +#include "obproxy/proxy/route/obproxy_expr_calculator.h" + namespace oceanbase { @@ -32,7 +34,6 @@ bool RangePartition::less_than(const RangePartition &a, const RangePartition &b) ObPartDescRange::ObPartDescRange() : part_array_ (NULL) , part_array_size_(0) - , collation_type_(CS_TYPE_UTF8MB4_BIN) { } @@ -124,7 +125,8 @@ RangePartition::RangePartition() : is_max_value_(false) int ObPartDescRange::get_part(ObNewRange &range, ObIAllocator &allocator, - ObIArray &part_ids) + ObIArray &part_ids, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; part_ids.reset(); @@ -136,10 +138,10 @@ int ObPartDescRange::get_part(ObNewRange &range, ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "invalid argument", K_(part_array), K_(part_array_size), K(range), K(ret)); // use the fisrt range as the type to cast - } else if (OB_FAIL(cast_key(range.start_key_, part_array_[0].high_bound_val_, allocator))) { + } else if (OB_FAIL(cast_key(range.start_key_, part_array_[0].high_bound_val_, allocator, ctx))) { COMMON_LOG(INFO, "fail to cast start key ", K(range), K(part_array_[0].high_bound_val_), K(ret)); - } else if (OB_FAIL(cast_key(range.end_key_, part_array_[0].high_bound_val_, allocator))) { + } else if (OB_FAIL(cast_key(range.end_key_, part_array_[0].high_bound_val_, allocator, ctx))) { COMMON_LOG(INFO, "fail to cast end key", K(range), K(part_array_[0].high_bound_val_), K(ret)); } else { @@ -167,14 +169,16 @@ int ObPartDescRange::get_part_by_num(const int64_t num, common::ObIArray(src_key.get_obj_ptr()[i]), const_cast(target_key.get_obj_ptr()[i]), - allocator))) { + allocator, + ctx))) { COMMON_LOG(INFO, "fail to cast obj", K(i), K(ret)); } else { // do nothing @@ -185,16 +189,35 @@ int ObPartDescRange::cast_key(ObRowkey &src_key, inline int ObPartDescRange::cast_obj(ObObj &src_obj, ObObj &target_obj, - ObIAllocator &allocator) + ObIAllocator &allocator, + ObPartDescCtx &ctx) { int ret = OB_SUCCESS; - COMMON_LOG(DEBUG, "begin to cast obj for range", K(src_obj), K(target_obj), K_(collation_type)); - ObCastCtx cast_ctx(&allocator, NULL, CM_NULL_ON_WARN, target_obj.get_collation_type()); - // use src_obj as buf_obj - if (OB_FAIL(ObObjCasterV2::to_type(target_obj.get_type(), collation_type_, cast_ctx, src_obj, src_obj))) { - COMMON_LOG(WARN, "failed to cast obj", K(src_obj), K(target_obj), K_(collation_type), K(ret)); + COMMON_LOG(DEBUG, "begin to cast obj for range", K(src_obj), K(target_obj)); + + ObTimeZoneInfo tz_info; + ObDataTypeCastParams dtc_params; + ObObjType obj_type = target_obj.get_type(); + + if (OB_FAIL(obproxy::proxy::ObExprCalcTool::build_dtc_params_with_tz_info(ctx.get_session_info(), + obj_type, tz_info, dtc_params))) { + COMMON_LOG(WARN, "fail to build dtc params with ctx session", K(ret), K(obj_type)); + } else { + ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NULL_ON_WARN, target_obj.get_collation_type()); + ObAccuracy accuracy(accuracy_.length_, accuracy_.precision_, accuracy_.scale_); + const ObObj *res_obj = &src_obj; + + // use src_obj as buf_obj + if (OB_FAIL(ObObjCasterV2::to_type(obj_type, target_obj.get_collation_type(), cast_ctx, src_obj, src_obj))) { + COMMON_LOG(WARN, "failed to cast obj", K(ret), K(src_obj), K(target_obj)); + } else if (ctx.need_accurate() + && OB_FAIL(obj_accuracy_check(cast_ctx, accuracy, target_obj.get_collation_type(), *res_obj, src_obj, res_obj))) { + COMMON_LOG(WARN, "fail to obj accuracy check", K(ret), K(src_obj)); + } else { + COMMON_LOG(DEBUG, "end to cast obj for range", K(src_obj), K(target_obj)); + } } - COMMON_LOG(DEBUG, "end to cast obj for range", K(src_obj), K(target_obj), K_(collation_type)); + return ret; } diff --git a/src/share/part/ob_part_desc_range.h b/src/share/part/ob_part_desc_range.h index 17892d1140356f33893c7ae2be2c32fd085504d2..e7465b02e9ab94bf0951ea07ff3667c8b28b466c 100644 --- a/src/share/part/ob_part_desc_range.h +++ b/src/share/part/ob_part_desc_range.h @@ -43,7 +43,8 @@ public: virtual int get_part(common::ObNewRange &range, common::ObIAllocator &allocator, - ObIArray &part_ids); + ObIArray &part_ids, + ObPartDescCtx &ctx); virtual int get_part_by_num(const int64_t num, common::ObIArray &part_ids); RangePartition* get_part_array() { return part_array_; } int set_part_array(RangePartition *part_array, int64_t size) { @@ -52,11 +53,6 @@ public: return common::OB_SUCCESS; } - void set_collation_type(const ObCollationType collation_type) - { - collation_type_ = collation_type; - } - DECLARE_VIRTUAL_TO_STRING; private: int64_t get_start(const RangePartition *part_array, @@ -69,15 +65,16 @@ private: int cast_key(ObRowkey &src_key, ObRowkey &target_key, - ObIAllocator &allocator); + ObIAllocator &allocator, + ObPartDescCtx &ctx); int cast_obj(ObObj &src_obj, ObObj &target_obj, - ObIAllocator &allocator); + ObIAllocator &allocator, + ObPartDescCtx &ctx); private: RangePartition *part_array_; int64_t part_array_size_; - ObCollationType collation_type_; }; } } diff --git a/unittest/obproxy/Makefile.am b/unittest/obproxy/Makefile.am index 9a1a19fa15e7772e120225fa0ccc945176408aeb..ec9c1f389eea3aa2981fcaadadeaa663307d6cd6 100644 --- a/unittest/obproxy/Makefile.am +++ b/unittest/obproxy/Makefile.am @@ -8,6 +8,7 @@ AM_CPPFLAGS += -I${top_srcdir}/src/obproxy/iocore/eventsystem \ -I${top_srcdir}/src/obproxy/proxy/mysql \ -I${top_srcdir}/src/obproxy/proxy/mysqllib \ -I${top_srcdir}/src/obproxy/proxy/api \ + -I${top_srcdir}/src/obproxy/proxy/omt \ -I${top_srcdir}/src/obproxy/obutils \ -I${top_srcdir}/src/obproxy/utils \ -I${top_srcdir}/src/lib/hash \ @@ -22,6 +23,7 @@ LIBTOOLFLAGS=--preserve-dup-deps LDADD = $(top_builddir)/src/obproxy/libobproxy.la \ $(top_builddir)/src/lib/compress/libzlib_1.0.la \ + $(top_builddir)/src/lib/compress/libzstd_1.3.8.la \ ${DEP_DIR}/lib64/libprometheus-cpp-pull.a \ ${DEP_DIR}/lib64/libprometheus-cpp-core.a \ ${RUNTIME_DIR}/lib/libob_sql_proxy_parser_static.a \ @@ -36,6 +38,7 @@ LDADD = $(top_builddir)/src/obproxy/libobproxy.la \ ${DEP_DIR}/lib/libssl.a \ ${DEP_DIR}/lib/libcurl.a \ ${DEP_DIR}/lib/libcrypto.a \ + ${DEP_DIR}/lib/sqlite/libsqlite3.a \ -lc -lm -lrt -ldl -lpthread pub_sources= \ @@ -63,11 +66,13 @@ bin_PROGRAMS = \ test_resultset_fetcher \ test_config_server_processor \ test_vip_tenant_cache \ + test_white_list_processor \ + test_tenant_processor \ test_proxy_json_config_info \ test_mysql_transact \ test_ldc_location \ obproxy_task_flow_controller_checker \ - ob_expr_parser_checker \ + ob_expr_parser_checker \ ob_func_expr_parser_checker \ test_zlib_stream_compressor \ test_fast_zlib_stream_compressor \ @@ -107,6 +112,8 @@ test_unix_net_SOURCES = test_unix_net.cpp ${pub_sources} test_unix_net_vconnection_SOURCES = test_unix_net_vconnection.cpp ${pub_sources} test_resultset_fetcher_SOURCES = test_resultset_fetcher.cpp ${pub_sources} test_vip_tenant_cache_SOURCES = test_vip_tenant_cache.cpp +test_white_list_processor_SOURCES = test_white_list_processor.cpp +test_tenant_processor_SOURCES = test_tenant_processor.cpp test_proxy_json_config_info_SOURCES = test_proxy_json_config_info.cpp test_mysql_transact_SOURCES = test_mysql_transact.cpp test_ldc_location_SOURCES = test_ldc_location.cpp diff --git a/unittest/obproxy/foo_client.cpp b/unittest/obproxy/foo_client.cpp index 10aa8fdf901780b75b21195c3a3e598d876a11d1..33bb2d02388ffc5999e3580b633c877c1bc611cb 100644 --- a/unittest/obproxy/foo_client.cpp +++ b/unittest/obproxy/foo_client.cpp @@ -85,7 +85,7 @@ void FooClient::refresh_server_addr() } while (!in.eof()) { - if (in.getline(buffer, MAX_BUF_LEN) <= 0) { + if (in.getline(buffer, MAX_BUF_LEN)) { break; } sscanf(buffer, "%[^:]:%d", ip, &port); diff --git a/unittest/obproxy/ob_expr_parser_checker.cpp b/unittest/obproxy/ob_expr_parser_checker.cpp index 64c1a2dc90d85220c2f905e4d89043dccb268087..da4fcf7e12d0b1d39cc0e71d2478673687d612ef 100644 --- a/unittest/obproxy/ob_expr_parser_checker.cpp +++ b/unittest/obproxy/ob_expr_parser_checker.cpp @@ -10,17 +10,18 @@ * See the Mulan PubL v2 for more details. */ -#include "ob_expr_parser_checker.h" -#include "lib/utility/ob_print_utils.h" -#include "obproxy/opsql/expr_parser/ob_expr_parser_utils.h" -#include "obutils/ob_proxy_config_processor.h" -#include "obproxy/proxy/route/obproxy_part_info.h" #include #include #include #include #include #include +#include "ob_expr_parser_checker.h" +#define private public +#include "lib/utility/ob_print_utils.h" +#include "obproxy/opsql/expr_parser/ob_expr_parser_utils.h" +#include "obutils/ob_proxy_config_processor.h" +#include "obproxy/proxy/route/obproxy_part_info.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::opsql; @@ -293,7 +294,7 @@ void ObExprParserChecker::print_stat() } // end of namespace obproxy } // end of namespace oceanbase -extern int ob_expr_parser_utf8_yydebug; +int ob_expr_parser_utf8_yydebug; using namespace oceanbase::obproxy::test; int main(int argc, char **argv) { diff --git a/unittest/obproxy/ob_expr_parser_checker.h b/unittest/obproxy/ob_expr_parser_checker.h index 127f1e9db5f7065447c458e0cfe62bfe7b03aadc..9c17953516f3c5081087426a77d8f1d18e499234 100644 --- a/unittest/obproxy/ob_expr_parser_checker.h +++ b/unittest/obproxy/ob_expr_parser_checker.h @@ -10,7 +10,6 @@ * See the Mulan PubL v2 for more details. */ -#define private public #include "lib/allocator/page_arena.h" //#include "sql/parser/ob_parser.h" #include "obutils/ob_proxy_sql_parser.h" diff --git a/unittest/obproxy/ob_func_expr_parser_checker.cpp b/unittest/obproxy/ob_func_expr_parser_checker.cpp index fbbcd9d400baebc1756dcaa133bdfc6de3d3994e..67587d50c302ef389b1338c1f0a3d7fb7449c3d0 100644 --- a/unittest/obproxy/ob_func_expr_parser_checker.cpp +++ b/unittest/obproxy/ob_func_expr_parser_checker.cpp @@ -11,12 +11,13 @@ */ #define USING_LOG_PREFIX PROXY -#include "ob_func_expr_parser_checker.h" -#include "lib/utility/ob_print_utils.h" -#include "opsql/func_expr_parser/ob_func_expr_parser_utils.h" + #include #include #include +#include "ob_func_expr_parser_checker.h" +#include "lib/utility/ob_print_utils.h" +#include "opsql/func_expr_parser/ob_func_expr_parser_utils.h" using namespace oceanbase::common; using namespace oceanbase::obproxy::opsql; @@ -96,7 +97,7 @@ void ObFuncExprParserChecker::print_stat() } // end of namespace obproxy } // end of namespace oceanbase -extern int obfuncexprdebug; +int obfuncexprdebug; using namespace oceanbase::obproxy::test; int main(int argc, char **argv) { diff --git a/unittest/obproxy/ob_func_expr_parser_checker.h b/unittest/obproxy/ob_func_expr_parser_checker.h index 8a62cf6f147fea02cad545b97eacdb01d55e3b0f..2f49337795dbfca6042298200b08a8bf0e17da8a 100644 --- a/unittest/obproxy/ob_func_expr_parser_checker.h +++ b/unittest/obproxy/ob_func_expr_parser_checker.h @@ -10,7 +10,6 @@ * See the Mulan PubL v2 for more details. */ -#define private public #include "lib/allocator/page_arena.h" //#include "sql/parser/ob_parser.h" #include "obutils/ob_proxy_sql_parser.h" diff --git a/unittest/obproxy/obproxy_parser_checker.cpp b/unittest/obproxy/obproxy_parser_checker.cpp index 10108df6a6b3ab1ce351e0b3b7a86f81a8a888f4..f6446e9e1d310d7f6345fab0c5f656f714abea62 100644 --- a/unittest/obproxy/obproxy_parser_checker.cpp +++ b/unittest/obproxy/obproxy_parser_checker.cpp @@ -162,7 +162,7 @@ void ObProxyParserChecker::print_stat() #define YYDEBUG 1 #if YYDEBUG -extern int ob_proxy_parser_utf8_yydebug; +int ob_proxy_parser_utf8_yydebug; #endif using namespace oceanbase::obproxy::test; int main(int argc, char **argv) diff --git a/unittest/obproxy/obproxy_parser_checker.h b/unittest/obproxy/obproxy_parser_checker.h index 690a45740bd1826366f36491c209dc45dafbf0c5..c4a1e48d64f00ad4817fdc44e54b7bcf8e02e8fc 100644 --- a/unittest/obproxy/obproxy_parser_checker.h +++ b/unittest/obproxy/obproxy_parser_checker.h @@ -10,9 +10,9 @@ * See the Mulan PubL v2 for more details. */ -#define private public #include "lib/allocator/page_arena.h" //#include "sql/parser/ob_parser.h" +#define private public #include "obutils/ob_proxy_sql_parser.h" #include "opsql/parser/ob_proxy_parser.h" diff --git a/unittest/obproxy/obproxy_parser_test.cpp b/unittest/obproxy/obproxy_parser_test.cpp index ec74427591edc9a9434dd80184f3e85364e27d58..f091eebcd2881cd0ce23aa6084538d1690995ed2 100755 --- a/unittest/obproxy/obproxy_parser_test.cpp +++ b/unittest/obproxy/obproxy_parser_test.cpp @@ -60,9 +60,9 @@ void show_sql_result(SqlFieldResult& sql_result) { if(sql_result.fields_[i].value_type_ == TOKEN_INT_VAL) { snprintf(buf, 256, "column_value:%ld", sql_result.fields_[i].column_int_value_); } else if (sql_result.fields_[i].value_type_ == TOKEN_STR_VAL){ - snprintf(buf, sql_result.fields_[i].column_value_.string_.length()+1, "column_value:%.*s", - sql_result.fields_[i].column_value_.string_.length(), - sql_result.fields_[i].column_value_.string_.ptr()); + snprintf(buf, sql_result.fields_[i].column_value_.config_string_.length()+1, "column_value:%.*s", + sql_result.fields_[i].column_value_.config_string_.length(), + sql_result.fields_[i].column_value_.config_string_.ptr()); } printf("%s\n", buf); // fprintf(stdout, "%s %s %s\n", sql_result.fields_[i].column_name_, @@ -242,7 +242,7 @@ void extract_local_fileds(const ObExprParseResult& result, ObProxyMysqlRequest & // sql_result.fields_[sql_result.field_num_].column_value_ = std::string(buf); } else if(relation_expr->right_value_->head_->type_ == TOKEN_STR_VAL) { field.value_type_ = TOKEN_STR_VAL; - field.column_value_.string_.assign_ptr( + field.column_value_.config_string_.assign_ptr( relation_expr->right_value_->head_->str_value_.str_, relation_expr->right_value_->head_->str_value_.str_len_); LOG_DEBUG("field.column_value", K(field.column_value_)); @@ -280,7 +280,7 @@ int parse_sql_fileds(ObProxyMysqlRequest &client_request, ObString expr_sql) { } else if (OB_ISNULL(allocator)) { ret = OB_ERR_UNEXPECTED; } else { - ObExprParseMode parse_mode = INVLIAD_PARSE_MODE; + ObExprParseMode parse_mode = INVALID_PARSE_MODE; if (sql_parse_result.is_select_stmt() || sql_parse_result.is_delete_stmt()) { // we treat delete as select parse_mode = SELECT_STMT_PARSE_MODE; diff --git a/unittest/obproxy/test_config_server_processor.cpp b/unittest/obproxy/test_config_server_processor.cpp index abf3261ec4e3bc330763971ac7429af7b5ab0259..d1636e028ed6e88cc81fed1203f528d354938da5 100644 --- a/unittest/obproxy/test_config_server_processor.cpp +++ b/unittest/obproxy/test_config_server_processor.cpp @@ -10,10 +10,10 @@ * See the Mulan PubL v2 for more details. */ -#define private public #define USING_LOG_PREFIX PROXY #include #include +#define private public #include "lib/oblog/ob_log.h" #include "lib/container/ob_se_array.h" #include "lib/string/ob_sql_string.h" diff --git a/unittest/obproxy/test_continuation.cpp b/unittest/obproxy/test_continuation.cpp index 51e5ab532e144fe64b9215ef9f5e5ee82f2a986b..2fc8c7f5c5363bf7c8e28363d39f734de50f8fb5 100644 --- a/unittest/obproxy/test_continuation.cpp +++ b/unittest/obproxy/test_continuation.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "obproxy/iocore/eventsystem/ob_continuation.h" diff --git a/unittest/obproxy/test_ethread.cpp b/unittest/obproxy/test_ethread.cpp index bfea83a4eb22e415520f326b7092d1bdb066da4d..1fac3f379311cfaafc787b5eddf220ca97df6f31 100644 --- a/unittest/obproxy/test_ethread.cpp +++ b/unittest/obproxy/test_ethread.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "obproxy/iocore/eventsystem/ob_ethread.h" diff --git a/unittest/obproxy/test_event.cpp b/unittest/obproxy/test_event.cpp index a49e40ebd4f7483f2f570a8f63134830a2a78399..59c29a27f232f23eab9724d29af8fb7e9ea68414 100644 --- a/unittest/obproxy/test_event.cpp +++ b/unittest/obproxy/test_event.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "obproxy/iocore/eventsystem/ob_event.h" #include "obproxy/iocore/eventsystem/ob_action.h" diff --git a/unittest/obproxy/test_event_processor.cpp b/unittest/obproxy/test_event_processor.cpp index cc7017dda9b6c5b938f6d22364ea0d0ab6dbd4a6..c249e237910aa7d213e07b05e98c69c7bdc37de5 100644 --- a/unittest/obproxy/test_event_processor.cpp +++ b/unittest/obproxy/test_event_processor.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" namespace oceanbase diff --git a/unittest/obproxy/test_field_heap.cpp b/unittest/obproxy/test_field_heap.cpp index 35988daf0f6644c5f00121f7b56f41ec673b002a..269244fa9ac5316df86aef54e7d8ac20ca5d077c 100644 --- a/unittest/obproxy/test_field_heap.cpp +++ b/unittest/obproxy/test_field_heap.cpp @@ -11,9 +11,9 @@ */ #define USING_LOG_PREFIX PROXY +#include #define private public #define protected public -#include #include "lib/oblog/ob_log.h" #include "lib/string/ob_string.h" #include "lib/number/ob_number_v2.h" diff --git a/unittest/obproxy/test_io_buffer.cpp b/unittest/obproxy/test_io_buffer.cpp index 11426d10dd255d60d3623f1b50f845a3d0a93999..15415a0ef6c9c4e435c980ce48c5de9647f34f8b 100644 --- a/unittest/obproxy/test_io_buffer.cpp +++ b/unittest/obproxy/test_io_buffer.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "ob_io_buffer.h" diff --git a/unittest/obproxy/test_ldc_location.cpp b/unittest/obproxy/test_ldc_location.cpp index 11570562b56dfdf07c7dee870aaacf2e7a80ad94..80f9bbe9ce8ebaee658b12b517c135fbd9a60a4c 100644 --- a/unittest/obproxy/test_ldc_location.cpp +++ b/unittest/obproxy/test_ldc_location.cpp @@ -11,9 +11,9 @@ */ #define USING_LOG_PREFIX PROXY +#include #define private public #define protected public -#include #include "lib/oblog/ob_log.h" #include "lib/string/ob_string.h" #include "proxy/route/ob_ldc_route.h" @@ -834,8 +834,9 @@ TEST_F(TesLDCLocation, fill_strong_read_location) ObSEArray region_names; ObSEArray servers_info; ObString proxy_primary_zone_name; + bool need_skip_leader = false; ASSERT_EQ(OB_ERR_UNEXPECTED, ObLDCLocation::fill_strong_read_location(&pl, dummy_ldc, leader_item, target_ldc, entry_need_update, is_only_readwrite_zone, - need_use_dup_replica, servers_info, region_names, proxy_primary_zone_name)); + need_use_dup_replica, need_skip_leader, servers_info, region_names, proxy_primary_zone_name)); ASSERT_EQ(OB_SUCCESS, dummy_ldc.assign(&ts, ss_info_, idc_name, true, cluster_name, OB_DEFAULT_CLUSTER_ID)); ASSERT_TRUE(dummy_ldc.is_ldc_used()); @@ -875,7 +876,7 @@ TEST_F(TesLDCLocation, fill_strong_read_location) ASSERT_TRUE(!dummy_ldc.is_empty()); ASSERT_EQ(OB_SUCCESS, ObLDCLocation::fill_strong_read_location(&pl, dummy_ldc, leader_item, - target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, servers_info, region_names, proxy_primary_zone_name)); + target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, need_skip_leader, servers_info, region_names, proxy_primary_zone_name)); ASSERT_TRUE(target_ldc.is_ldc_used()); ASSERT_TRUE(!entry_need_update); ASSERT_TRUE(!leader_item.is_valid()); @@ -895,7 +896,7 @@ TEST_F(TesLDCLocation, fill_strong_read_location) is_only_readwrite_zone = true; ASSERT_EQ(OB_SUCCESS, ObLDCLocation::fill_strong_read_location(&pl, dummy_ldc, leader_item, - target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, servers_info, region_names, proxy_primary_zone_name)); + target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, need_skip_leader, servers_info, region_names, proxy_primary_zone_name)); ASSERT_TRUE(target_ldc.is_ldc_used()); ASSERT_TRUE(entry_need_update); ASSERT_TRUE(leader_item.is_valid()); @@ -911,7 +912,7 @@ TEST_F(TesLDCLocation, fill_strong_read_location) is_only_readwrite_zone = false; ASSERT_EQ(OB_SUCCESS, ObLDCLocation::fill_strong_read_location(&pl, dummy_ldc, leader_item, - target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, servers_info, region_names, proxy_primary_zone_name)); + target_ldc, entry_need_update, is_only_readwrite_zone, need_use_dup_replica, need_skip_leader, servers_info, region_names, proxy_primary_zone_name)); ASSERT_TRUE(target_ldc.is_ldc_used()); ASSERT_TRUE(entry_need_update); ASSERT_TRUE(leader_item.is_valid()); diff --git a/unittest/obproxy/test_mysql_compress_analyzer.cpp b/unittest/obproxy/test_mysql_compress_analyzer.cpp index fe6a556218ba2256584565b7e5fe9e276128aa62..19fd85d4cd456e27a6e17f40792120c48814c4cc 100644 --- a/unittest/obproxy/test_mysql_compress_analyzer.cpp +++ b/unittest/obproxy/test_mysql_compress_analyzer.cpp @@ -69,7 +69,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_ok_packet) "is_checksum_on", is_checksum_on[k]); ObMysqlCompressAnalyzer analyzer; - ret = analyzer.init(seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer.init(seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMysqlResp resp; @@ -85,7 +85,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_ok_packet) // stream analyze ObMysqlResp resp2; ObMysqlCompressAnalyzer analyzer2; - ret = analyzer2.init(seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer2.init(seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); for (int64_t i = 0; i < compress_packet_len; ++i) { ObString resp_str(1, compressed_buf + i); @@ -181,7 +181,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_eof_packet) // 4. analyze in one buf ObMysqlCompressAnalyzer analyzer; uint8_t request_seq = 1; - ret = analyzer.init(request_seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer.init(request_seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMysqlResp resp; @@ -196,7 +196,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_eof_packet) // 5. analyze in stream one byte by one byte ObMysqlCompressAnalyzer analyzer2; - ret = analyzer2.init(request_seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer2.init(request_seq, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMIOBuffer *mio_buffer = analyzer2.get_transfer_miobuf(); ObIOBufferReader *reader = mio_buffer->alloc_reader(); @@ -259,7 +259,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_error_packet) ObMysqlCompressAnalyzer analyzer; uint8_t request_req = 0; - ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMysqlResp resp; @@ -274,7 +274,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_error_packet) // stream analyze ObMysqlCompressAnalyzer analyzer2; - ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, false, 0, 0, 0); + ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_QUERY, OCEANBASE_MYSQL_PROTOCOL_MODE, false, 0, 0, 0); ObMysqlResp resp2; ASSERT_EQ(OB_SUCCESS, ret); for (int64_t i = 0; i < compress_packet_len; ++i) { @@ -325,7 +325,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_OB_MYSQL_COM_STATISTICS) ObMysqlCompressAnalyzer analyzer; uint8_t request_req = 0; - ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, true, 0, 0, 0); + ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, OCEANBASE_MYSQL_PROTOCOL_MODE, true, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMysqlResp resp; @@ -340,7 +340,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_OB_MYSQL_COM_STATISTICS) // stream analyze ObMysqlCompressAnalyzer analyzer2; - ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, true, 0, 0, 0); + ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, OCEANBASE_MYSQL_PROTOCOL_MODE, true, 0, 0, 0); ObMysqlResp resp2; ASSERT_EQ(OB_SUCCESS, ret); for (int64_t i = 0; i < compress_packet_len; ++i) { @@ -386,7 +386,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_OB_MYSQL_COM_STATISTICS2) ObMysqlCompressAnalyzer analyzer; uint8_t request_req = 0; - ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, true, 0, 0, 0); + ret = analyzer.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, OCEANBASE_MYSQL_PROTOCOL_MODE, true, 0, 0, 0); ASSERT_EQ(OB_SUCCESS, ret); ObMysqlResp resp; @@ -401,7 +401,7 @@ TEST_F(TestMysqlCompressAnalyzer, test_comressed_OB_MYSQL_COM_STATISTICS2) // stream analyze ObMysqlCompressAnalyzer analyzer2; - ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, true, 0, 0, 0); + ret = analyzer2.init(request_req, ObMysqlCompressAnalyzer::DECOMPRESS_MODE, OB_MYSQL_COM_STATISTICS, OCEANBASE_MYSQL_PROTOCOL_MODE, true, 0, 0, 0); ObMysqlResp resp2; ASSERT_EQ(OB_SUCCESS, ret); for (int64_t i = 0; i < compress_packet_len; ++i) { diff --git a/unittest/obproxy/test_mysql_request_analyzer.cpp b/unittest/obproxy/test_mysql_request_analyzer.cpp index 8f181e686202632c9889c8b6de8d65eb042e8783..e12eeb1d79221b1d0a054aa4d9cdbdee44e9e489 100644 --- a/unittest/obproxy/test_mysql_request_analyzer.cpp +++ b/unittest/obproxy/test_mysql_request_analyzer.cpp @@ -10,11 +10,11 @@ * See the Mulan PubL v2 for more details. */ -#define private public #define USING_LOG_PREFIX PROXY #include #include #include +#define private public #include "lib/objectpool/ob_concurrency_objpool.h" #include "obproxy/proxy/mysqllib/ob_mysql_request_analyzer.h" diff --git a/unittest/obproxy/test_mysql_transact.cpp b/unittest/obproxy/test_mysql_transact.cpp index 1e856668f3001bf05e03c3c4ae127310f2d99f40..d8f0a51656bc038d941a330723da9441413edc3f 100644 --- a/unittest/obproxy/test_mysql_transact.cpp +++ b/unittest/obproxy/test_mysql_transact.cpp @@ -11,9 +11,9 @@ */ #define USING_LOG_PREFIX PROXY +#include #define private public #define protected public -#include #include "lib/oblog/ob_log.h" #include "lib/string/ob_string.h" #include "lib/utility/utility.h" diff --git a/unittest/obproxy/test_mysql_transaction_analyzer.cpp b/unittest/obproxy/test_mysql_transaction_analyzer.cpp index ea690b06259ee2546c8bc561509ee232356136e5..854371de320f3753d0550bcfbdd28a94c9cd6b49 100644 --- a/unittest/obproxy/test_mysql_transaction_analyzer.cpp +++ b/unittest/obproxy/test_mysql_transaction_analyzer.cpp @@ -412,7 +412,7 @@ TEST_F(TestMysqlTransactionAnalyzer, test_OB_MYSQL_COM_CREATE_DB) analyze_mysql_response(trans_analyzer, hex, true); }; -TEST_F(TestMysqlTransactionAnalyzer, test_COM_SHANDSHAKE) +TEST_F(TestMysqlTransactionAnalyzer, test_OB_MYSQL_COM_SHANDSHAKE) { ObMysqlTransactionAnalyzer trans_analyzer; @@ -589,7 +589,7 @@ TEST_F(TestMysqlTransactionAnalyzer, test_OB_MYSQL_COM_STATISTICS_WITH_OK) bool is_completed = false; bool is_request_completed = false; char resp_s[DEFAULT_PKT_LEN]; - // COM_STASTICS response packet(String EOF packet + OK packet) + // OB_MYSQL_COM_STASTICS response packet(String EOF packet + OK packet) const char *hex = "1a00000141637469766520746872656164732" "06e6f7420737570706f7274080000020000002" "200000000"; @@ -618,7 +618,7 @@ TEST_F(TestMysqlTransactionAnalyzer, test_OB_MYSQL_COM_STATISTICS_WITHOUT_BODY_A bool is_completed = false; bool is_request_completed = false; char resp_s[DEFAULT_PKT_LEN]; - // COM_STASTICS response packe: String EOF packet(only has packet header with out body) + OK packet + // OB_MYSQL_COM_STASTICS response packe: String EOF packet(only has packet header with out body) + OK packet const char *hex = "00000000080000020000002" "200000000"; ret = covert_hex_to_string(hex, strlen(hex), resp_s); diff --git a/unittest/obproxy/test_mysql_version.cpp b/unittest/obproxy/test_mysql_version.cpp index 0c1e55bc4679ae5ff50db19599000ece9edbf273..059e2a4be896367caf732f32d60cfd44f330b617 100644 --- a/unittest/obproxy/test_mysql_version.cpp +++ b/unittest/obproxy/test_mysql_version.cpp @@ -10,9 +10,9 @@ * See the Mulan PubL v2 for more details. */ +#include #define private public #define protected public -#include #include "common/ob_version.h" namespace oceanbase diff --git a/unittest/obproxy/test_netsystem.cpp b/unittest/obproxy/test_netsystem.cpp index 3615f66a0ceaaf69df8a9b53166c3e17636f3cd0..7592e25fb6f3efafb78cd8ca9cb5358d10ae140e 100644 --- a/unittest/obproxy/test_netsystem.cpp +++ b/unittest/obproxy/test_netsystem.cpp @@ -10,10 +10,10 @@ * See the Mulan PubL v2 for more details. */ -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "ob_mysql_stats.h" #include "stat/ob_net_stats.h" diff --git a/unittest/obproxy/test_ob_blowfish.cpp b/unittest/obproxy/test_ob_blowfish.cpp index 9a334a91c1112d897cbdbf63a401e85156dad862..300af95eb6c775a750538972832e7af1c8c10a3f 100644 --- a/unittest/obproxy/test_ob_blowfish.cpp +++ b/unittest/obproxy/test_ob_blowfish.cpp @@ -11,10 +11,10 @@ */ #define USING_LOG_PREFIX PROXY -#define private public -#define protected public #include #include +#define private public +#define protected public #include "lib/oblog/ob_log.h" #include "lib/string/ob_string.h" #include "obproxy/utils/ob_proxy_blowfish.h" diff --git a/unittest/obproxy/test_priority_event_queue.cpp b/unittest/obproxy/test_priority_event_queue.cpp index e9c393454f09d482b0bcb543d48c23f5152e7281..281504dd1fd44fc79e0716859438d5b46df6c15e 100644 --- a/unittest/obproxy/test_priority_event_queue.cpp +++ b/unittest/obproxy/test_priority_event_queue.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "ob_priority_event_queue.h" diff --git a/unittest/obproxy/test_protected_queue.cpp b/unittest/obproxy/test_protected_queue.cpp index 9bd492eb0271662c7fb1a306d0969d555b59c3d8..102a40eed498f52311715c916cb6b9537959b914 100644 --- a/unittest/obproxy/test_protected_queue.cpp +++ b/unittest/obproxy/test_protected_queue.cpp @@ -12,10 +12,10 @@ #define USING_LOG_PREFIX PROXY_EVENT -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "ob_protected_queue.h" diff --git a/unittest/obproxy/test_proxy_json_config_info.cpp b/unittest/obproxy/test_proxy_json_config_info.cpp index 3202d76a1178dc91f8b121e64d3ad79b16ea7f34..55b377bbe8711fc265307a112d8ca3536bc0af95 100644 --- a/unittest/obproxy/test_proxy_json_config_info.cpp +++ b/unittest/obproxy/test_proxy_json_config_info.cpp @@ -10,9 +10,9 @@ * See the Mulan PubL v2 for more details. */ -#define private public #define USING_LOG_PREFIX PROXY #include +#define private public #include "obproxy/proxy/route/ob_table_cache.h" #include "obproxy/obutils/ob_proxy_json_config_info.h" diff --git a/unittest/obproxy/test_proxy_session_info.cpp b/unittest/obproxy/test_proxy_session_info.cpp index 14bb360b7bf9c52d01d545f31aa1c97a006a976f..aceb7cdd1602a5a86be85d214d7362076300ee09 100644 --- a/unittest/obproxy/test_proxy_session_info.cpp +++ b/unittest/obproxy/test_proxy_session_info.cpp @@ -11,10 +11,10 @@ */ #define USING_LOG_PREFIX PROXY -#define private public -#define protected public #include +#define private public +#define protected public #include "easy_io_struct.h" #include "lib/oblog/ob_log.h" #include "lib/time/ob_time_utility.h" diff --git a/unittest/obproxy/test_proxy_session_mgr.cpp b/unittest/obproxy/test_proxy_session_mgr.cpp index 70f7d40722a0ec40e0ba80dc49863ac4b2063b14..d102f58031f0d987efea57bf9268290897afeac4 100644 --- a/unittest/obproxy/test_proxy_session_mgr.cpp +++ b/unittest/obproxy/test_proxy_session_mgr.cpp @@ -11,10 +11,10 @@ */ #define USING_LOG_PREFIX PROXY -#define private public -#define protected public #include +#define private public +#define protected public #include "easy_io_struct.h" #include "lib/oblog/ob_log.h" #include "lib/time/ob_time_utility.h" diff --git a/unittest/obproxy/test_safe_snapshot_manager.cpp b/unittest/obproxy/test_safe_snapshot_manager.cpp index d0943c3204e4c2ac06aaf495302b3644a59d0d6b..7ea19c11026268a56eccfde5d3ccedd54d22a119 100644 --- a/unittest/obproxy/test_safe_snapshot_manager.cpp +++ b/unittest/obproxy/test_safe_snapshot_manager.cpp @@ -11,9 +11,9 @@ */ #define USING_LOG_PREFIX PROXY +#include #define private public #define protected public -#include #include "obutils/ob_safe_snapshot_manager.h" #include "utils/ob_proxy_utils.h" diff --git a/unittest/obproxy/test_session_field_mgr.cpp b/unittest/obproxy/test_session_field_mgr.cpp index 87183d23b17e1e9812eeb0aeee441b249213d7c8..b939f05399f44f2e2665fb21635c10482bb234ae 100644 --- a/unittest/obproxy/test_session_field_mgr.cpp +++ b/unittest/obproxy/test_session_field_mgr.cpp @@ -11,9 +11,9 @@ */ #define USING_LOG_PREFIX PROXY +#include #define private public #define protected public -#include #include "lib/oblog/ob_log.h" #include "lib/string/ob_sql_string.h" #include "lib/string/ob_string.h" diff --git a/unittest/obproxy/test_tenant_processor.cpp b/unittest/obproxy/test_tenant_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..78b1f7db825f6a233cdc0db9133d257371c9e3a5 --- /dev/null +++ b/unittest/obproxy/test_tenant_processor.cpp @@ -0,0 +1,617 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + * + * ************************************************************* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define private public +#define USING_LOG_PREFIX PROXY +#include +#include +#include "lib/objectpool/ob_concurrency_objpool.h" +#include "obproxy/omt/ob_resource_unit_table_processor.h" +#include "obproxy/obutils/ob_config_processor.h" +#include "lib/oblog/ob_log.h" + +namespace oceanbase +{ +using namespace common; +namespace obproxy +{ +using namespace omt; +using namespace obutils; + +struct conn_map_def { + ObString key_; + int32_t value_; +}; + +class TestTenantProcessor : public ::testing::Test +{ +public: + TestTenantProcessor(): t_processor_(get_global_resource_unit_table_processor()) {} + int init_env(); + int check_map_value(conn_map_def* conn_map, uint32_t size); + + virtual void SetUp() { } + + virtual void TearDown() + { + //t_processor_.destroy(); + } + + ObResourceUnitTableProcessor& t_processor_; +}; + +static int callback(void *NotUsed, int argc, char **argv, char **azColName) +{ + (void)(NotUsed); + int i; + for(i=0; iend(); + if (size != vt_conn_map->count()) { + ret = -1; + } else { + int i = 0; + for (ObVipTenantConnCache::VTHashMap::iterator it = vt_conn_map->begin(); it != last; ++it) { + if (conn_map[i].key_ != it->full_name_ || conn_map[i].value_ != it->max_connections_) { + ret = -1; + break; + } + i++; + } + } + + return ret; +} + + +// Insert a kv in a single line +TEST_F(TestTenantProcessor, test_one_vt_conn) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + /* replace SQL statement */ + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_EQ(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(1, t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); +} + +// Insert multiple kvs in a single row +TEST_F(TestTenantProcessor, test_multi_vt_conn) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + + /* replace SQL statement */ + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}, {\"vip\": \"127.0.0.2\", \"value\": 3000}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_EQ(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); +} + +// Insert multiple kvs in multiple rows +TEST_F(TestTenantProcessor, test_multi_row_vt_conn) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + + /* replace SQL statement */ + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}, {\"vip\": \"127.0.0.2\", \"value\": 3000}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}, {\"vip\": \"127.0.0.2\", \"value\": 3000}]', 'ob_cluster_1', 'ob_tenant_1');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_EQ(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + ASSERT_EQ(4, t_processor_.get_conn_map_count()); +} + +// Insert duplicate key +TEST_F(TestTenantProcessor, test_repeat_key) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + + /* replace SQL statement */ + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}," \ + "{\"vip\": \"127.0.0.2\", \"value\": 3000}, {\"vip\": \"127.0.0.2\", \"value\": 4450}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_EQ(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); +} + +// insert a different name +TEST_F(TestTenantProcessor, test_diff_key) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + + // Insert connection related information + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_max_connections', '[{\"vip\": \"127.0.0.1\", \"value\": 2000}," \ + "{\"vip\": \"127.0.0.2\", \"value\": 4450}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + // Insert cpu related information + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_cpu', '[{\"vip\": \"127.0.0.1\", \"value\": 50}," \ + "{\"vip\": \"127.0.0.2\", \"value\": 50}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_NE(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + ASSERT_EQ(0, t_processor_.get_conn_map_count()); +} + +TEST_F(TestTenantProcessor, test_no_conn_key) +{ + int rc; + char *sql; + char *zErrMsg = 0; + + ASSERT_EQ(0, init_env()); + + // Insert cpu related information + sql = (char*)"replace into resource_unit(name, value, cluster_name, tenant_name)" \ + "values('resource_cpu', '[{\"vip\": \"127.0.0.1\", \"value\": 50}," \ + "{\"vip\": \"127.0.0.2\", \"value\": 50}]', 'ob_cluster', 'ob_tenant');"; + + /* Execute SQL statement */ + rc = sqlite3_exec(db, sql, callback, 0, &zErrMsg); + if( rc != SQLITE_OK ){ + fprintf(stderr, "SQL error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(stdout, "Records created successfully\n"); + } + + ASSERT_NE(0, get_global_config_processor().init()); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(0, t_processor_.get_conn_map_count()); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); +} + +TEST_F(TestTenantProcessor, test_execute) +{ + ASSERT_EQ(0, init_env()); + + SqlFieldResult sql_result; + sql_result.field_num_ = 4; + SqlField sql_field; + sql_field.column_name_.set("cluster"); + sql_field.column_value_.set_value("ob_cluster"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("tenant"); + sql_field.column_value_.set_value("ob_tenant"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("name"); + sql_field.column_value_.set_value("resource_max_connections"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("value"); + sql_field.column_value_.set_value("[{\"vip\": \"127.0.0.1\", \"value\": 5000}]"); + sql_result.fields_.push_back(sql_field); + + ObCloudFnParams cloud_params; + cloud_params.config_type_ = OBPROXY_CONFIG_CLOUD; + cloud_params.stmt_type_ = OBPROXY_T_REPLACE; + cloud_params.fields_ = &sql_result; + cloud_params.cluster_name_ = "ob_cluster"; + cloud_params.tenant_name_ = "ob_tenant"; + cloud_params.table_name_ = "resource_unit"; + + ASSERT_EQ(0, get_global_config_processor().init()); + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(1, t_processor_.get_conn_map_count()); + conn_map_def conn_map[1] = { + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map, 1)); + + // vt_conn_map_replica is empty + ObVipTenantConnCache::VTHashMap& vt_conn_map_replica = t_processor_.get_conn_map_replica(); + ObVipTenantConnCache::dump_conn_map(vt_conn_map_replica); +} + +TEST_F(TestTenantProcessor, test_rollback) +{ + ASSERT_EQ(0, init_env()); + + SqlFieldResult sql_result; + sql_result.field_num_ = 4; + SqlField sql_field; + sql_field.column_name_.set("cluster"); + sql_field.column_value_.set_value("ob_cluster"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("tenant"); + sql_field.column_value_.set_value("ob_tenant"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("name"); + sql_field.column_value_.set_value("resource_max_connections"); + sql_result.fields_.push_back(sql_field); + + sql_field.column_name_.set("value"); + sql_field.column_value_.set_value("[{\"vip\": \"127.0.0.1\", \"value\": 5000}]"); + sql_result.fields_.push_back(sql_field); + + ObCloudFnParams cloud_params; + cloud_params.config_type_ = OBPROXY_CONFIG_CLOUD; + cloud_params.stmt_type_ = OBPROXY_T_REPLACE; + cloud_params.fields_ = &sql_result; + cloud_params.cluster_name_ = "ob_cluster"; + cloud_params.tenant_name_ = "ob_tenant"; + cloud_params.table_name_ = "resource_unit"; + + ASSERT_EQ(0, get_global_config_processor().init()); + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(1, t_processor_.get_conn_map_count()); + conn_map_def conn_map[1] = { + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map, 1)); + + // vt_conn_map is empty after rollback + ObResourceUnitTableProcessor::commit(false); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(0, t_processor_.get_conn_map_count()); +} + +TEST_F(TestTenantProcessor, test_execute_update_value) +{ + ASSERT_EQ(0, init_env()); + + SqlFieldResult sql_result; + sql_result.field_num_ = 4; + SqlField sql_field[4]; + sql_field[0].column_name_.set("cluster"); + sql_field[0].column_value_.set_value("ob_cluster"); + sql_result.fields_.push_back(sql_field[0]); + + sql_field[1].column_name_.set("tenant"); + sql_field[1].column_value_.set_value("ob_tenant"); + sql_result.fields_.push_back(sql_field[1]); + + sql_field[2].column_name_.set("name"); + sql_field[2].column_value_.set_value("resource_max_connections"); + sql_result.fields_.push_back(sql_field[2]); + + sql_field[3].column_name_.set("value"); + sql_field[3].column_value_.set_value("[{\"vip\": \"127.0.0.1\", \"value\": 5000}, {\"vip\": \"127.0.0.2\", \"value\": 6000}]"); + sql_result.fields_.push_back(sql_field[3]); + + ObCloudFnParams cloud_params; + cloud_params.config_type_ = OBPROXY_CONFIG_CLOUD; + cloud_params.stmt_type_ = OBPROXY_T_REPLACE; + cloud_params.fields_ = &sql_result; + cloud_params.cluster_name_ = "ob_cluster"; + cloud_params.tenant_name_ = "ob_tenant"; + cloud_params.table_name_ = "resource_unit"; + + ASSERT_EQ(0, get_global_config_processor().init()); + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); + conn_map_def conn_map[2] = { + {"ob_tenant#ob_cluster|127.0.0.2", 6000}, + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map, 2)); + ObResourceUnitTableProcessor::commit(true); + LOG_DEBUG("success1"); + + sql_result.reset(); + sql_result.field_num_ = 4; + sql_result.fields_.push_back(sql_field[0]); + sql_result.fields_.push_back(sql_field[1]); + sql_result.fields_.push_back(sql_field[2]); + sql_field[3].column_name_.set("value"); + sql_field[3].column_value_.set_value("[{\"vip\": \"127.0.0.3\", \"value\": 4000}, {\"vip\": \"127.0.0.2\", \"value\": 7777}]"); + sql_result.fields_.push_back(sql_field[3]); + cloud_params.fields_ = &sql_result; + + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(3, t_processor_.get_conn_map_count()); + conn_map_def conn_map_1[3] = { + {"ob_tenant#ob_cluster|127.0.0.3", 4000}, + {"ob_tenant#ob_cluster|127.0.0.2", 7777}, + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map_1, 3)); + LOG_DEBUG("success2"); + + ObResourceUnitTableProcessor::commit(false); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + conn_map_def conn_map_2[2] = { + {"ob_tenant#ob_cluster|127.0.0.1", 5000}, + {"ob_tenant#ob_cluster|127.0.0.2", 6000}, + }; + ASSERT_EQ(0, check_map_value(conn_map_2, 2)); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); +} + +TEST_F(TestTenantProcessor, test_delete_tenant) +{ + ASSERT_EQ(0, init_env()); + + SqlFieldResult sql_result; + sql_result.field_num_ = 4; + SqlField sql_field[4]; + sql_field[0].column_name_.set("cluster"); + sql_field[0].column_value_.set_value("ob_cluster"); + sql_result.fields_.push_back(sql_field[0]); + + sql_field[1].column_name_.set("tenant"); + sql_field[1].column_value_.set_value("ob_tenant"); + sql_result.fields_.push_back(sql_field[1]); + + sql_field[2].column_name_.set("name"); + sql_field[2].column_value_.set_value("resource_max_connections"); + sql_result.fields_.push_back(sql_field[2]); + + sql_field[3].column_name_.set("value"); + sql_field[3].column_value_.set_value("[{\"vip\": \"127.0.0.1\", \"value\": 5000}, {\"vip\": \"127.0.0.2\", \"value\": 6000}]"); + sql_result.fields_.push_back(sql_field[3]); + + ObCloudFnParams cloud_params; + cloud_params.config_type_ = OBPROXY_CONFIG_CLOUD; + cloud_params.stmt_type_ = OBPROXY_T_REPLACE; + cloud_params.fields_ = &sql_result; + cloud_params.cluster_name_ = "ob_cluster"; + cloud_params.tenant_name_ = "ob_tenant"; + cloud_params.table_name_ = "resource_unit"; + + ASSERT_EQ(0, get_global_config_processor().init()); + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); + conn_map_def conn_map[2] = { + {"ob_tenant#ob_cluster|127.0.0.2", 6000}, + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map, 2)); + LOG_DEBUG("success1"); + + sql_result.reset(); + sql_result.field_num_ = 4; + sql_result.fields_.push_back(sql_field[0]); + sql_field[1].column_name_.set("tenant"); + sql_field[1].column_value_.set_value("ob_tenant_1"); + sql_result.fields_.push_back(sql_field[1]); + sql_result.fields_.push_back(sql_field[2]); + sql_field[3].column_name_.set("value"); + sql_field[3].column_value_.set_value("[{\"vip\": \"127.0.0.3\", \"value\": 4000}, {\"vip\": \"127.0.0.2\", \"value\": 7777}]"); + sql_result.fields_.push_back(sql_field[3]); + cloud_params.fields_ = &sql_result; + cloud_params.tenant_name_ = "ob_tenant_1"; + + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + ObVipTenantConnCache::VTHashMap* vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(4, t_processor_.get_conn_map_count()); + conn_map_def conn_map_1[4] = { + {"ob_tenant_1#ob_cluster|127.0.0.2", 7777}, + {"ob_tenant_1#ob_cluster|127.0.0.3", 4000}, + {"ob_tenant#ob_cluster|127.0.0.2", 6000}, + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map_1, 4)); + LOG_DEBUG("success2"); + + // delete configuration + cloud_params.stmt_type_ = OBPROXY_T_DELETE; + cloud_params.tenant_name_ = "ob_tenant_1"; + ASSERT_EQ(0, ObResourceUnitTableProcessor::execute(&cloud_params)); + vt_conn_map = t_processor_.get_conn_map(); + ObVipTenantConnCache::dump_conn_map(*vt_conn_map); + LOG_DEBUG("conn map", "count", t_processor_.get_conn_map_count()); + ASSERT_EQ(2, t_processor_.get_conn_map_count()); + conn_map_def conn_map_2[2] = { + {"ob_tenant#ob_cluster|127.0.0.2", 6000}, + {"ob_tenant#ob_cluster|127.0.0.1", 5000} + }; + ASSERT_EQ(0, check_map_value(conn_map_2, 2)); + LOG_DEBUG("success3"); +} + +} +} + +int main(int argc, char **argv) +{ + oceanbase::common::ObLogger::get_logger().set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/obproxy/test_unix_net.cpp b/unittest/obproxy/test_unix_net.cpp index 0106fe95776ce2d5599306d76932f54260ad620a..1d566b023d7bdc025db11b3b9acda1ba13578554 100644 --- a/unittest/obproxy/test_unix_net.cpp +++ b/unittest/obproxy/test_unix_net.cpp @@ -10,10 +10,10 @@ * See the Mulan PubL v2 for more details. */ -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" namespace oceanbase diff --git a/unittest/obproxy/test_unix_net_processor.cpp b/unittest/obproxy/test_unix_net_processor.cpp index 55463aa96caf79258d475995bfa4529d21c9f4e7..04dba50d186ab8ddd163e2c5920342d5866604b1 100644 --- a/unittest/obproxy/test_unix_net_processor.cpp +++ b/unittest/obproxy/test_unix_net_processor.cpp @@ -10,10 +10,10 @@ * See the Mulan PubL v2 for more details. */ -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" #include "obproxy/stat/ob_mysql_stats.h" #include "ob_event_io.h" diff --git a/unittest/obproxy/test_unix_net_vconnection.cpp b/unittest/obproxy/test_unix_net_vconnection.cpp index 5cb8b6a94dbedaf2b80c71923aebce138784a676..9e29d6f27c6a86820ed0a1d22cbf3edfc186c0f2 100644 --- a/unittest/obproxy/test_unix_net_vconnection.cpp +++ b/unittest/obproxy/test_unix_net_vconnection.cpp @@ -10,10 +10,10 @@ * See the Mulan PubL v2 for more details. */ -#define private public -#define protected public #include #include +#define private public +#define protected public #include "test_eventsystem_api.h" namespace oceanbase diff --git a/unittest/obproxy/test_vip_tenant_cache.cpp b/unittest/obproxy/test_vip_tenant_cache.cpp index a0237148fdabc180bb777e27a6ca1cc24708a3c9..6153bdf4f880542dc31b4e49cd3868db3ab43b8d 100644 --- a/unittest/obproxy/test_vip_tenant_cache.cpp +++ b/unittest/obproxy/test_vip_tenant_cache.cpp @@ -10,9 +10,9 @@ * See the Mulan PubL v2 for more details. */ -#define private public #define USING_LOG_PREFIX PROXY #include +#define private public #include "lib/objectpool/ob_concurrency_objpool.h" #include "obproxy/obutils/ob_vip_tenant_cache.h" diff --git a/unittest/obproxy/test_white_list_processor.cpp b/unittest/obproxy/test_white_list_processor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aec7a728153798e47342fbf7816d0a66ee90bc4f --- /dev/null +++ b/unittest/obproxy/test_white_list_processor.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase Database Proxy(ODP) is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX PROXY +#include +#define private public +#include "obproxy/omt/ob_white_list_table_processor.h" + +using namespace oceanbase; +using namespace oceanbase::common; +using namespace oceanbase::obproxy::omt; + +class TestWhiteListProcessor : public ::testing::Test +{ +public: + void init_white_list_processor(); + + ObWhiteListTableProcessor white_list_processor_; +}; + +void TestWhiteListProcessor::init_white_list_processor() +{ + ASSERT_EQ(OB_SUCCESS, white_list_processor_.addr_hash_map_array_[0].create(32, ObModIds::OB_HASH_BUCKET)); + ASSERT_EQ(OB_SUCCESS, white_list_processor_.addr_hash_map_array_[1].create(32, ObModIds::OB_HASH_BUCKET)); + ObString cluster_name = "cluster1"; + ObString tenant_name = "tenant1"; + ObString ip_list = "100.88.147.128/26"; + ASSERT_EQ(OB_SUCCESS, white_list_processor_.set_ip_list(cluster_name, tenant_name, ip_list)); + white_list_processor_.inc_index(); + ObString cluster_name2 = "cluster2"; + ObString tenant_name2 = "tenant2"; + ObString ip_list2 = "127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,127.0.0.1,168.1.1.1"; + ASSERT_EQ(OB_SUCCESS, white_list_processor_.set_ip_list(cluster_name2, tenant_name2, ip_list2)); + white_list_processor_.inc_index(); +} + +TEST_F(TestWhiteListProcessor, test_ip_net) +{ + init_white_list_processor(); + ObString cluster_name = "cluster1"; + ObString tenant_name = "tenant1"; + ObString user_name = "user1"; + char ip1[32] = "127.0.0.2"; + char ip2[32] = "127.0.0.3"; + char ip3[32] = "127.0.0.4"; + char ip4[32] = "127.0.0.5"; + ASSERT_EQ(true, white_list_processor_.can_ip_pass(cluster_name, tenant_name, user_name, ip1)); + ASSERT_EQ(true, white_list_processor_.can_ip_pass(cluster_name, tenant_name, user_name, ip2)); + ASSERT_EQ(false, white_list_processor_.can_ip_pass(cluster_name, tenant_name, user_name, ip3)); + ASSERT_EQ(false, white_list_processor_.can_ip_pass(cluster_name, tenant_name, user_name, ip4)); + ObString cluster_name2 = "cluster2"; + ObString tenant_name2 = "tenant2"; + ObString user_name2 = "user2"; + char ip5[32] = "127.0.0.1"; + char ip6[32] = "127.0.0.2"; + char ip7[32] = "127.0.0.3"; + ASSERT_EQ(true, white_list_processor_.can_ip_pass(cluster_name2, tenant_name2, user_name2, ip5)); + ASSERT_EQ(false, white_list_processor_.can_ip_pass(cluster_name2, tenant_name2, user_name2, ip6)); + ASSERT_EQ(true, white_list_processor_.can_ip_pass(cluster_name2, tenant_name2, user_name2, ip7)); +} + +int main(int argc, char **argv) +{ + oceanbase::common::ObLogger::get_logger().set_log_level("DEBUG"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}