merge develop into master 20240312

.github/workflows/buildbase/action.yml

@@ -17,8 +17,10 @@ runs:
         bash build.sh init
         echo "$GITHUB_WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin" >> $GITHUB_PATH
 
+    # on centos7, we cannot use the latest `node`.
+    # ccache-action@v1.2 uses `node20` which cannot works on centos7 but node16 works fun
     - name: Setup ccache
-      uses: hendrikmuhs/ccache-action@v1.2
+      uses: hendrikmuhs/ccache-action@v1.2.11
       with:
         max-size: 800M
         save: ${{inputs.save_cache}}

.github/workflows/compile.yml

@@ -7,40 +7,32 @@ on:
     branches: [ master, develop ]
     paths-ignore:
       - 'docs/**'
-      - '.github/**'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/pull_request_template.md'
       - 'README.md'
       - 'README_CN.md'
       - 'CONTRIBUTING.md'
 
 jobs:
-  centos-build:
-    runs-on: ubuntu-20.04
-    container: centos:7
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install centos environment
-      shell: bash
-      run: yum install -y git wget rpm* cpio make glibc-devel glibc-headers binutils m4
-
-    - name: Cache deps
-      id: cache-deps
-      uses: actions/cache@v3
-      env:
-        cache-name: cache-deps
-      with:
-        key: ${{ runner.os }}-build-${{ env.cache-name }}-el7.x86_64-${{ hashFiles('deps/init/oceanbase.el7.x86_64.deps') }}
-        path: deps/3rd
-
-    - name: Build
-      uses: ./.github/workflows/buildbase
-      with:
-        save_cache: ${{github.event_name == 'push'}}
-        os: 'centos7'
-
   ubuntu-build:
     runs-on: ubuntu-22.04
     steps:
+    - name: Free Disk Space
+      uses: insightsengineering/disk-space-reclaimer@v1
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tools-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        swap-storage: true
+        docker-images: true
+
     - uses: actions/checkout@v3
 
     - name: Install ubuntu environment

.github/workflows/farm.yml

@@ -5,7 +5,8 @@ on:
     branches: [ master,develop ]
     paths-ignore:
       - 'docs/**'
-      - '.github/**'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/pull_request_template.md'
       - 'README.md'
       - 'README_CN.md'
       - 'CONTRIBUTING.md'

.github/workflows/mkbook.yml

+# Sample workflow for building and deploying a mdBook site to GitHub Pages
+#
+# To get started with mdBook see: https://rust-lang.github.io/mdBook/index.html
+#
+name: Deploy mdBook site to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["develop"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    env:
+      MDBOOK_VERSION: 0.4.36
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare
+        run: |
+          mkdir mdbook
+          mv docs mdbook/src
+
+      - name: Install mdBook
+        run: |
+          curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf -y | sh
+          rustup update
+          cargo install --version ${MDBOOK_VERSION} mdbook
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v4
+      - name: Build with mdBook
+        run: mdbook build
+        working-directory: mdbook
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./mdbook/book
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
\ No newline at end of file

README.md

@@ -32,6 +32,9 @@
     <a href="https://join.slack.com/t/oceanbase/shared_invite/zt-1e25oz3ol-lJ6YNqPHaKwY_mhhioyEuw">
         <img alt="Join Slack" src="https://img.shields.io/badge/slack-Join%20Oceanbase-brightgreen?logo=slack" />
     </a>
+    <a href="https://stackoverflow.com/questions/tagged/oceanbase">
+        <img alt="Ask on Stack Overflow" src="https://img.shields.io/badge/Stack-Ask%20on%20Stack%20Overflow-brightgreen?logo=stackoverflow" />
+    </a>
 </p>
 
 English | [中文版](README_CN.md)
@@ -135,6 +138,7 @@ OceanBase Database is licensed under the Mulan Public License, Version 2. See th
 Join the OceanBase community via:
 
 * [Slack Workspace](https://join.slack.com/t/oceanbase/shared_invite/zt-1e25oz3ol-lJ6YNqPHaKwY_mhhioyEuw)
+* [Ask on Stack Overflow](https://stackoverflow.com/questions/tagged/oceanbase)
 * [Chinese User Forum](https://ask.oceanbase.com/)
 * DingTalk Group: 33254054 ([QR code](images/dingtalk.svg))
 * WeChat Group (Add the assistant with WeChat ID: OBCE666)

README_CN.md

@@ -31,6 +31,9 @@
     <a href="https://join.slack.com/t/oceanbase/shared_invite/zt-1e25oz3ol-lJ6YNqPHaKwY_mhhioyEuw">
         <img alt="Join Slack" src="https://img.shields.io/badge/slack-Join%20Oceanbase-brightgreen?logo=slack" />
     </a>
+    <a href="https://stackoverflow.com/questions/tagged/oceanbase">
+        <img alt="Ask on Stack Overflow" src="https://img.shields.io/badge/Stack-Ask%20on%20Stack%20Overflow-brightgreen?logo=stackoverflow" />
+    </a>
 </p>
 
 [English](README.md) | 中文版
@@ -135,5 +138,6 @@ OceanBase 数据库根据 Mulan 公共许可证版本 2 获得许可。有关详
 
 * [中文论坛](https://ask.oceanbase.com/)
 * [Slack Workspace](https://join.slack.com/t/oceanbase/shared_invite/zt-1e25oz3ol-lJ6YNqPHaKwY_mhhioyEuw)
+* [Ask on Stack Overflow](https://stackoverflow.com/questions/tagged/oceanbase)
 * 钉钉群: 33254054 ([二维码](images/dingtalk.svg))
 * 微信群 (添加微信小助手: OBCE666)

docs/README.md

@@ -12,11 +12,19 @@ At present, the guide is composed of the following parts:
 1. **Get started**: Setting up the development environment, build and connect to the OceanBase server, the subsections are based on an imagined newbie user journey.
     1. [Install toolchain](toolchain.md)
     2. [Get the code, build and run](build-and-run.md)
-    3. Set up an IDE
-    4. [Write and run unit tests](unittest.md)
-    5. [Debug](debug.md)
-    6. Commit code and submit a pull request
-2. **Contribute to OceanBase**: helps you quickly get involved in the OceanBase community, which illustrates what contributions you can make and how to quickly make one.
+    3. [Set up an IDE](ide-settings.md)
+    4. [Coding Convensions](coding-convension.md)
+    5. [Write and run unit tests](unittest.md)
+    6. [Running MySQL test](mysqltest.md)
+    7. [Debug](debug.md)
+    8. Commit code and submit a pull request
+
+    More information before you start a big feature developing, you should read content below and it can help you understand oceanbase better.
+    1. [Logging System](logging.md)
+    2. [Memory Management](memory.md)
+    3. [Containers](container.md)
+    4. [Coding Standard](coding_standard.md)
+3. **Contribute to OceanBase**: helps you quickly get involved in the OceanBase community, which illustrates what contributions you can make and how to quickly make one.
 
 ## User documents
 

docs/SUMMARY.md

+# Summary
+
+[OceanBase Development Guide](README.md)
+[Install toolchain](toolchain.md)
+[Get the code, build and run](build-and-run.md)
+[Set up an IDE](ide-settings.md)
+[Coding Convensions](coding-convension.md)
+[Write and run unit tests](unittest.md)
+[Running MySQL test](mysqltest.md)
+[Debug](debug.md)
+[Logging System](logging.md)
+[Memory Management](memory.md)
+[Containers](container.md)
+[Coding Standard](coding_standard.md)

docs/coding-convension.md

+OceanBase is a giant project that has been developed for more than ten years and contains millions of lines of C++ code. It already has many unique programming habits. Here are some OceanBase programming habits to help people who come into contact with the OceanBase source code for the first time have an easier time accepting and understanding. For more detailed information, please refer to ["OceanBase C++ Coding Standard"](./coding_standard.md).
+
+# Naming Convention
+
+- File naming
+
+Code file names in OceanBase all start with `ob_`. But there are some old exception files.
+
+- Class naming
+
+Classes all start with `Ob` and use camelCase/Pascal form, and there are also some exceptions for old classes.
+
+- Function names, variables, etc.
+
+Both function names and variables use lowercase naming separated by `_`. Member variables also have `_` added as a suffix.
+
+# Coding Style
+
+OceanBase uses some relatively simple coding styles to try to make the code readable and clear, such as adding necessary spaces for operator brackets, not too long codes, not too long functions, adding necessary comments, reasonable naming, etc. Since the coding style has many details, new developers can just refer to the coding style in the current code to write code. This is also a suggestion for participating in other projects for the first time. We should try to keep it consistent with the original style.
+
+There is no need to worry about the styles that you are not sure about. You can discuss it with us, or after submitting the code, someone will give suggestions or code together.
+
+# Functional Coding Habits
+
+## Prohibitting STL Containers
+
+Since OceanBase supports multi-tenants resource isolation, in order to facilitate memory control, OceanBase prohibits the use of STL, boost and other containers. At the same time, OceanBase provides its own containers, such as `ObSEArray`, etc. For more information about OceanBase containers, please refer to [OceanBase Container Introduction] (./container.md).
+
+## Be Caution with the New C++ Standard
+
+OceanBase does not encourage the use of some syntax of the new C++ standard, such as auto, smart pointers, move semantics, range-based loops, lambda, etc. OceanBase believes that these will bring many negative effects, such as
+
+- Improper use of `auto` can cause serious performance problems, but it only brings syntactic convenience;
+- Smart pointers cannot solve the problem of object memory usage, and improper use can also cause performance problems;
+- The use of move is extremely complex, and it will lead to deeply hidden BUGs without ensuring that everyone understands it correctly.
+
+Of course, OceanBase does not exclude all new standards, such as encouraging the use of override, final, constexpr, etc. If you are not sure whether a certain syntax can be used, you can search and confirm in ["OceanBase C++ Coding Standard"](./coding_standard.md).
+
+## Single Entrance and Single Exit
+
+It is mandatory for all functions to return at the end, and it is prohibited to call global jump instructions such as return, goto, and exit midway. This is also the most confusing part for everyone who comes into contact with OceanBase code for the first time.
+
+In order to achieve this requirement, there will be a lot of `if/else if` in the code, and there are many less intuitive conditional judgments such as `OB_SUCC(ret)` in the `for` loop. At the same time, in order to reduce nesting, the macro `FALSE_IT` will be used to execute certain statements. for example
+
+```cpp
+int ObMPStmtReset::process()
+{
+  int ret = OB_SUCCESS;
+  ...
+  if (OB_ISNULL(req_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid packet", K(ret), KP(req_));
+  } else if (OB_INVALID_STMT_ID == stmt_id_) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("stmt_id is invalid", K(ret));
+  } else if (OB_FAIL(get_session(session))) {
+    LOG_WARN("get session failed");
+  } else if (OB_ISNULL(session)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("session is NULL or invalid", K(ret), K(session));
+  } else if (OB_FAIL(process_kill_client_session(*session))) {
+    LOG_WARN("client session has been killed", K(ret));
+  } else if (FALSE_IT(session->set_txn_free_route(pkt.txn_free_route()))) {
+  } else if (OB_FAIL(process_extra_info(*session, pkt, need_response_error))) {
+    LOG_WARN("fail get process extra info", K(ret));
+  } else if (FALSE_IT(session->post_sync_session_info())) {
+  } else if (FALSE_IT(need_disconnect = false)) {
+  } else if (OB_FAIL(update_transmission_checksum_flag(*session))) {
+    LOG_WARN("update transmisson checksum flag failed", K(ret));
+  } else {
+    // ...
+  }
+  return ret;
+}
+```
+
+A lot of `if/else if` are used in the code, and the FALSE_IF macro is used to minimize the nesting of ifs.
+
+It is worth mentioning that similar functions will write `int ret = OB_SUCCESS;` at the beginning of the function, using ret as the function return value, and many macros will also default to the existence of ret.
+
+## Function Returns Error Code
+
+For most functions, the function is required to have an int return value, and the return value can be explained using the error code `ob_errno.h`.
+Most of the functions mentioned here include some functions for obtaining values, such as the `at` function of `ObSEArray`
+
+```cpp
+int at(int64_t idx, T &obj);
+```
+
+**Which functions do not need to return int values?**
+
+Relatively simple functions that return class attributes, such as ObSEArray's function:
+
+```cpp
+int64_t get_capacity();
+```
+
+The value will be returned directly without the int error code.
+Or similar simple judgment functions do not need to return int error codes.
+
+## Need to Determine the Validity of All Return Values and Parameters
+
+OceanBase requires that as long as the function has a return value, the return value must be tested, and "check if possible." Function parameters, especially pointers, must be checked for validity before use.
+
+For example:
+
+```cpp
+int ObDDLServerClient::abort_redef_table(const obrpc::ObAbortRedefTableArg &arg, sql::ObSQLSessionInfo *session)
+{
+  int ret = OB_SUCCESS;
+  ...
+  obrpc::ObCommonRpcProxy *common_rpc_proxy = GCTX.rs_rpc_proxy_;
+  if (OB_UNLIKELY(!arg.is_valid())) { // Check the validity of the parameters passed in
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arg", K(ret), K(arg));
+  } else if (OB_ISNULL(common_rpc_proxy)) { // Before using pointers, check it first
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("common rpc proxy is null", K(ret));
+  } else {
+  ...
+  }
+  return ret;
+}
+```
+
+## Memory Management
+
+Memory management is a very troublesome issue in C/C++ programs. OceanBase has done a lot of work for memory management, including efficient memory allocation, memory problem detection, tenant memory isolation, etc. OceanBase provides a set of memory management mechanisms for this purpose, and also prohibits the direct use of C/C++ native memory allocation interfaces in programs, such as malloc, new, etc.
+
+The simplest, OceanBase provides the `ob_malloc/ob_free` interface to allocate and release memory:
+
+```cpp
+void *ptr = ob_malloc(100, ObModIds::OB_COMMON_ARRAY);
+
+// do something
+
+if (NULL != ptr) {
+  // release resource
+  ob_free(ptr, ObModIds::OB_COMMON_ARRAY);
+  ptr = NULL;  // set the pointer to null after free
+}
+```
+
+OceanBase requires that the pointer must be assigned to null immediately after the memory is released.
+For more information about memory management, please refer to [OceanBase Memory Management](./memory.md).
+
+# Some Conventional Interfaces
+
+## init/destroy
+
+OceanBase requires that only some very lightweight data initialization work can be implemented in the constructor, such as variables initialized to 0, pointers initialized to nullptr, etc. Because in the constructor, it is not easy to handle some complex exception scenarios, and the return value cannot be given. Most classes in OceanBase have an init function, which is usually executed after the constructor and has an int error code as the return value. Do some more complex initialization work here. Correspondingly, the destroy function is usually provided to do resource destruction.
+
+## reuse/reset
+
+Memory caching is a very effective way of improving performance. Many classes in OceanBase will have reuse/reset interfaces to facilitate the subsequent reuse of an object. Reuse usually represents lightweight cleanup work, while reset will do more resource cleanup work. But you need to look at the specific implementation class and cannot generalize.
+
+## Operator Overloading
+
+C++ provides operator overloading functions that are very convenient for writing programs, but these overloadings often bring a lot of burden, making the code difficult to read and the functions misused. For example, operator overloading may lead to implicit type conversion without the programmer's knowledge, or a seemingly simple operation may have a relatively high overhead.
+
+In addition, try to avoid using `operator=` and try to copy objects using `deep_copy` and `shallow_copy`.
+
+# Commonly Used Macros
+
+**OB_SUCC**
+
+Determine whether a statement returns successfully, equivalent to `OB_SUCCESS == (ret = func())`
+
+```cpp
+ret = OB_SUCCESS;
+if (OB_SUCC(func())) {
+  // do something
+}
+```
+
+**OB_FAIL**
+
+Similar to `OB_SUCC`, it just determines whether a certain statement fails to execute:
+
+```cpp
+ret = OB_SUCCESS;
+if (OB_FAIL(func())) {
+  // do something
+}
+```
+
+**OB_ISNULL**
+
+Determine whether the pointer is null, equivalent to `nullptr == ptr`,
+
+```cpp
+if (OB_ISNULL(ptr)) {
+  // do something
+}
+```
+
+**OB_NOT_NULL**
+
+Determine whether the pointer is non-null, equivalent to `nullptr != ptr`,
+
+```cpp
+if (OB_NOT_NULL(ptr)) {
+  // do something
+}
+```
+
+**K**
+
+Usually used in output logs, the usage K(obj), where obj can be a common type variable or a class object (must implement `to_string`), will be expanded into `"obj", obj`, and will eventually be output in the log "obj=123". for example:
+
+```cpp
+LOG_WARN("fail to exec func, ", K(ret));
+```
+
+**DISALLOW_COPY_AND_ASSIGN**
+
+Used in a class declaration to indicate that operations such as copy assignment are prohibited.
+
+```cpp
+class LogReconfirm
+{
+  ...
+private:
+  DISALLOW_COPY_AND_ASSIGN(LogReconfirm);
+};
+```

src/logservice/libobcdc/src/ob_log_config.h

@@ -111,7 +111,7 @@ public:
   DEF_INT(storager_queue_length, OB_CLUSTER_PARAMETER, "0", "[0,]", "storager queue length");
   DEF_INT(reader_thread_num, OB_CLUSTER_PARAMETER, "10", "[1,]", "reader thread number");
   DEF_INT(reader_queue_length, OB_CLUSTER_PARAMETER, "0", "[0,]", "reader queue length");
-  DEF_INT(br_queue_length, OB_CLUSTER_PARAMETER, "0", "[0, ]", "user_binlog_record queue length");
+  DEF_INT(br_queue_length, OB_CLUSTER_PARAMETER, "0", "[0,]", "user_binlog_record queue length");
   DEF_INT(cached_schema_version_count, OB_CLUSTER_PARAMETER, "32", "[1,]", "cached schema version count");
   DEF_INT(history_schema_version_count, OB_CLUSTER_PARAMETER, "16", "[1,]", "history schema version count");
   DEF_INT(resource_collector_thread_num, OB_CLUSTER_PARAMETER, "11", "[1,]", "resource collector thread number");

src/sql/code_generator/ob_static_engine_cg.cpp

@@ -8157,11 +8157,6 @@ int ObStaticEngineCG::get_phy_op_type(ObLogicalOperator &log_op,
           type = PHY_PX_DIST_TRANSMIT;
         } else if (op.get_plan()->get_optimizer_context().is_online_ddl() && ObPQDistributeMethod::PARTITION_RANGE == op.get_dist_method()) {
           type = PHY_PX_REPART_TRANSMIT;
-        } else if (OB_REPARTITION_NO_REPARTITION != op.get_repartition_type()
-                && !op.is_slave_mapping()) {
-          type = PHY_PX_REPART_TRANSMIT;
-        } else if (ObPQDistributeMethod::LOCAL != op.get_dist_method()) {
-          type = PHY_PX_DIST_TRANSMIT;
         } else {
           // NOTE: 优化器需要和执行器保持一致，既没有分区、又没有HASH、或其它重分区方式时，就使用All To One
           type = PHY_PX_REDUCE_TRANSMIT;