init push

.dockerignore

+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore all files which are not go type
+!**/*.go
+!**/*.mod
+!**/*.sum

.gitignore

+# Mac OS X
+.DS_Store
+
+# IDE
+.idea
+*.out
+*.swp
+*.swo
+*~
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin
+
+# lib & package
+.vendor/
+.glide/
+
+# Test binary
+*.test
+cover.out
+testbin/*
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+# !vendor/**/zz_generated.*

Dockerfile

+# Build the manager binary
+FROM golang:1.16 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY . .
+
+# Build
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOPROXY=https://mirrors.aliyun.com/goproxy/ go build -a -o manager main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM openanolis/anolisos:8.4-x86_64
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]

LICENSE

+                             Mulan Permissive Software License，Version 2
+
+Mulan Permissive Software License，Version 2 (Mulan PSL v2)
+
+January 2020 http://license.coscl.org.cn/MulanPSL2
+
+Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions:
+
+0. Definition
+
+Software means the program and related documents which are licensed under this License and comprise all Contribution(s).
+
+Contribution means the copyrightable work licensed by a particular Contributor under this License.
+
+Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License.
+
+Legal Entity means the entity making a Contribution and all its Affiliates.
+
+Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity.
+
+1. Grant of Copyright License
+
+Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not.
+
+2. Grant of Patent License
+
+Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken.
+
+3. No Trademark License
+
+No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in section 4.
+
+4. Distribution Restriction
+
+You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software.
+
+5. Disclaimer of Warranty and Limitation of Liability
+
+THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+6. Language
+
+THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL.
+
+END OF THE TERMS AND CONDITIONS

LICENSE.Apache

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE.MIT

+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

+VERSION := $(shell git rev-parse --short HEAD)
+# Image URL to use all building/pushing image targets
+IMG ?= oceanbase/obce-operator:v0.0.1
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
+CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef
+
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized beneath their categories.
+# The categories are represented by '##@' and the target descriptions by '##'.
+# The awk commands is responsible for reading the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help.
+# Then, if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Generate
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1)
+
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	go mod tidy
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)
+
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+generate-client: ## Generate kube-client.
+	sh hack/update-codegen.sh
+
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+vet: ## Run go vet against code.
+	go vet ./...
+
+generate-all: manifests kustomize generate fmt vet ## Generate all, include controller & client.
+
+##@ Build
+
+local-build: generate fmt vet ## Build manager binary.
+	go build -o bin/manager main.go
+
+docker-build: ## Build docker image with the manager.
+	docker build -t ${IMG} .
+
+docker-push: ## Push docker image with the manager.
+	docker push ${IMG}
+
+##@ Deployment
+
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+##@ Export
+
+export-crd: manifests kustomize ## Export CDR yaml.
+	$(KUSTOMIZE) build config/crd > deploy/crd.yaml
+
+export-operator: manifests kustomize docker-build docker-push ## Export operator yaml.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default > deploy/operator.yaml
+
+##@ Debug
+
+run-local: install ## Run for debug.
+	go run ./main.go
+
+run-remote: docker-build docker-push deploy ## Run on cluster.

PROJECT

+domain: oceanbase.com
+layout:
+- go.kubebuilder.io/v3
+multigroup: true
+projectName: ob-operator
+repo: github.com/oceanbase/ob-operator
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: oceanbase.com
+  group: cloud
+  kind: StatefulApp
+  path: github.com/oceanbase/ob-operator/api/v1
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: oceanbase.com
+  group: cloud
+  kind: OBCluster
+  path: github.com/oceanbase/ob-operator/apis/cloud/v1
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: oceanbase.com
+  group: cloud
+  kind: RootService
+  path: github.com/oceanbase/ob-operator/apis/cloud/v1
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: oceanbase.com
+  group: cloud
+  kind: OBZone
+  path: github.com/oceanbase/ob-operator/apis/cloud/v1
+  version: v1
+version: "3"

README-CN.md

+# ob-operator
+
+ob-operator 可以让 OceanBase 以容器的方式，无缝运行在公有云或私有部署的 Kubernetes 集群上。
+
+ob-operator 现已支持 OceanBase 集群的创建、删除，完整的节点生命周期管理，并通过 Service 的形式暴露给用户使用。后续会支持租户管理、ob-proxy 管理、多 Kubernetes 集群等功能。
+
+目前支持通过 YAML、Kustomize 来部署 ob-operator。后续会支持 Helm 形式部署。
+
+![01](docs/img/01.png)
+
+## 环境依赖
+
+构建 ob-operator 需要 Go 1.16 版本及以上。
+
+运行 ob-operator 需要 Kubernetes 1.16 版本及以上。
+尚未支持多 Kubernetes 版本，后续会支持。
+
+## 支持的 OceanBase 版本
+
+后续会不断支持新的 OceanBase 社区版本。
+暂不支持 3.1.2 之前的旧的 OceanBase 版本。目前仅对 OceanBase 社区版 3.1.2 进行了测试。
+
+## 开始体验
+
+### 使用 YAML 部署 ob-operator
+
+1. 部署 CRD 相关文件。
+
+```yaml
+kubectl apply -f ./deploy/crd.yaml
+```
+
+2. 部署 ob-operator 相关文件。
+
+注意配置 `--cluster-name` 启动参数，推荐与 Kubernetes 集群名称保持一致。
+
+```yaml
+kubectl apply -f ./deploy/operator.yaml
+```
+
+3. 配置节点 label。
+
+需要将 Kubernetes 节点打 label，label 需要与 obcluster.yaml 中 `nodeSelector` 配置相匹配。
+ob-operator 会将 Pod 调度到具有相应 label 的节点上。
+
+推荐配置 label 的 key 为 `topology.kubernetes.io/zone`。
+
+```yaml
+kubectl label node nodename topology.kubernetes.io/zone=zonename
+```
+
+4. 部署 OceanBase 集群。
+
+```yaml
+kubectl apply -f ./deploy/obcluster.yaml
+```
+
+5. 集群部署成功后可以使用 Service 的 ClusterIP 连接 OceanBase 集群。
+
+## 文档
+
+参考 [ob-operator 文档](docs/hello-cn.md)。
+
+## 获取帮助
+
+如果您在使用 ob-operator 时遇到任何问题，欢迎通过以下方式寻求帮助：
+
+- [GitHub Issue](https://github.com/oceanbase/ob-operator/issues)
+- [官方网站](https://open.oceanbase.com/)
+
+## 许可证
+
+ob-operator 使用 [MulanPSL - 2.0](http://license.coscl.org.cn/MulanPSL2) 许可证。
+您可以免费复制及使用源代码。当您修改或分发源代码时，请遵守木兰协议。

README.md

+# ob-operator
+
+ob-operator 可以让 OceanBase 以容器的方式，无缝运行在公有云或私有部署的 Kubernetes 集群上。
+
+ob-operator 现已支持 OceanBase 集群的创建、删除，完整的节点生命周期管理，并通过 Service 的形式暴露给用户使用。后续会支持租户管理、ob-proxy 管理、多 Kubernetes 集群等功能。
+
+目前支持通过 YAML、Kustomize 来部署 ob-operator。后续会支持 Helm 形式部署。
+
+![01](docs/img/01.png)
+
+## 环境依赖
+
+构建 ob-operator 需要 Go 1.16 版本及以上。
+
+运行 ob-operator 需要 Kubernetes 1.16 版本及以上。
+尚未支持多 Kubernetes 版本，后续会支持。
+
+## 支持的 OceanBase 版本
+
+后续会不断支持新的 OceanBase 社区版本。
+暂不支持 3.1.2 之前的旧的 OceanBase 版本。目前仅对 OceanBase 社区版 3.1.2 进行了测试。
+
+## 开始体验
+
+### 使用 YAML 部署 ob-operator
+
+1. 部署 CRD 相关文件。
+
+```yaml
+kubectl apply -f ./deploy/crd.yaml
+```
+
+2. 部署 ob-operator 相关文件。
+
+注意配置 `--cluster-name` 启动参数，推荐与 Kubernetes 集群名称保持一致。
+
+```yaml
+kubectl apply -f ./deploy/operator.yaml
+```
+
+3. 配置节点 label。
+
+需要将 Kubernetes 节点打 label，label 需要与 obcluster.yaml 中 `nodeSelector` 配置相匹配。
+ob-operator 会将 Pod 调度到具有相应 label 的节点上。
+
+推荐配置 label 的 key 为 `topology.kubernetes.io/zone`。
+
+```yaml
+kubectl label node nodename topology.kubernetes.io/zone=zonename
+```
+
+4. 部署 OceanBase 集群。
+
+```yaml
+kubectl apply -f ./deploy/obcluster.yaml
+```
+
+5. 集群部署成功后可以使用 Service 的 ClusterIP 连接 OceanBase 集群。
+
+## 文档
+
+参考 [ob-operator 文档](docs/hello-cn.md)。
+
+## 获取帮助
+
+如果您在使用 ob-operator 时遇到任何问题，欢迎通过以下方式寻求帮助：
+
+- [GitHub Issue](https://github.com/oceanbase/ob-operator/issues)
+- [官方网站](https://open.oceanbase.com/)
+
+## 许可证
+
+ob-operator 使用 [MulanPSL - 2.0](http://license.coscl.org.cn/MulanPSL2) 许可证。
+您可以免费复制及使用源代码。当您修改或分发源代码时，请遵守木兰协议。

apis/apis.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package apis
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// AddToSchemes may be used to add all resources defined in the project to a Scheme
+var AddToSchemes runtime.SchemeBuilder
+
+// AddToScheme adds all Resources to the Scheme
+func AddToScheme(s *runtime.Scheme) error {
+	return AddToSchemes.AddToScheme(s)
+}

apis/cloud/v1/doc.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+// +groupName=cloud.oceanbase.com
+
+package v1

apis/cloud/v1/groupversion_info.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+// Package v1 contains API Schema definitions for the cloud v1 API group
+// +kubebuilder:object:generate=true
+// +groupName=cloud.oceanbase.com
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "cloud.oceanbase.com", Version: "v1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

apis/cloud/v1/obcluster_types.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// OBClusterSpec defines the desired state of OBCluster
+type OBClusterSpec struct {
+	Version string `json:"version"`
+	// +kubebuilder:validation:Minimum=1
+	ClusterID int           `json:"clusterID"`
+	Topology  []Cluster     `json:"topology"`
+	Resources ResourcesSpec `json:"resources"`
+}
+
+type Cluster struct {
+	Cluster string   `json:"cluster"`
+	Zone    []Subset `json:"zone"`
+}
+
+type ResourcesSpec struct {
+	CPU     resource.Quantity `json:"cpu"`
+	Memory  resource.Quantity `json:"memory"`
+	Storage []StorageSpec     `json:"storage"`
+}
+
+type StorageSpec struct {
+	Name             string            `json:"name"`
+	StorageClassName string            `json:"storageClassName"`
+	Size             resource.Quantity `json:"size"`
+}
+
+// OBClusterStatus defines the observed state of OBCluster
+type OBClusterStatus struct {
+	Status   string          `json:"status"`
+	Topology []ClusterStatus `json:"topology"`
+}
+
+type ClusterStatus struct {
+	Cluster            string       `json:"cluster"`
+	ClusterStatus      string       `json:"clusterStatus"`
+	LastTransitionTime metav1.Time  `json:"lastTransitionTime"`
+	Zone               []ZoneStatus `json:"zone"`
+}
+
+type ZoneStatus struct {
+	Name              string `json:"name"`
+	Region            string `json:"region"`
+	ZoneStatus        string `json:"zoneStatus"`
+	ExpectedReplicas  int32  `json:"expectedReplicas"`
+	AvailableReplicas int32  `json:"availableReplicas"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// OBCluster is the Schema for the obclusters API
+type OBCluster struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata"`
+
+	Spec   OBClusterSpec   `json:"spec"`
+	Status OBClusterStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// OBClusterList contains a list of OBCluster
+type OBClusterList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OBCluster `json:"items"`
+}

apis/cloud/v1/obzone_types.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// OBZoneSpec defines the desired state of OBZone
+type OBZoneSpec struct {
+	Topology []Cluster `json:"topology"`
+}
+
+// OBZoneStatus defines the observed state of OBZone
+type OBZoneStatus struct {
+	Topology []ClusterOBZoneStatus `json:"topology"`
+}
+
+type ClusterOBZoneStatus struct {
+	Cluster string       `json:"cluster"`
+	Zone    []OBZoneInfo `json:"zone"`
+}
+
+type OBZoneInfo struct {
+	Name  string   `json:"name"`
+	Nodes []OBNode `json:"nodes"`
+}
+
+type OBNode struct {
+	ServerIP string `json:"serverIP"`
+	Status   string `json:"status"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// OBZone is the Schema for the obzones API
+type OBZone struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   OBZoneSpec   `json:"spec,omitempty"`
+	Status OBZoneStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// OBZoneList contains a list of OBZone
+type OBZoneList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OBZone `json:"items"`
+}

apis/cloud/v1/register.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// SchemeGroupVersion is group version used to register these objects.
+var SchemeGroupVersion = GroupVersion
+
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+func init() {
+	SchemeBuilder.Register(&StatefulApp{}, &StatefulAppList{}, &OBCluster{}, &OBClusterList{}, &RootService{}, &RootServiceList{}, &OBZone{}, &OBZoneList{})
+}

apis/cloud/v1/rootservice_types.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// RootServiceSpec defines the desired state of RootService
+type RootServiceSpec struct {
+	Topology []Cluster `json:"topology"`
+}
+
+// RootServiceStatus defines the observed state of RootService
+type RootServiceStatus struct {
+	Topology []ClusterRootServiceStatus `json:"topology"`
+}
+
+type ClusterRootServiceStatus struct {
+	Cluster string                  `json:"cluster"`
+	Zone    []ZoneRootServiceStatus `json:"zoneRootService"`
+}
+
+type ZoneRootServiceStatus struct {
+	Name     string `json:"name"`
+	ServerIP string `json:"serverIP"`
+	Role     int    `json:"role"`
+	Status   string `json:"status"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// RootService is the Schema for the rootservices API
+type RootService struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   RootServiceSpec   `json:"spec,omitempty"`
+	Status RootServiceStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// RootServiceList contains a list of RootService
+type RootServiceList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []RootService `json:"items"`
+}

apis/cloud/v1/statefulapp_types.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// StatefulAppSpec defines the desired state of StatefulApp
+type StatefulAppSpec struct {
+	Cluster          string            `json:"cluster"`
+	Subsets          []Subset          `json:"subsets"`
+	PodTemplate      corev1.PodSpec    `json:"podTemplate"`
+	StorageTemplates []StorageTemplate `json:"storageTemplates"`
+}
+
+type Subset struct {
+	Name         string            `json:"name"`
+	Region       string            `json:"region,omitempty"`
+	NodeSelector map[string]string `json:"nodeSelector"`
+	// +kubebuilder:validation:Minimum=1
+	Replicas int32 `json:"replicas"`
+}
+
+type StorageTemplate struct {
+	Name string                           `json:"name"`
+	PVC  corev1.PersistentVolumeClaimSpec `json:"pvc"`
+}
+
+// StatefulAppStatus defines the observed state of StatefulApp
+type StatefulAppStatus struct {
+	Cluster       string         `json:"cluster"`
+	ClusterStatus string         `json:"clusterStatus"`
+	Subsets       []SubsetStatus `json:"subsets"`
+}
+
+type SubsetStatus struct {
+	Name              string      `json:"name"`
+	Region            string      `json:"region,omitempty"`
+	ExpectedReplicas  int32       `json:"expectedReplicas"`
+	AvailableReplicas int32       `json:"availableReplicas"`
+	Pods              []PodStatus `json:"pods"`
+}
+
+type PodStatus struct {
+	Name     string          `json:"name"`
+	Index    int             `json:"index"`
+	PodPhase corev1.PodPhase `json:"podPhase"`
+	PodIP    string          `json:"podIP"`
+	NodeIP   string          `json:"nodeIP"`
+	PVCs     []PVCStatus     `json:"pvcs,omitempty"`
+}
+
+type PVCStatus struct {
+	Name  string                       `json:"name"`
+	Phase corev1.PersistentVolumePhase `json:"phase"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// StatefulApp is the Schema for the statefulapps API
+type StatefulApp struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata"`
+
+	Spec   StatefulAppSpec   `json:"spec"`
+	Status StatefulAppStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// StatefulAppList contains a list of StatefulApp
+type StatefulAppList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []StatefulApp `json:"items"`
+}

apis/cloud/v1/zz_generated.deepcopy.go

+//go:build !ignore_autogenerated
+// +build !ignore_autogenerated
+
+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Cluster) DeepCopyInto(out *Cluster) {
+	*out = *in
+	if in.Zone != nil {
+		in, out := &in.Zone, &out.Zone
+		*out = make([]Subset, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster.
+func (in *Cluster) DeepCopy() *Cluster {
+	if in == nil {
+		return nil
+	}
+	out := new(Cluster)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterOBZoneStatus) DeepCopyInto(out *ClusterOBZoneStatus) {
+	*out = *in
+	if in.Zone != nil {
+		in, out := &in.Zone, &out.Zone
+		*out = make([]OBZoneInfo, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterOBZoneStatus.
+func (in *ClusterOBZoneStatus) DeepCopy() *ClusterOBZoneStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterOBZoneStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterRootServiceStatus) DeepCopyInto(out *ClusterRootServiceStatus) {
+	*out = *in
+	if in.Zone != nil {
+		in, out := &in.Zone, &out.Zone
+		*out = make([]ZoneRootServiceStatus, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterRootServiceStatus.
+func (in *ClusterRootServiceStatus) DeepCopy() *ClusterRootServiceStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterRootServiceStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) {
+	*out = *in
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	if in.Zone != nil {
+		in, out := &in.Zone, &out.Zone
+		*out = make([]ZoneStatus, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus.
+func (in *ClusterStatus) DeepCopy() *ClusterStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ClusterStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBCluster) DeepCopyInto(out *OBCluster) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBCluster.
+func (in *OBCluster) DeepCopy() *OBCluster {
+	if in == nil {
+		return nil
+	}
+	out := new(OBCluster)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *OBCluster) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBClusterList) DeepCopyInto(out *OBClusterList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]OBCluster, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBClusterList.
+func (in *OBClusterList) DeepCopy() *OBClusterList {
+	if in == nil {
+		return nil
+	}
+	out := new(OBClusterList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *OBClusterList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBClusterSpec) DeepCopyInto(out *OBClusterSpec) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]Cluster, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	in.Resources.DeepCopyInto(&out.Resources)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBClusterSpec.
+func (in *OBClusterSpec) DeepCopy() *OBClusterSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(OBClusterSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBClusterStatus) DeepCopyInto(out *OBClusterStatus) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]ClusterStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBClusterStatus.
+func (in *OBClusterStatus) DeepCopy() *OBClusterStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(OBClusterStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBNode) DeepCopyInto(out *OBNode) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBNode.
+func (in *OBNode) DeepCopy() *OBNode {
+	if in == nil {
+		return nil
+	}
+	out := new(OBNode)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBZone) DeepCopyInto(out *OBZone) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBZone.
+func (in *OBZone) DeepCopy() *OBZone {
+	if in == nil {
+		return nil
+	}
+	out := new(OBZone)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *OBZone) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBZoneInfo) DeepCopyInto(out *OBZoneInfo) {
+	*out = *in
+	if in.Nodes != nil {
+		in, out := &in.Nodes, &out.Nodes
+		*out = make([]OBNode, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBZoneInfo.
+func (in *OBZoneInfo) DeepCopy() *OBZoneInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(OBZoneInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBZoneList) DeepCopyInto(out *OBZoneList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]OBZone, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBZoneList.
+func (in *OBZoneList) DeepCopy() *OBZoneList {
+	if in == nil {
+		return nil
+	}
+	out := new(OBZoneList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *OBZoneList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBZoneSpec) DeepCopyInto(out *OBZoneSpec) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]Cluster, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBZoneSpec.
+func (in *OBZoneSpec) DeepCopy() *OBZoneSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(OBZoneSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OBZoneStatus) DeepCopyInto(out *OBZoneStatus) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]ClusterOBZoneStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OBZoneStatus.
+func (in *OBZoneStatus) DeepCopy() *OBZoneStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(OBZoneStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PVCStatus) DeepCopyInto(out *PVCStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PVCStatus.
+func (in *PVCStatus) DeepCopy() *PVCStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(PVCStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PodStatus) DeepCopyInto(out *PodStatus) {
+	*out = *in
+	if in.PVCs != nil {
+		in, out := &in.PVCs, &out.PVCs
+		*out = make([]PVCStatus, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodStatus.
+func (in *PodStatus) DeepCopy() *PodStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(PodStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ResourcesSpec) DeepCopyInto(out *ResourcesSpec) {
+	*out = *in
+	out.CPU = in.CPU.DeepCopy()
+	out.Memory = in.Memory.DeepCopy()
+	if in.Storage != nil {
+		in, out := &in.Storage, &out.Storage
+		*out = make([]StorageSpec, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourcesSpec.
+func (in *ResourcesSpec) DeepCopy() *ResourcesSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ResourcesSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RootService) DeepCopyInto(out *RootService) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RootService.
+func (in *RootService) DeepCopy() *RootService {
+	if in == nil {
+		return nil
+	}
+	out := new(RootService)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *RootService) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RootServiceList) DeepCopyInto(out *RootServiceList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]RootService, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RootServiceList.
+func (in *RootServiceList) DeepCopy() *RootServiceList {
+	if in == nil {
+		return nil
+	}
+	out := new(RootServiceList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *RootServiceList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RootServiceSpec) DeepCopyInto(out *RootServiceSpec) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]Cluster, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RootServiceSpec.
+func (in *RootServiceSpec) DeepCopy() *RootServiceSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(RootServiceSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RootServiceStatus) DeepCopyInto(out *RootServiceStatus) {
+	*out = *in
+	if in.Topology != nil {
+		in, out := &in.Topology, &out.Topology
+		*out = make([]ClusterRootServiceStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RootServiceStatus.
+func (in *RootServiceStatus) DeepCopy() *RootServiceStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(RootServiceStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulApp) DeepCopyInto(out *StatefulApp) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulApp.
+func (in *StatefulApp) DeepCopy() *StatefulApp {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulApp)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *StatefulApp) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulAppList) DeepCopyInto(out *StatefulAppList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]StatefulApp, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulAppList.
+func (in *StatefulAppList) DeepCopy() *StatefulAppList {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulAppList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *StatefulAppList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulAppSpec) DeepCopyInto(out *StatefulAppSpec) {
+	*out = *in
+	if in.Subsets != nil {
+		in, out := &in.Subsets, &out.Subsets
+		*out = make([]Subset, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	in.PodTemplate.DeepCopyInto(&out.PodTemplate)
+	if in.StorageTemplates != nil {
+		in, out := &in.StorageTemplates, &out.StorageTemplates
+		*out = make([]StorageTemplate, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulAppSpec.
+func (in *StatefulAppSpec) DeepCopy() *StatefulAppSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulAppSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulAppStatus) DeepCopyInto(out *StatefulAppStatus) {
+	*out = *in
+	if in.Subsets != nil {
+		in, out := &in.Subsets, &out.Subsets
+		*out = make([]SubsetStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulAppStatus.
+func (in *StatefulAppStatus) DeepCopy() *StatefulAppStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulAppStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StorageSpec) DeepCopyInto(out *StorageSpec) {
+	*out = *in
+	out.Size = in.Size.DeepCopy()
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageSpec.
+func (in *StorageSpec) DeepCopy() *StorageSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(StorageSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StorageTemplate) DeepCopyInto(out *StorageTemplate) {
+	*out = *in
+	in.PVC.DeepCopyInto(&out.PVC)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageTemplate.
+func (in *StorageTemplate) DeepCopy() *StorageTemplate {
+	if in == nil {
+		return nil
+	}
+	out := new(StorageTemplate)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Subset) DeepCopyInto(out *Subset) {
+	*out = *in
+	if in.NodeSelector != nil {
+		in, out := &in.NodeSelector, &out.NodeSelector
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subset.
+func (in *Subset) DeepCopy() *Subset {
+	if in == nil {
+		return nil
+	}
+	out := new(Subset)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SubsetStatus) DeepCopyInto(out *SubsetStatus) {
+	*out = *in
+	if in.Pods != nil {
+		in, out := &in.Pods, &out.Pods
+		*out = make([]PodStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubsetStatus.
+func (in *SubsetStatus) DeepCopy() *SubsetStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(SubsetStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ZoneRootServiceStatus) DeepCopyInto(out *ZoneRootServiceStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ZoneRootServiceStatus.
+func (in *ZoneRootServiceStatus) DeepCopy() *ZoneRootServiceStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ZoneRootServiceStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ZoneStatus) DeepCopyInto(out *ZoneStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ZoneStatus.
+func (in *ZoneStatus) DeepCopy() *ZoneStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ZoneStatus)
+	in.DeepCopyInto(out)
+	return out
+}

cmd/cable/app.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package main
+
+import (
+	"github.com/oceanbase/ob-operator/pkg/cable/observer"
+	"github.com/oceanbase/ob-operator/pkg/cable/provider"
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func main() {
+	util.FuncList = make([]func(), 0)
+	observer.Paused = false
+
+	provider.InitForK8s()
+
+	util.SignalHandler(util.FuncList)
+	select {}
+}

config/crd/bases/cloud.oceanbase.com_obclusters.yaml

+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: obclusters.cloud.oceanbase.com
+spec:
+  group: cloud.oceanbase.com
+  names:
+    kind: OBCluster
+    listKind: OBClusterList
+    plural: obclusters
+    singular: obcluster
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: OBCluster is the Schema for the obclusters API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OBClusterSpec defines the desired state of OBCluster
+            properties:
+              clusterID:
+                minimum: 1
+                type: integer
+              resources:
+                properties:
+                  cpu:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                  memory:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                  storage:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        size:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        storageClassName:
+                          type: string
+                      required:
+                      - name
+                      - size
+                      - storageClassName
+                      type: object
+                    type: array
+                required:
+                - cpu
+                - memory
+                - storage
+                type: object
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    zone:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          nodeSelector:
+                            additionalProperties:
+                              type: string
+                            type: object
+                          region:
+                            type: string
+                          replicas:
+                            format: int32
+                            minimum: 1
+                            type: integer
+                        required:
+                        - name
+                        - nodeSelector
+                        - replicas
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - zone
+                  type: object
+                type: array
+              version:
+                type: string
+            required:
+            - clusterID
+            - resources
+            - topology
+            - version
+            type: object
+          status:
+            description: OBClusterStatus defines the observed state of OBCluster
+            properties:
+              status:
+                type: string
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    clusterStatus:
+                      type: string
+                    lastTransitionTime:
+                      format: date-time
+                      type: string
+                    zone:
+                      items:
+                        properties:
+                          availableReplicas:
+                            format: int32
+                            type: integer
+                          expectedReplicas:
+                            format: int32
+                            type: integer
+                          name:
+                            type: string
+                          region:
+                            type: string
+                          zoneStatus:
+                            type: string
+                        required:
+                        - availableReplicas
+                        - expectedReplicas
+                        - name
+                        - region
+                        - zoneStatus
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - clusterStatus
+                  - lastTransitionTime
+                  - zone
+                  type: object
+                type: array
+            required:
+            - status
+            - topology
+            type: object
+        required:
+        - metadata
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/bases/cloud.oceanbase.com_obzones.yaml

+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: obzones.cloud.oceanbase.com
+spec:
+  group: cloud.oceanbase.com
+  names:
+    kind: OBZone
+    listKind: OBZoneList
+    plural: obzones
+    singular: obzone
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: OBZone is the Schema for the obzones API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OBZoneSpec defines the desired state of OBZone
+            properties:
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    zone:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          nodeSelector:
+                            additionalProperties:
+                              type: string
+                            type: object
+                          region:
+                            type: string
+                          replicas:
+                            format: int32
+                            minimum: 1
+                            type: integer
+                        required:
+                        - name
+                        - nodeSelector
+                        - replicas
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - zone
+                  type: object
+                type: array
+            required:
+            - topology
+            type: object
+          status:
+            description: OBZoneStatus defines the observed state of OBZone
+            properties:
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    zone:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          nodes:
+                            items:
+                              properties:
+                                serverIP:
+                                  type: string
+                                status:
+                                  type: string
+                              required:
+                              - serverIP
+                              - status
+                              type: object
+                            type: array
+                        required:
+                        - name
+                        - nodes
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - zone
+                  type: object
+                type: array
+            required:
+            - topology
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/bases/cloud.oceanbase.com_rootservices.yaml

+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: rootservices.cloud.oceanbase.com
+spec:
+  group: cloud.oceanbase.com
+  names:
+    kind: RootService
+    listKind: RootServiceList
+    plural: rootservices
+    singular: rootservice
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: RootService is the Schema for the rootservices API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: RootServiceSpec defines the desired state of RootService
+            properties:
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    zone:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          nodeSelector:
+                            additionalProperties:
+                              type: string
+                            type: object
+                          region:
+                            type: string
+                          replicas:
+                            format: int32
+                            minimum: 1
+                            type: integer
+                        required:
+                        - name
+                        - nodeSelector
+                        - replicas
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - zone
+                  type: object
+                type: array
+            required:
+            - topology
+            type: object
+          status:
+            description: RootServiceStatus defines the observed state of RootService
+            properties:
+              topology:
+                items:
+                  properties:
+                    cluster:
+                      type: string
+                    zoneRootService:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          role:
+                            type: integer
+                          serverIP:
+                            type: string
+                          status:
+                            type: string
+                        required:
+                        - name
+                        - role
+                        - serverIP
+                        - status
+                        type: object
+                      type: array
+                  required:
+                  - cluster
+                  - zoneRootService
+                  type: object
+                type: array
+            required:
+            - topology
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/kustomization.yaml

+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+- bases/cloud.oceanbase.com_statefulapps.yaml
+- bases/cloud.oceanbase.com_obclusters.yaml
+- bases/cloud.oceanbase.com_rootservices.yaml
+- bases/cloud.oceanbase.com_obzones.yaml
+#+kubebuilder:scaffold:crdkustomizeresource
+
+patchesStrategicMerge:
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
+# patches here are for enabling the conversion webhook for each CRD
+#- patches/webhook_in_statefulapps.yaml
+#- patches/webhook_in_obclusters.yaml
+#- patches/webhook_in_rootservices.yaml
+#- patches/webhook_in_obzones.yaml
+#+kubebuilder:scaffold:crdkustomizewebhookpatch
+
+# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
+# patches here are for enabling the CA injection for each CRD
+#- patches/cainjection_in_statefulapps.yaml
+#- patches/cainjection_in_obclusters.yaml
+#- patches/cainjection_in_rootservices.yaml
+#- patches/cainjection_in_obzones.yaml
+#+kubebuilder:scaffold:crdkustomizecainjectionpatch
+
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+configurations:
+- kustomizeconfig.yaml

config/crd/kustomizeconfig.yaml

+# This file is for teaching kustomize how to substitute name and namespace reference in CRD
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: CustomResourceDefinition
+    version: v1
+    group: apiextensions.k8s.io
+    path: spec/conversion/webhook/clientConfig/service/name
+
+namespace:
+- kind: CustomResourceDefinition
+  version: v1
+  group: apiextensions.k8s.io
+  path: spec/conversion/webhook/clientConfig/service/namespace
+  create: false
+
+varReference:
+- path: metadata/annotations

config/crd/patches/cainjection_in_obclusters.yaml

+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: obclusters.cloud.oceanbase.com

config/crd/patches/cainjection_in_obzones.yaml

+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: obzones.cloud.oceanbase.com

config/crd/patches/cainjection_in_rootservices.yaml

+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: rootservices.cloud.oceanbase.com

config/crd/patches/cainjection_in_statefulapps.yaml

+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: statefulapps.cloud.oceanbase.com

config/crd/patches/webhook_in_obclusters.yaml

+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: obclusters.cloud.oceanbase.com
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/crd/patches/webhook_in_obzones.yaml

+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: obzones.cloud.oceanbase.com
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/crd/patches/webhook_in_rootservices.yaml

+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: rootservices.cloud.oceanbase.com
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/crd/patches/webhook_in_statefulapps.yaml

+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: statefulapps.cloud.oceanbase.com
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/default/kustomization.yaml

+# Adds namespace to all resources.
+namespace: oceanbase-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: ob-operator-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+bases:
+# - ../crd
+- ../rbac
+- ../manager
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- ../webhook
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
+#- ../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+#- ../prometheus
+
+patchesStrategicMerge:
+# Protect the /metrics endpoint by putting it behind auth.
+# If you want your controller-manager to expose the /metrics
+# endpoint w/o any authn/z, please comment the following line.
+- manager_auth_proxy_patch.yaml
+
+# Mount the controller config file for loading manager configurations
+# through a ComponentConfig type
+#- manager_config_patch.yaml
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- manager_webhook_patch.yaml
+
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
+# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
+# 'CERTMANAGER' needs to be enabled to use ca injection
+#- webhookcainjection_patch.yaml
+
+# the following config is for teaching kustomize how to do var substitution
+vars:
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
+#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+#  objref:
+#    kind: Certificate
+#    group: cert-manager.io
+#    version: v1
+#    name: serving-cert # this name should match the one in certificate.yaml
+#  fieldref:
+#    fieldpath: metadata.namespace
+#- name: CERTIFICATE_NAME
+#  objref:
+#    kind: Certificate
+#    group: cert-manager.io
+#    version: v1
+#    name: serving-cert # this name should match the one in certificate.yaml
+#- name: SERVICE_NAMESPACE # namespace of the service
+#  objref:
+#    kind: Service
+#    version: v1
+#    name: webhook-service
+#  fieldref:
+#    fieldpath: metadata.namespace
+#- name: SERVICE_NAME
+#  objref:
+#    kind: Service
+#    version: v1
+#    name: webhook-service

config/default/manager_auth_proxy_patch.yaml

+# This patch inject a sidecar container which is a HTTP proxy for the
+# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-rbac-proxy
+        image: oceanbase/obce-operator:kube-rbac-proxy-v0.8.0
+        args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=10"
+        ports:
+        - containerPort: 8443
+          name: https
+      - name: manager
+        args:
+        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--leader-elect"

config/default/manager_config_patch.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        args:
+        - "--config=controller_manager_config.yaml"
+        volumeMounts:
+        - name: manager-config
+          mountPath: /controller_manager_config.yaml
+          subPath: controller_manager_config.yaml
+      volumes:
+      - name: manager-config
+        configMap:
+          name: manager-config

config/manager/controller_manager_config.yaml

+apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
+kind: ControllerManagerConfig
+health:
+  healthProbeBindAddress: :8081
+metrics:
+  bindAddress: 127.0.0.1:8080
+webhook:
+  port: 9443
+leaderElection:
+  leaderElect: true
+  resourceName: ee2dec96.oceanbase.com

config/manager/kustomization.yaml

+resources:
+- manager.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configMapGenerator:
+- files:
+  - controller_manager_config.yaml
+  name: manager-config
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: oceanbase/obce-operator
+  newTag: v0.0.1

config/manager/manager.yaml

+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: oceanbase-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+  labels:
+    control-plane: controller-manager
+spec:
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      securityContext:
+        runAsNonRoot: true
+      containers:
+      - command:
+        - /manager
+        args:
+        - --leader-elect
+        image: controller:latest
+        imagePullPolicy: Always
+        name: manager
+        securityContext:
+          allowPrivilegeEscalation: false
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 1
+            memory: 1Gi
+      serviceAccountName: controller-manager
+      terminationGracePeriodSeconds: 10

config/prometheus/kustomization.yaml

+resources:
+- monitor.yaml

config/prometheus/monitor.yaml

+
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: controller-manager-metrics-monitor
+  namespace: system
+spec:
+  endpoints:
+    - path: /metrics
+      port: https
+      scheme: https
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      tlsConfig:
+        insecureSkipVerify: true
+  selector:
+    matchLabels:
+      control-plane: controller-manager

config/rbac/auth_proxy_client_clusterrole.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get

config/rbac/auth_proxy_role.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: proxy-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/rbac/auth_proxy_role_binding.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: proxy-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: proxy-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/auth_proxy_service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: controller-manager-metrics-service
+  namespace: system
+spec:
+  ports:
+  - name: https
+    port: 8443
+    targetPort: https
+  selector:
+    control-plane: controller-manager

config/rbac/kustomization.yaml

+resources:
+# All RBAC will be applied under this service account in
+# the deployment namespace. You may comment out this resource
+# if your manager will use a service account that exists at
+# runtime. Be sure to update RoleBinding and ClusterRoleBinding
+# subjects if changing service account names.
+- service_account.yaml
+- role.yaml
+- role_binding.yaml
+- leader_election_role.yaml
+- leader_election_role_binding.yaml
+# Comment the following 4 lines if you want to disable
+# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
+# which protects your /metrics endpoint.
+- auth_proxy_service.yaml
+- auth_proxy_role.yaml
+- auth_proxy_role_binding.yaml
+- auth_proxy_client_clusterrole.yaml

config/rbac/leader_election_role.yaml

+# permissions to do leader election.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: leader-election-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch

config/rbac/leader_election_role_binding.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: leader-election-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: leader-election-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/obcluster_editor_role.yaml

+# permissions for end users to edit obclusters.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: obcluster-editor-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/status
+  verbs:
+  - get

config/rbac/obcluster_viewer_role.yaml

+# permissions for end users to view obclusters.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: obcluster-viewer-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/status
+  verbs:
+  - get

config/rbac/obzone_editor_role.yaml

+# permissions for end users to edit obzones.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: obzone-editor-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obzones
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obzones/status
+  verbs:
+  - get

config/rbac/obzone_viewer_role.yaml

+# permissions for end users to view obzones.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: obzone-viewer-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obzones
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obzones/status
+  verbs:
+  - get

config/rbac/role.yaml

+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: manager-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/sfinalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - get
+  - patch
+  - update

config/rbac/role_binding.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: manager-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: manager-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/rootservice_editor_role.yaml

+# permissions for end users to edit rootservices.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rootservice-editor-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - rootservices
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - rootservices/status
+  verbs:
+  - get

config/rbac/rootservice_viewer_role.yaml

+# permissions for end users to view rootservices.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rootservice-viewer-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - rootservices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - rootservices/status
+  verbs:
+  - get

config/rbac/service_account.yaml

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: controller-manager
+  namespace: system

config/rbac/statefulapp_editor_role.yaml

+# permissions for end users to edit statefulapps.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: statefulapp-editor-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/status
+  verbs:
+  - get

config/rbac/statefulapp_viewer_role.yaml

+# permissions for end users to view statefulapps.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: statefulapp-viewer-role
+rules:
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/status
+  verbs:
+  - get

config/samples/cloud_v1_obcluster.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: ob
+  name: ob-test
+spec:
+  version: v3.1.2-10000392021123010  # OB 版本
+  clusterID: 1
+  topology:  # 拓扑分布
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+  resources:  # OB 所需要的资源
+    cpu: 7
+    memory: 60Gi
+    storage:
+      - name: log
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: data
+        storageClassName: "local-path"
+        size: 50Gi
+status:
+  status: Ready
+  topology:
+    - cluster: cn
+      clusterStatus: Ready
+      lastTransitionTime: "2021-12-08T07:07:55Z"
+      zone:
+      - name: zone1
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1
+      - name: zone2
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1
+      - name: zone3
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1

config/samples/cloud_v1_obzone.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: OBZone
+metadata:
+  labels:
+    app: ob-test
+  name: obzone-ob-test
+  namespace: ob
+spec:
+  topology:
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+status:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      nodes:
+      - serverIP: 10.10.10.1
+        status: active
+    - name: zone2
+      nodes:
+      - serverIP: 10.10.10.2
+        status: active
+    - name: zone3
+      nodes:
+      - serverIP: 10.10.10.3
+        status: active

config/samples/cloud_v1_rootservice.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: RootService
+metadata:
+  labels:
+    app: ob-test
+  name: rs-ob-test
+  namespace: ob
+spec:
+  topology:
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+status:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      role: 1
+      serverIP: 10.10.10.1
+      status: active
+    - name: zone2
+      role: 2
+      serverIP: 10.10.10.2
+      status: active
+    - name: zone3
+      role: 2
+      serverIP: 10.10.10.3
+      status: active

config/samples/cloud_v1_statefulapp.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: StatefulApp
+metadata:
+  namespace: ob
+  name: sapp
+spec:
+  cluster: cn  # Operator 监控的集群名称，启动的时候需要指定
+  subsets:
+  - name: zone1
+    nodeSelector:  # 强制匹配到某个 Zone
+      topology.kubernetes.io/zone: zone1
+    replicas: 2
+  - name: zone2
+    nodeSelector:
+      topology.kubernetes.io/zone: zone2
+    replicas: 1
+  - name: zone3
+    nodeSelector:
+      topology.kubernetes.io/zone: zone3
+    replicas: 1
+  podTemplate:  # Pod 自身资源的描述
+    containers:
+      - name: observer
+        image: nginx
+        resources:
+          requests:
+            cpu: "1"
+            memory: "2Gi"
+        volumeMounts:
+          - name: log
+            mountPath: /home/admin/oceanbase/log
+          - name: data
+            mountPath: /home/admin/oceanbase/store
+    volumes:
+      - name: log
+        persistentVolumeClaim:
+          claimName: log
+      - name: data
+        persistentVolumeClaim:
+          claimName: data
+  storageTemplates:  # 对 Pod 所依赖的存储的描述
+  - name: log
+    pvc:
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi
+  - name: data
+    pvc:
+      accessModes:
+          - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi
+status:
+  cluster: cn
+  clusterStatus: Ready
+  subsets:
+  - name: zone1
+    expectedReplicas: 3
+    availableReplicas: 1
+    pods:
+    - name: sapp-1
+      index: 0
+      podPhase: Running
+      podIP: 10.10.10.1
+      nodeIP: 1.1.1.1
+      pvcs:
+      - name: sapp-cn-zone1-0-data-store
+        phase: Bound

deploy/obcluster.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: ob
+  name: ob-test
+spec:
+  version: v3.1.2-10000392021123010
+  clusterID: 1
+  topology:
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+  resources:
+    cpu: 2
+    memory: 10Gi
+    storage:
+      - name: data-file
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: data-log
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: log
+        storageClassName: "local-path"
+        size: 30Gi

deploy/operator.yaml

+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: oceanbase-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ob-operator-leader-election-role
+  namespace: oceanbase-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: ob-operator-manager-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - obclusters/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/sfinalizers
+  verbs:
+  - update
+- apiGroups:
+  - cloud.oceanbase.com
+  resources:
+  - statefulapps/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - get
+  - patch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ob-operator-metrics-reader
+rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ob-operator-proxy-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ob-operator-leader-election-rolebinding
+  namespace: oceanbase-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ob-operator-leader-election-role
+subjects:
+- kind: ServiceAccount
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ob-operator-manager-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ob-operator-manager-role
+subjects:
+- kind: ServiceAccount
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ob-operator-proxy-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ob-operator-proxy-role
+subjects:
+- kind: ServiceAccount
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+---
+apiVersion: v1
+data:
+  controller_manager_config.yaml: |
+    apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
+    kind: ControllerManagerConfig
+    health:
+      healthProbeBindAddress: :8081
+    metrics:
+      bindAddress: 127.0.0.1:8080
+    webhook:
+      port: 9443
+    leaderElection:
+      leaderElect: true
+      resourceName: ee2dec96.oceanbase.com
+kind: ConfigMap
+metadata:
+  name: ob-operator-manager-config
+  namespace: oceanbase-system
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: ob-operator-controller-manager-metrics-service
+  namespace: oceanbase-system
+spec:
+  ports:
+  - name: https
+    port: 8443
+    targetPort: https
+  selector:
+    control-plane: controller-manager
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      containers:
+      - args:
+        - --secure-listen-address=0.0.0.0:8443
+        - --upstream=http://127.0.0.1:8080/
+        - --logtostderr=true
+        - --v=10
+        image: oceanbase/obce-operator:kube-rbac-proxy-v0.8.0
+        name: kube-rbac-proxy
+        ports:
+        - containerPort: 8443
+          name: https
+      - args:
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --leader-elect
+        command:
+        - /manager
+        image: oceanbase/obce-operator:v0.0.1
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        name: manager
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 1
+            memory: 1Gi
+        securityContext:
+          allowPrivilegeEscalation: false
+      securityContext:
+        runAsNonRoot: true
+      serviceAccountName: ob-operator-controller-manager
+      terminationGracePeriodSeconds: 10

docs/arch-cn.md

+# 架构设计
+
+ob-operator 与其他 operator 一样，旨在让 OceanBase 以容器的方式，无缝运行在 Kubernetes 集群上。
+
+现阶段的 ob-operator 是一个孵化版，即为一个具备基础功能的框架，后续还需不断的迭代与完善。
+
+ob-operator 现已支持 OceanBase 集群的创建、删除，完整的节点生命周期管理，并通过 Service 的形式暴露给用户使用。
+后续会支持租户管理、ob-proxy 管理、多 Kubernetes 集群等功能。
+
+目前支持通过 YAML、Kustomize 来部署 ob-operator。
+后续会支持 Helm 形式部署。
+
+## 基础概念
+
+![01](img/02.png)
+
+OceanBase 具有很好的高可用能力。
+
+OceanBase 集群由多个 Zone 和多台 OBServer 构成。
+OceanBase 集群的唯一标识是集群名称和集群 ID。
+
+![01](img/03.png)
+
+一个 OceanBase 集群，由若干个 Zone 组成。
+Zone 是可用区（Availability Zone）的简写。
+
+Zone 本身是一个逻辑概念，一般是同一机房的一组机器的组合。
+物理层面来讲一个 Zone 通常等价于一个机房、一个数据中心或一个 IDC。
+为保障更高级别的数据安全性和服务可用性， 一个 OceanBase 集群通常会分布在同城的 3 个机房中，同一份数据的三个副本分别分布在 3 个机房中（即三个 Zone 中）。
+
+OceanBase 数据库支持数据跨地域（Region）部署，且不同 Region 的距离通常较远，从而满足地域级容灾的需求。一个 Region 可以包含一个或者多个 Zone。
+
+Kubernetes 场景下需要对不同的节点打 label，以标识不同的 Zone 和 Region。
+
+现阶段 ob-operator 已支持多 Zone 部署，尚不支持多 Region 部署。
+
+## 分层架构
+
+![01](img/01.png)
+
+整个项目分三块：StatefulApp Controller、OBServer Controller 和 Operator Orchestrator。
+
+StatefulApp Controller 负责 Pod、PVC、PV 的维护。
+OBServer Controller 负责 OBServer 的维护。
+Operator Orchestrator 负责在多个 Kubernetes 之间管理 ob-operator（该模块尚未实现）。
+
+由于 OBServer 的注册信息与通信地址以 IP 为主，所以 Kubernetes 资源层基于 Simple Pod 封装自定义 CRD 实现。
+
+StatefulApp Controller 是一个类似 StatefulSet 的管控工具，但与 StatefulSet 略有区别：
+
+- StatefulApp 为每个 Pod 维护了一个有粘性的 ID。
+- 每个 Pod 有自己独立的存储，相互不共享。
+- StatefulApp 支持 Pod 间配置异构。
+- StatefulApp 的部署、更新是有序的，同 StatefulSet 一样。
+- StatefulApp 支持对指定 Pod 做操作。
+- StatefulApp 不支持基于 serviceName 的通信。
+
+![01](img/04.png)
+
+每个 Pod 会映射为一个 OBServer。
+
+目前一个 Pod 中只包含一个 OBServer 容器，后续会以 Sidecar 的形式支持日志与监控容器。
+
+OBServer 容器中会有一个很轻量的 Agent，负责与 OBServer Controller 交互、执行运维操作、维护 OBServer 进程。
+
+![01](img/05.png)
+
+OB Operator 会创建三种 CRD：OBCluster、OBRootService 和 OBServer。OBZone、OBRootService 为 OBCluster 的子资源。
+
+OBCluster：
+对 OceanBase 集群的定义，以整体的状态管理，全局视角，不涉及细节。用户需要自己定义 OBCluster 的配置。
+
+```yaml
+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: oceanbase
+  name: ob-test
+spec:
+  version: 3.1.1-4
+  clusterID: 1
+  topology:
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+  resources:
+    cpu: 2
+    memory: 10Gi
+    storage:
+      - name: data-file
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: data-log
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: log
+        storageClassName: "local-path"
+        size: 30Gi
+status:
+  status: Ready
+  topology:
+    - cluster: cn
+      clusterStatus: Ready
+      lastTransitionTime: "2021-12-01T07:07:55Z"
+      zone:
+      - name: zone1
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1
+      - name: zone2
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1
+      - name: zone3
+        region: region1
+        zoneStatus: Ready
+        expectedReplicas: 1
+        availableReplicas: 1
+```
+
+OBZone：
+Zone 维度 OBServer 的状态。用户可以通过 OBZone 来查看 OceanBase 集群中 OBServer 的状态。用户不需要创建，ob-operator 会自动维护并定期更新。
+
+```yaml
+apiVersion: cloud.oceanbase.com/v1
+kind: OBZone
+metadata:
+  namespace: oceanbase
+  name: obzone-ob-test
+  labels:
+    app: ob-test
+spec:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      nodeSelector:
+        topology.kubernetes.io/zone: zone1
+      region: region1
+      replicas: 1
+    - name: zone2
+      nodeSelector:
+        topology.kubernetes.io/zone: zone2
+      region: region1
+      replicas: 1
+    - name: zone3
+      nodeSelector:
+        topology.kubernetes.io/zone: zone3
+      region: region1
+      replicas: 1
+status:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      nodes:
+      - serverIP: 10.10.10.1
+        status: active
+    - name: zone2
+      nodes:
+      - serverIP: 10.10.10.2
+        status: active
+    - name: zone3
+      nodes:
+      - serverIP: 10.10.10.3
+        status: active
+```
+
+RootService：
+集群维度 RootService 的分布情况、单个 RootService 的健康状态。用户可以通过 RootService 来查看 OceanBase 集群中 RootService 的状态。用户不需要创建，ob-operator 会自动维护并定期更新。
+
+```yaml
+apiVersion: cloud.oceanbase.com/v1
+kind: RootService
+metadata:
+  namespace: oceanbase
+  name: rs-ob-test
+  labels:
+    app: ob-test
+spec:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      nodeSelector:
+        topology.kubernetes.io/zone: zone1
+      region: region1
+      replicas: 1
+    - name: zone2
+      nodeSelector:
+        topology.kubernetes.io/zone: zone2
+      region: region1
+      replicas: 1
+    - name: zone3
+      nodeSelector:
+        topology.kubernetes.io/zone: zone3
+      region: region1
+      replicas: 1
+status:
+  topology:
+  - cluster: cn
+    zone:
+    - name: zone1
+      role: 1
+      serverIP: 10.10.10.1
+      status: active
+    - name: zone2
+      role: 2
+      serverIP: 10.10.10.2
+      status: active
+    - name: zone3
+      role: 2
+      serverIP: 10.10.10.3
+      status: active
+```

docs/deploy-cn.md

+# 自定义配置
+
+## 部署 CRD
+
+```yaml
+kubectl apply -f ./deploy/crd.yaml
+```
+
+## 部署 ob-operator
+
+您需要根据您的配置修改 `operator.yaml` 。
+
+您需要添加启动参数 `--cluster-name`，该参数需要与 obcluster 中的 `cluster` 配置一致。
+该配置的含义：ob-operator 只会处理 `cluster` 的值与自身启动参数 `--cluster-name` 的值相同的 CRD。
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      containers:
+      - args:
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --leader-elect
+        - --cluster-name=cn
+        command:
+        - /manager
+        image: ob-operator:latest
+        imagePullPolicy: Always
+        name: manager
+```
+
+## 配置节点 label
+
+需要将 Kubernetes 节点打 label，label 需要与 obcluster.yaml 中 `nodeSelector` 配置相匹配。ob-operator 会将 Pod 调度到具有相应 label 的节点上。
+
+推荐配置 label 的 key 为 `topology.kubernetes.io/zone`。不同 Zone 推荐配置不同的 label 以做容灾。
+
+```yaml
+kubectl label node nodename topology.kubernetes.io/zone=zonename
+```
+
+## 部署 OceanBase 集群
+
+`obcluster.yaml` 中需要用户根据自己的配置做一些修改。
+
+```yaml
+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: oceanbase
+  name: ob-test
+spec:
+  version: 3.1.1-4
+  clusterID: 1
+  topology:
+    - cluster: cn
+      zone:
+      - name: zone1
+        region: regio1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: regio1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: regio1
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+  resources:
+    cpu: 2
+    memory: 10Gi
+    storage:
+      - name: data-file
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: data-log
+        storageClassName: "local-path"
+        size: 50Gi
+      - name: log
+        storageClassName: "local-path"
+        size: 30Gi
+```
+
+- `version` OceanBase 集群的版本。
+- `cluster` 需要按需配置，如果需要在该 Kubernetes 集群中部署 OceanBase 集群，请将 `cluster` 配置为与 ob-operator 启动参数 `--cluster-name` 相同的配置。
+- `cpu` 配置建议为大于 2 的整数，小于 2 会引发系统异常。
+- `memory` 配置建议为大于 10Gi 的整数，小于 10Gi 会引发系统异常。
+- `storage` 的 `data-file` 部分为 OBServer 系统配置项 `datafile_size` 的大小，建议为 `memory` 的 3 倍以上。`storageClassName` 用户可以自行按需配置。
+- `storage` 的 `data-log` 部分为 OBServer 系统配置项 `data_dir` 的大小，建议为 `memory` 的 5 倍以上。`storageClassName` 用户可以自行按需配置。
+- `storage` 的 `log` 部分为 OBServer 系统日志的大小，建议为 30Gi 以上。`storageClassName` 用户可以自行按需配置。
+
+`nodeSelector` 的数据结构：
+
+```go
+// NodeSelector is a selector which must be true for the pod to fit on a node.
+// Selector which must match a node's labels for the pod to be scheduled on that node.
+// More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
+// +optional
+// +mapType=atomic
+NodeSelector map[string]string `json:"nodeSelector,omitempty" protobuf:"bytes,7,rep,name=nodeSelector"`
+```

docs/develop-cn.md

+# 如何参与开发
+
+## 代码目录结构
+
+```yaml
+.
+├── Dockerfile
+├── LEGAL.md
+├── LICENSE
+├── LICENSE.Apache
+├── LICENSE.MIT
+├── Makefile
+├── PROJECT
+├── README-CN.md
+├── README.md
+├── apis  // CRD 定义
+├── cmd
+├── config  // kustomize 相关配置与生成文件
+├── deploy  // 部署服务所使用的的文件
+├── docs  // 文档
+├── go.mod
+├── go.sum
+├── hack
+├── main.go
+├── pkg
+│ ├── cable  // 守护进程逻辑
+│ ├── config
+│ ├── controllers
+│ │ ├── observer  // observer controller 逻辑
+│ │ │ └── cable  // 与 Agent 进程交互的部分
+│ │ └── statefulapp  // statefulapp controller 逻辑
+│ ├── infrastructure
+│ │ ├── kube
+│ │ └── ob
+│ ├── kubeclient
+│ └── util
+├── scripts
+│ └── observer
+└── test
+    └── e2e
+```

docs/hello-cn.md

+# 文档
+
+- [架构设计](arch-cn.md)
+- [配置 OceanBase 集群](deploy-cn.md)
+- [参与开发](develop-cn.md)

go.mod

+module github.com/oceanbase/ob-operator
+
+go 1.16
+
+require (
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/gin-gonic/gin v1.7.4
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/go-cmp v0.5.5
+	github.com/jinzhu/gorm v1.9.16
+	github.com/pkg/errors v0.9.1
+	github.com/shirou/gopsutil/v3 v3.20.10
+	github.com/sirupsen/logrus v1.8.1
+	github.com/smartystreets/goconvey v1.6.4
+	github.com/stretchr/testify v1.7.0
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	golang.org/x/tools v0.1.8 // indirect
+	k8s.io/api v0.22.1
+	k8s.io/apiextensions-apiserver v0.22.1
+	k8s.io/apimachinery v0.22.1
+	k8s.io/client-go v0.22.1
+	k8s.io/code-generator v0.22.1
+	k8s.io/klog/v2 v2.9.0
+	sigs.k8s.io/controller-runtime v0.10.0
+)

go.sum

+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
+github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
+github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM=
+github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
+github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
+github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
+github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
+github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
+github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
+github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shirou/gopsutil/v3 v3.20.10 h1:7zomV9HJv6UGk225YtvEa5+camNLpbua3MAz/GqiVJY=
+github.com/shirou/gopsutil/v3 v3.20.10/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
+go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
+go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
+go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
+go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
+go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
+go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
+go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
+go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
+go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
+go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
+go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
+golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
+gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY=
+k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
+k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE=
+k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c=
+k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM=
+k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400=
+k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw=
+k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
+k8s.io/code-generator v0.22.1 h1:zAcKpn+xe9Iyc4qtZlfg4tD0f+SO2h5+e/s4pZPOVhs=
+k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
+k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo=
+k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
+k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2MoKj5U=
+sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

hack/boilerplate.go.txt

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
\ No newline at end of file

hack/tools.go

+//go:build tools
+// +build tools
+
+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+// 建立 tools.go 来依赖 code-generator
+// 因为在没有代码使用 code-generator 时，go module 默认不依赖此包
+package tools
+
+import _ "k8s.io/code-generator"

hack/update-codegen.sh

+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# get code-generator
+K8S_VERSION=v0.22.1
+go get k8s.io/code-generator@$K8S_VERSION
+go get k8s.io/client-go@$K8S_VERSION
+go get k8s.io/apimachinery@$K8S_VERSION
+go get sigs.k8s.io/controller-runtime@v0.10.0
+go mod vendor
+chmod +x vendor/k8s.io/code-generator/generate-groups.sh
+
+# corresponding to go mod init <module>
+MODULE=github.com/oceanbase/ob-operator
+# client package
+OUTPUT_PKG=pkg/kubeclient
+# api package
+APIS_PKG=apis
+# group-version
+GROUP_VERSION=cloud:v1
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)}
+
+rm -rf ${OUTPUT_PKG}/clientset
+rm -rf ${OUTPUT_PKG}/listers
+rm -rf ${OUTPUT_PKG}/informers
+
+bash "${CODEGEN_PKG}"/generate-groups.sh "client,lister,informer" \
+  ${MODULE}/${OUTPUT_PKG} \
+  ${MODULE}/${APIS_PKG} \
+  ${GROUP_VERSION} \
+  --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate.go.txt
+
+rm -rf vendor/

hack/verify-codegen.sh

+#! /usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+DIFFROOT="${SCRIPT_ROOT}/pkg"
+TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/pkg"
+_tmp="${SCRIPT_ROOT}/_tmp"
+
+cleanup() {
+  rm -rf "${_tmp}"
+}
+trap "cleanup" EXIT SIGINT
+
+cleanup
+
+mkdir -p "${TMP_DIFFROOT}"
+cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}"
+
+"${SCRIPT_ROOT}/hack/update-codegen.sh"
+echo "diffing ${DIFFROOT} against freshly generated codegen"
+ret=0
+diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$?
+cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}"
+if [[ $ret -eq 0 ]]
+then
+  echo "${DIFFROOT} up to date."
+else
+  echo "${DIFFROOT} is out of date. Please run hack/update-codegen.sh"
+  exit 1
+fi

main.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package main
+
+import (
+	"flag"
+	"net/http"
+	"os"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/klogr"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	controller "github.com/oceanbase/ob-operator/pkg/controllers"
+	"github.com/oceanbase/ob-operator/pkg/util"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	// exit func list
+	funcList []func()
+	scheme   = runtime.NewScheme()
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(cloudv1.AddToScheme(scheme))
+	// +kubebuilder:scaffold:scheme
+}
+
+func main() {
+	var clusterName string
+	var namespace string
+	var enableLeaderElection bool
+	var managerNamespace string
+	var syncPeriodStr string
+	var probeAddr string
+	var metricsAddr string
+	var enablePprof bool
+	var pprofAddr string
+
+	flag.StringVar(&clusterName, "cluster-name", "cn",
+		"Which cluster to run oceanbase. Defaults is cn.")
+	flag.StringVar(&namespace, "namespace", "",
+		"Which namespace to run oceanbase. Defaults is all namespace.")
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
+		"Enable leader election for controller manager.")
+	flag.StringVar(&managerNamespace, "manager-namespace", "oceanbase-system",
+		"Which namespace to run manager tools.")
+	flag.StringVar(&syncPeriodStr, "sync-period", "5s",
+		"How often should data be synchronized.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":9090",
+		"The address the probe endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080",
+		"The address the metric endpoint binds to.")
+	flag.BoolVar(&enablePprof, "enable-pprof", false,
+		"Enable pprof for controller manager.")
+	flag.StringVar(&pprofAddr, "pprof-addr", ":9091",
+		"The address the pprof binds to.")
+
+	flag.Parse()
+
+	klog.InitFlags(nil)
+	ctrl.SetLogger(klogr.New())
+
+	myconfig.ClusterName = clusterName
+
+	// pprof
+	if enablePprof {
+		go func() {
+			err := http.ListenAndServe(pprofAddr, nil)
+			if err != nil {
+				klog.Errorln(err, "unable to start pprof")
+			}
+		}()
+	}
+
+	// syncPeriod
+	var syncPeriod *time.Duration
+	if syncPeriodStr != "" {
+		duration, err := time.ParseDuration(syncPeriodStr)
+		if err != nil {
+			klog.Errorln(err, "invalid sync period flag")
+		} else {
+			syncPeriod = &duration
+		}
+	}
+
+	mgr, err := ctrl.NewManager(
+		ctrl.GetConfigOrDie(),
+		ctrl.Options{
+			Scheme:                     scheme,
+			Namespace:                  namespace,
+			LeaderElection:             enableLeaderElection,
+			LeaderElectionNamespace:    managerNamespace,
+			LeaderElectionID:           "oceanbase-system",
+			LeaderElectionResourceLock: resourcelock.ConfigMapsResourceLock,
+			SyncPeriod:                 syncPeriod,
+			HealthProbeBindAddress:     probeAddr,
+			MetricsBindAddress:         metricsAddr,
+		},
+	)
+	if err != nil {
+		klog.Errorln(err, "unable to start manager.")
+		os.Exit(1)
+	}
+
+	if err = controller.SetupWithManager(mgr); err != nil {
+		klog.Errorln(err, "unable to setup controllers.")
+		os.Exit(1)
+	}
+
+	if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		klog.Errorln(err, "unable to set up health check.")
+		os.Exit(1)
+	}
+	if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		klog.Errorln(err, "unable to set up ready check.")
+		os.Exit(1)
+	}
+
+	klog.Infoln("starting manager.")
+	if err = mgr.Start(util.SignalHandler(funcList)); err != nil {
+		klog.Errorln(err, "problem running manager.")
+		os.Exit(1)
+	}
+}

pkg/cable/observer/const.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+const (
+	ProcessObserver = "observer"
+)
+
+var OBStarted bool
+var Paused bool

pkg/cable/observer/status.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"log"
+	"time"
+
+	"github.com/oceanbase/ob-operator/pkg/util/system"
+)
+
+var Liveness bool
+var Readiness bool
+
+const GracefulTime = 10 * time.Second
+const TickTime = 5 * time.Second
+
+func CheckOBServeStatus() {
+	time.Sleep(GracefulTime)
+	// checker
+	tick := time.Tick(TickTime)
+	for {
+		select {
+		case <-tick:
+			checkerOBServer()
+		}
+	}
+}
+
+func checkerOBServer() {
+	name := ProcessObserver
+	pm := &system.ProcessManager{}
+	status := pm.ProcessIsRunningByName(name)
+	if status {
+		// update liveness
+		Liveness = true
+	} else {
+		if Paused {
+			// update liveness
+			Liveness = true
+		} else {
+			log.Println("not Paused")
+			system.Exit()
+		}
+	}
+}

pkg/cable/observer/stop.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"log"
+	"time"
+
+	"github.com/oceanbase/ob-operator/pkg/util/system"
+)
+
+func StopProcess() {
+	name := ProcessObserver
+	pm := &system.ProcessManager{}
+	err := pm.TerminateProcessByName(name)
+	if err != nil {
+		log.Println(err)
+	}
+	time.Sleep(2 * time.Second)
+	err = pm.KillProcessByName(name)
+	if err != nil {
+		log.Println(err)
+	}
+	OBStarted = false
+}

pkg/cable/provider/controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package provider
+
+import (
+	"log"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/oceanbase/ob-operator/pkg/cable/observer"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/ob"
+	"github.com/oceanbase/ob-operator/pkg/util"
+	"github.com/oceanbase/ob-operator/pkg/util/system"
+)
+
+func Info(c *gin.Context) {
+	data, err := system.GetNICInfo(ob.NIC)
+	if err != nil {
+		panic(err)
+	}
+	Sender(c, 200, data)
+}
+
+func Paused(c *gin.Context) {
+	observer.Paused = true
+	log.Println("Paused is", observer.Paused)
+	data := make(map[string]interface{})
+	Sender(c, 200, data)
+}
+
+func Rework(c *gin.Context) {
+	observer.Paused = false
+	log.Println("Paused is", observer.Paused)
+	data := make(map[string]interface{})
+	Sender(c, 200, data)
+}
+
+func OBStart(c *gin.Context) {
+	param := new(ob.StartObServerProcessArguments)
+	if err := c.ShouldBind(&param); err != nil {
+		panic(err)
+	}
+	log.Println(util.CovertToJSON(param))
+
+	if !observer.OBStarted {
+		go ob.StartOBServerProcess(*param)
+		go observer.CheckOBServeStatus()
+		observer.OBStarted = true
+		data := make(map[string]interface{})
+		Sender(c, 200, data)
+	} else {
+		data := make(map[string]interface{})
+		Sender(c, 400, data)
+	}
+}
+
+func OBStop(c *gin.Context) {
+	go observer.StopProcess()
+	data := make(map[string]interface{})
+	Sender(c, 200, data)
+}
+
+func OBStatus(c *gin.Context) {
+	data := make(map[string]interface{})
+	if observer.Liveness {
+		Sender(c, 200, data)
+	} else {
+		Sender(c, 400, data)
+	}
+}
+
+func OBReadiness(c *gin.Context) {
+	data := make(map[string]interface{})
+	if observer.Readiness {
+		Sender(c, 200, data)
+	} else {
+		Sender(c, 400, data)
+	}
+}
+
+func OBReadinessUpdate(c *gin.Context) {
+	data := make(map[string]interface{})
+	observer.Readiness = true
+	Sender(c, 200, data)
+}

pkg/cable/provider/dir.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package provider
+
+import (
+	"os/exec"
+)
+
+func DirInit() {
+	cmd := exec.Command("rm", "-rf", "/home/admin/oceanbase/log")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/log")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/log", "/home/admin/oceanbase/log")
+	cmd.Run()
+	cmd = exec.Command("rm", "-rf", "/home/admin/oceanbase/store")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/oceanbase/store")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/data_log/clog")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/data_log/clog", "/home/admin/oceanbase/store/clog")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/data_log/ilog")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/data_log/ilog", "/home/admin/oceanbase/store/ilog")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/data_log/slog")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/data_log/slog", "/home/admin/oceanbase/store/slog")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/data_file/sort_dir")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/data_file/sort_dir", "/home/admin/oceanbase/store/sort_dir")
+	cmd.Run()
+	cmd = exec.Command("mkdir", "-p", "/home/admin/data_file/sstable")
+	cmd.Run()
+	cmd = exec.Command("ln", "-sf", "/home/admin/data_file/sstable", "/home/admin/oceanbase/store/sstable")
+	cmd.Run()
+	cmd = exec.Command("chown", "-R", "admin:admin", "/home/admin/")
+	cmd.Run()
+}

pkg/cable/provider/http.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package provider
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+type Server struct {
+	Router     *gin.Engine
+	HttpServer *http.Server
+}
+
+var Tiny Server
+
+func (S *Server) Init() {
+	router := RouteCustom()
+	systemoperator := router.Group("/api/system")
+	{
+		// status for nic & ip
+		systemoperator.GET("/info", Info)
+		// paused
+		systemoperator.POST("/paused", Paused)
+		systemoperator.POST("/rework", Rework)
+	}
+	oboperator := router.Group("/api/ob")
+	{
+		// start
+		oboperator.POST("/start", OBStart)
+		// stop
+		oboperator.POST("/stop", OBStop)
+		// status
+		oboperator.GET("/status", OBStatus)
+		// readiness
+		oboperator.GET("/readiness", OBReadiness)
+		// readiness update
+		oboperator.POST("/readinessUpdate", OBReadinessUpdate)
+	}
+	S.Router = router
+	S.HttpServer = &http.Server{
+		Addr:    ":19001",
+		Handler: router,
+	}
+}
+
+func (S *Server) Run() {
+	err := S.HttpServer.ListenAndServe()
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+func (S *Server) Stop(ctx context.Context) {
+	S.HttpServer.Shutdown(ctx)
+}
+
+func RouteCustom() *gin.Engine {
+	gin.SetMode(gin.ReleaseMode)
+	router := gin.New()
+	router.Use(GinLogger())
+	router.Use(GinPanic())
+	return router
+}
+
+func GinLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		timeStart := time.Now()
+		c.Next()
+		timeEnd := time.Now()
+		clientIP := c.ClientIP()
+		statusCode := c.Writer.Status()
+		method := c.Request.Method
+		path := c.Request.URL.Path
+		latency := timeEnd.Sub(timeStart)
+		comment := c.Errors
+		log.Println(clientIP, statusCode, method, path, latency, comment)
+	}
+}
+
+func GinPanic() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Printf("[Error Info] %s", err)
+				// return
+				data := make(map[string]interface{})
+				Sender(c, 400, data)
+			}
+		}()
+		c.Next()
+	}
+}
+
+func Sender(c *gin.Context, code int, data interface{}) {
+	responseJSON, _ := json.Marshal(data)
+	c.Writer.Header().Set("Content-Type", "application/json")
+	c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+	c.String(code, string(responseJSON))
+}

pkg/cable/provider/k8s.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package provider
+
+import (
+	"context"
+
+	"github.com/oceanbase/ob-operator/pkg/cable/observer"
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func InitForK8s() {
+	DirInit()
+	// init http server
+	Tiny.Init()
+	// run http server
+	go Tiny.Run()
+	observer.Readiness = false
+	observer.OBStarted = false
+	util.FuncList = append(util.FuncList, StopForK8s)
+}
+
+func StopForK8s() {
+	Tiny.Stop(context.TODO())
+}

pkg/config/config.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package config
+
+var ClusterName string

pkg/controllers/controllers.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package controllers
+
+import (
+	"reflect"
+	"runtime"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp"
+)
+
+var controllerAddFuncs []func(manager.Manager) error
+
+func init() {
+	controllerAddFuncs = append(controllerAddFuncs, statefulapp.Add)
+	controllerAddFuncs = append(controllerAddFuncs, observer.Add)
+}
+
+// SetupWithManager load controller
+func SetupWithManager(m manager.Manager) error {
+	for _, f := range controllerAddFuncs {
+		klog.Infoln("load", runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
+		err := f(m)
+		if err != nil {
+			kindMatchErr, ok := err.(*meta.NoKindMatchError)
+			if ok {
+				klog.Errorf("CRD %v is not installed.", kindMatchErr.GroupKind)
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}

pkg/controllers/observer/cable/cable_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package cable
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func CableStatusCheck(subsets []cloudv1.SubsetStatus) error {
+	for _, subset := range subsets {
+		podList := subset.Pods
+		for _, pod := range podList {
+			podIP := pod.PodIP
+			err := CableStatusCheckExecuter(podIP)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func OBServerStart(obCluster cloudv1.OBCluster, subsets []cloudv1.SubsetStatus, rsList string) {
+	for _, subset := range subsets {
+		podList := subset.Pods
+		for _, pod := range podList {
+			obServerStartArgs := GenerateOBServerStartArgs(obCluster, subset.Name, rsList)
+			podIP := pod.PodIP
+			// check OBServer is already running, for OBServer Scale UP
+			err := OBServerStatusCheckExecuter(obCluster.ClusterName, podIP)
+			// nil is OBServer is already running
+			if err != nil {
+				OBServerStartExecuter(podIP, obServerStartArgs)
+			}
+		}
+	}
+}
+
+func OBServerStatusCheck(clusterName string, subsets []cloudv1.SubsetStatus) error {
+	for _, subset := range subsets {
+		podList := subset.Pods
+		for _, pod := range podList {
+			podIP := pod.PodIP
+			err := OBServerStatusCheckExecuter(clusterName, podIP)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func CableReadinessUpdate(subsets []cloudv1.SubsetStatus) error {
+	for _, subset := range subsets {
+		podList := subset.Pods
+		err := CableReadinessUpdateExecuter(podList[0].PodIP)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/controllers/observer/cable/cable_converter.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package cable
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/ob"
+)
+
+func GenerateRSListFromSubset(subsets []cloudv1.SubsetStatus) string {
+	var rsList string
+	for _, subset := range subsets {
+		podList := subset.Pods
+		if len(podList) > 0 {
+			podIP := podList[0].PodIP
+			if podIP != "" {
+				if rsList == "" {
+					rsList = fmt.Sprintf("%s:%s:%s", podIP, ob.OBSERVER_RPC_PORT, ob.OBSERVER_MYSQL_PORT)
+				} else {
+					rsList = fmt.Sprintf("%s,%s:%s:%s", rsList, podIP, ob.OBSERVER_RPC_PORT, ob.OBSERVER_MYSQL_PORT)
+				}
+			} else {
+				klog.Errorln("pod ip is empty", subsets)
+			}
+		}
+	}
+	return rsList
+}
+
+func GenerateRSListFromRootServiceStatus(topology []cloudv1.ClusterRootServiceStatus) string {
+	var rsList string
+	for _, cluster := range topology {
+		if cluster.Cluster == myconfig.ClusterName {
+			for _, zone := range cluster.Zone {
+				if zone.ServerIP != "" && zone.Status == observerconst.OBServerActive {
+					if rsList == "" {
+						rsList = fmt.Sprintf("%s:%s:%s", zone.ServerIP, ob.OBSERVER_RPC_PORT, ob.OBSERVER_MYSQL_PORT)
+					} else {
+						rsList = fmt.Sprintf("%s,%s:%s:%s", rsList, zone.ServerIP, ob.OBSERVER_RPC_PORT, ob.OBSERVER_MYSQL_PORT)
+					}
+				}
+			}
+		}
+		return rsList
+	}
+	return rsList
+}
+
+func GenerateOBServerStartArgs(obCluster cloudv1.OBCluster, zoneName, rsList string) map[string]interface{} {
+	obServerStartArgs := make(map[string]interface{})
+	obServerStartArgs["clusterName"] = obCluster.Name
+	obServerStartArgs["clusterId"] = obCluster.Spec.ClusterID
+	obServerStartArgs["zoneName"] = zoneName
+	obServerStartArgs["rsList"] = rsList
+	cpu, _ := obCluster.Spec.Resources.CPU.AsInt64()
+	obServerStartArgs["cpuLimit"] = cpu
+	memory, _ := obCluster.Spec.Resources.Memory.AsInt64()
+	obServerStartArgs["memoryLimit"] = memory / 1024 / 1024 / 1024
+	return obServerStartArgs
+}
+
+func GenerateOBClusterBootstrapArgs(subsets []cloudv1.SubsetStatus) (string, error) {
+	rsList := make([]ob.RSInfo, 0)
+	for _, subset := range subsets {
+		podList := subset.Pods
+		if len(podList) > 0 {
+			podIP := podList[0].PodIP
+			port, err := strconv.Atoi(ob.OBSERVER_RPC_PORT)
+			if err != nil {
+				klog.Errorln("the zone don't have first server", subset)
+				return "", errors.New("the zone don't have first server")
+			}
+			var rsInfo ob.RSInfo
+			rsInfo.Region = subset.Region
+			rsInfo.Zone = subset.Name
+			rsInfo.Server.Ip = podIP
+			rsInfo.Server.Port = port
+			rsList = append(rsList, rsInfo)
+		} else {
+			klog.Errorln("the zone don't have first server", subset)
+			return "", errors.New("the zone don't have first server")
+		}
+	}
+	var bootstrapParam ob.BootstrapParam
+	bootstrapParam.ClusterType = ob.PRIMARY
+	bootstrapParam.RSInfoList = rsList
+	obclusterBootstrapArgs := ob.GenerateBootstrapSQL(bootstrapParam)
+	klog.Infoln("OBCluster bootstrap args", obclusterBootstrapArgs)
+	return obclusterBootstrapArgs, nil
+}

pkg/controllers/observer/cable/cable_executer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package cable
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func CableStatusCheckExecuter(podIP string) error {
+	url := fmt.Sprintf("%s%s:%d%s", observerconst.CableUrlProfix, podIP, observerconst.CablePort, observerconst.CableInfoUrl)
+	code, _ := util.HTTPGET(url)
+	if code != 200 {
+		return errors.New("cable get ip failed")
+	}
+	return nil
+}
+
+func OBServerStartExecuter(podIP string, obServerStartArgs map[string]interface{}) {
+	url := fmt.Sprintf("%s%s:%d%s", observerconst.CableUrlProfix, podIP, observerconst.CablePort, observerconst.CableStartUrl)
+	code, _ := util.HTTPPOST(url, util.CovertToJSON(obServerStartArgs))
+	if code != 200 {
+		// TODO: need to check, why the OBServer start failed
+		klog.Errorln("start observer", podIP, "failed", util.CovertToJSON(obServerStartArgs))
+	} else {
+		klog.Infoln("start observer", podIP, "succeed", util.CovertToJSON(obServerStartArgs))
+	}
+}
+
+func OBServerStatusCheckExecuter(clusterName, podIP string) error {
+	url := fmt.Sprintf("%s%s:%d%s", observerconst.CableUrlProfix, podIP, observerconst.CablePort, observerconst.CableStatusUrl)
+	code, _ := util.HTTPGET(url)
+	if code != 200 {
+		klog.Errorln("OBCluster", clusterName, "observer", podIP, "starting not ready")
+		return errors.New("wait for OBCluster starting")
+	}
+	return nil
+}
+
+func CableReadinessUpdateExecuter(podIP string) error {
+	url := fmt.Sprintf("%s%s:%d%s", observerconst.CableUrlProfix, podIP, observerconst.CablePort, observerconst.CableReadinessUpdateUrl)
+	code, _ := util.HTTPPOST(url, "")
+	if code != 200 {
+		return errors.New("update cable readiness failed")
+	}
+	return nil
+}

pkg/controllers/observer/const/cable.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+const (
+	CableUrlProfix          = "http://"
+	CablePort               = 19001
+	CableInfoUrl            = "/api/system/info"
+	CableStartUrl           = "/api/ob/start"
+	CableStatusUrl          = "/api/ob/status"
+	CableReadinessUrl       = "/api/ob/readiness"
+	CableReadinessUpdateUrl = "/api/ob/readinessUpdate"
+)

pkg/controllers/observer/const/controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// controller
+const (
+	ControllerName       = "obcluster-controller"
+	ConcurrentReconciles = 1
+)

pkg/controllers/observer/const/error.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+const (
+	DataBaseError = "database error"
+)

pkg/controllers/observer/const/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// OBCluster Status
+const (
+	ClusterReady = "Ready"
+
+	TopologyPrepareing = "Prepareing"
+	TopologyNotReady   = "Not Ready"
+	TopologyReady      = "Ready"
+
+	ResourcePrepareing      = "Resource Prepareing"
+	ResourceReady           = "Resource Ready"
+	OBServerPrepareing      = "OBServer Prepareing"
+	OBServerReady           = "OBServer Ready"
+	OBClusterBootstraping   = "OBCluster Bootstraping"
+	OBClusterBootstrapReady = "OBCluster Bootstrap Ready"
+	OBClusterReady          = "OBCluster Ready"
+)

pkg/controllers/observer/const/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// OBServer Status
+const (
+	OBServerAdd = "Add OBServer"
+	OBServerDel = "Delete OBServer"
+)
+
+const (
+	OBServerActive   = "active"
+	OBServerInactive = "inactive"
+	OBServerDeleting = "deleting"
+)
+
+const (
+	RSJobStatusSuccess = "SUCCESS"
+)

pkg/controllers/observer/const/obzone.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// OBZone Status
+const (
+	OBZonePrepareing = "Prepareing"
+	OBZoneReady      = "Ready"
+	OBZoneOffline    = "Offline"
+)

pkg/controllers/observer/const/pipeline.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+import (
+	"time"
+)
+
+// scale state machine
+const (
+	ScaleUP   = "Scale UP"
+	ScaleDown = "Scale Down"
+	Maintain  = "Maintain"
+)
+
+// Step
+const (
+	StepBootstrap = "Bootstrap"
+	StepMaintain  = "Maintain"
+)
+
+const (
+	BootstrapTimeout = 600
+
+	AddServerTimeout                 = 60
+	TickPeriodForOBServerStatusCheck = 5 * time.Second
+	TickNumForOBServerStatusCheck    = 6
+
+	TickPeriodForRSJobStatusCheck = 5 * time.Second
+	TickNumForRSJobStatusCheck    = 12
+
+	DelServerTimeout = 30
+)

pkg/controllers/observer/const/service.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// Service
+const (
+	CablePortName = "cable"
+	MysqlPortName = "mysql"
+	MysqlPort     = 2881
+	RpcPortName   = "rpc"
+	RpcPort       = 2882
+)

pkg/controllers/observer/const/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observerconst
+
+// StatefulApp
+const (
+	ImgProfix     = "oceanbase/oceanbase-ce:operator"
+	ImgOb         = "observer"
+	ImgPullPolicy = "Always"
+
+	DatafileStorageName = "data-file"
+	DatafileStoragePath = "/home/admin/data_file"
+	DatalogStorageName  = "data-log"
+	DatalogStoragePath  = "/home/admin/data_log"
+	LogStorageName      = "log"
+	LogStoragePath      = "/home/admin/log"
+
+	CableReadinessPeriod = 2
+)

pkg/controllers/observer/core/converter/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+)
+
+func GetClusterSpecFromOBTopology(topology []cloudv1.Cluster) cloudv1.Cluster {
+	var cluster cloudv1.Cluster
+	for _, cluster = range topology {
+		if cluster.Cluster == myconfig.ClusterName {
+			break
+		}
+	}
+	return cluster
+}
+
+func GetClusterStatusFromOBTopologyStatus(clusterStatus []cloudv1.ClusterStatus) cloudv1.ClusterStatus {
+	var cluster cloudv1.ClusterStatus
+	for _, cluster = range clusterStatus {
+		if cluster.Cluster == myconfig.ClusterName {
+			break
+		}
+	}
+	return cluster
+}

pkg/controllers/observer/core/converter/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"github.com/pkg/errors"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	statefulappCore "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+)
+
+func IsAllOBServerActive(obServerList []model.AllServer, obClusters []cloudv1.Cluster) bool {
+	obServerCurrentReplicas := make(map[string]bool)
+	for _, obServer := range obServerList {
+		if obServer.Status == observerconst.OBServerActive {
+			obServerCurrentReplicas[obServer.Zone] = true
+		}
+	}
+	for _, obCluster := range obClusters {
+		if obCluster.Cluster == myconfig.ClusterName {
+			for _, zone := range obCluster.Zone {
+				tmp := obServerCurrentReplicas[zone.Name]
+				if !tmp {
+					return false
+				}
+			}
+			return true
+		}
+	}
+	return false
+}
+
+func IsOBServerDeleted(clusterIP, podIP string) bool {
+	obServerList := sql.GetOBServer(clusterIP)
+	for _, obServer := range obServerList {
+		if obServer.SvrIP == podIP {
+			return false
+		}
+	}
+	return true
+}
+
+func IsPodNotInOBServerList(zoneName, ip string, nodeMap map[string][]cloudv1.OBNode) bool {
+	zoneIPList := nodeMap[zoneName]
+	if len(zoneIPList) > 0 {
+		for _, tmpIP := range zoneIPList {
+			if tmpIP.ServerIP == ip {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func IsOBServerInactiveOrDeletingAndNotInPodList(server cloudv1.OBNode, podRunningList []string) bool {
+	if server.Status == observerconst.OBServerInactive || server.Status == observerconst.OBServerDeleting {
+		for _, podIP := range podRunningList {
+			if podIP == server.ServerIP {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func GetInfoForAddServerByZone(clusterIP string, statefulApp cloudv1.StatefulApp) (error, string, string) {
+	obServerList := sql.GetOBServer(clusterIP)
+	if len(obServerList) == 0 {
+		return errors.New(observerconst.DataBaseError), "", ""
+	}
+
+	nodeMap := GenerateNodeMapByOBServerList(obServerList)
+
+	// judge witch ip need add
+	for _, subset := range statefulApp.Status.Subsets {
+		for _, pod := range subset.Pods {
+			if pod.PodPhase == statefulappCore.PodStatusRunning {
+				status := IsPodNotInOBServerList(subset.Name, pod.PodIP, nodeMap)
+				// Pod IP not in OBServerList, need to add server
+				// do one thing at a time
+				if status {
+					return nil, subset.Name, pod.PodIP
+				}
+			}
+		}
+	}
+
+	return errors.New("none ip need add"), "", ""
+}
+
+func GetInfoForDelServerByZone(clusterIP string, clusterSpec cloudv1.Cluster, statefulApp cloudv1.StatefulApp) (error, string, string) {
+	obServerList := sql.GetOBServer(clusterIP)
+	if len(obServerList) == 0 {
+		return errors.New(observerconst.DataBaseError), "", ""
+	}
+
+	nodeMap := GenerateNodeMapByOBServerList(obServerList)
+
+	// judge witch ip need del
+	for _, subset := range statefulApp.Status.Subsets {
+		runningPodList := getRunningPodListFromSubsetStatus(subset)
+		zoneSpec := GetZoneSpecFromClusterSpec(subset.Name, clusterSpec)
+		// number of observer in db > replica
+		if len(nodeMap[subset.Name]) > int(zoneSpec.Replicas) {
+			for _, pod := range nodeMap[subset.Name] {
+				status := IsOBServerInactiveOrDeletingAndNotInPodList(pod, runningPodList)
+				if status {
+					// OBServer IP not in PodList, need to delete server
+					// do one thing at a time
+					return nil, subset.Name, pod.ServerIP
+				}
+			}
+		}
+	}
+
+	return errors.New("none ip need del"), "", ""
+}
+
+func getRunningPodListFromSubsetStatus(subset cloudv1.SubsetStatus) []string {
+	runningPodList := make([]string, 0)
+	for _, pod := range subset.Pods {
+		if pod.PodPhase == statefulappCore.PodStatusRunning {
+			runningPodList = append(runningPodList, pod.PodIP)
+		}
+	}
+	return runningPodList
+}

pkg/controllers/observer/core/converter/obzone.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+	"sort"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerutil "github.com/oceanbase/ob-operator/pkg/controllers/observer/core/util"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func GenerateOBZoneName(name string) string {
+	statefulAppName := fmt.Sprintf("obzone-%s", name)
+	return statefulAppName
+}
+
+func GenerateOBZoneSpec(obCluster cloudv1.OBCluster) cloudv1.OBZoneSpec {
+	spec := cloudv1.OBZoneSpec{
+		Topology: obCluster.Spec.Topology,
+	}
+	return spec
+}
+
+func GenerateOBZoneObject(obCluster cloudv1.OBCluster) cloudv1.OBZone {
+	name := GenerateOBZoneName(obCluster.Name)
+	objectMeta := observerutil.GenerateObjectMeta(obCluster, name)
+	spec := GenerateOBZoneSpec(obCluster)
+	obZone := cloudv1.OBZone{
+		ObjectMeta: objectMeta,
+		Spec:       spec,
+	}
+	return obZone
+}
+
+func GenerateOBZoneInfoListByCluster(cluster cloudv1.Cluster, nodeMap map[string][]cloudv1.OBNode) []cloudv1.OBZoneInfo {
+	zoneList := make([]cloudv1.OBZoneInfo, 0)
+	for _, zone := range cluster.Zone {
+		v := nodeMap[zone.Name]
+		var zoneTemp cloudv1.OBZoneInfo
+		zoneTemp.Name = zone.Name
+		zoneTemp.Nodes = v
+		zoneList = append(zoneList, zoneTemp)
+	}
+	return zoneList
+}
+
+func GenerateMultiClusterOBZoneStatus(obZoneCurrent cloudv1.OBZone, clusterOBZoneStatus cloudv1.ClusterOBZoneStatus) []cloudv1.ClusterOBZoneStatus {
+	obZoneTopologyStatus := make([]cloudv1.ClusterOBZoneStatus, 0)
+	if len(obZoneCurrent.Status.Topology) > 0 {
+		for _, otherClusterStatus := range obZoneCurrent.Status.Topology {
+			if otherClusterStatus.Cluster != myconfig.ClusterName {
+				obZoneTopologyStatus = append(obZoneTopologyStatus, otherClusterStatus)
+			}
+		}
+	}
+	obZoneTopologyStatus = append(obZoneTopologyStatus, clusterOBZoneStatus)
+	return obZoneTopologyStatus
+}
+
+func GenerateNodeMapByOBServerList(obServerList []model.AllServer) map[string][]cloudv1.OBNode {
+	nodeMap := make(map[string][]cloudv1.OBNode, 0)
+	for _, server := range obServerList {
+		nodes := nodeMap[server.Zone]
+		var node cloudv1.OBNode
+		node.ServerIP = server.SvrIP
+		node.Status = server.Status
+		nodes = append(nodes, node)
+		tmp := SortNodesStatus(nodes)
+		nodeMap[server.Zone] = tmp
+	}
+	return nodeMap
+}
+
+func UpdateOBZoneSpec(obzone cloudv1.OBZone, topology []cloudv1.Cluster) cloudv1.OBZone {
+	obzone.Spec.Topology = topology
+	return obzone
+}
+
+func OBServerListToOBZoneStatus(cluster cloudv1.Cluster, obZoneCurrent cloudv1.OBZone, obServerList []model.AllServer) cloudv1.OBZone {
+	nodeMap := GenerateNodeMapByOBServerList(obServerList)
+
+	zoneList := GenerateOBZoneInfoListByCluster(cluster, nodeMap)
+
+	var clusterOBZoneStatus cloudv1.ClusterOBZoneStatus
+	clusterOBZoneStatus.Cluster = myconfig.ClusterName
+	clusterOBZoneStatus.Zone = zoneList
+
+	// multi cluster
+	obZoneTopologyStatus := GenerateMultiClusterOBZoneStatus(obZoneCurrent, clusterOBZoneStatus)
+
+	obZoneCurrent.Status.Topology = obZoneTopologyStatus
+	return obZoneCurrent
+}
+
+func SortNodesStatus(nodesStatus []cloudv1.OBNode) []cloudv1.OBNode {
+	sort.Slice(nodesStatus, func(i, j int) bool {
+		if nodesStatus[i].ServerIP < nodesStatus[j].ServerIP {
+			return true
+		}
+		return false
+	})
+	return nodesStatus
+}
+
+func GetZoneSpecFromClusterSpec(zoneName string, clusterSpec cloudv1.Cluster) cloudv1.Subset {
+	var res cloudv1.Subset
+	for _, zone := range clusterSpec.Zone {
+		if zone.Name == zoneName {
+			res = zone
+			break
+		}
+	}
+	return res
+}

pkg/controllers/observer/core/converter/rootservice.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerutil "github.com/oceanbase/ob-operator/pkg/controllers/observer/core/util"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func GenerateRootServiceName(name string) string {
+	statefulAppName := fmt.Sprintf("rs-%s", name)
+	return statefulAppName
+}
+
+func GenerateRootServiceSpec(obCluster cloudv1.OBCluster) cloudv1.RootServiceSpec {
+	spec := cloudv1.RootServiceSpec{
+		Topology: obCluster.Spec.Topology,
+	}
+	return spec
+}
+
+func GenerateRootServiceObject(obCluster cloudv1.OBCluster) cloudv1.RootService {
+	name := GenerateRootServiceName(obCluster.Name)
+	objectMeta := observerutil.GenerateObjectMeta(obCluster, name)
+	spec := GenerateRootServiceSpec(obCluster)
+	rootService := cloudv1.RootService{
+		ObjectMeta: objectMeta,
+		Spec:       spec,
+	}
+	return rootService
+}
+
+func GenerateZoneRSList(cluster cloudv1.Cluster, rsList []model.AllVirtualCoreMeta, obServerList []model.AllServer) []cloudv1.ZoneRootServiceStatus {
+	zoneRSList := make([]cloudv1.ZoneRootServiceStatus, 0)
+	for _, zone := range cluster.Zone {
+		zoneRS := GenerateZoneRootServiceStatusByRSList(zone.Name, rsList, obServerList)
+		zoneRSList = append(zoneRSList, zoneRS)
+	}
+	return zoneRSList
+}
+
+func GenerateZoneRootServiceStatusByRSList(zoneName string, rsList []model.AllVirtualCoreMeta, obServerList []model.AllServer) cloudv1.ZoneRootServiceStatus {
+	var zrs cloudv1.ZoneRootServiceStatus
+	for _, rs := range rsList {
+		if rs.Zone == zoneName {
+			zrs.Name = rs.Zone
+			zrs.ServerIP = rs.SvrIP
+			zrs.Role = int(rs.Role)
+			for _, server := range obServerList {
+				if rs.SvrIP == server.SvrIP {
+					zrs.Status = server.Status
+					break
+				}
+			}
+			break
+		}
+	}
+	return zrs
+}
+
+func GenerateMultiClusterRootServiceStatus(rootService cloudv1.RootService, rsCurrentStatus cloudv1.ClusterRootServiceStatus) []cloudv1.ClusterRootServiceStatus {
+	rsTopologyStatus := make([]cloudv1.ClusterRootServiceStatus, 0)
+	if len(rootService.Status.Topology) > 0 {
+		for _, otherClusterStatus := range rootService.Status.Topology {
+			if otherClusterStatus.Cluster != myconfig.ClusterName {
+				rsTopologyStatus = append(rsTopologyStatus, otherClusterStatus)
+			}
+		}
+	}
+	rsTopologyStatus = append(rsTopologyStatus, rsCurrentStatus)
+	return rsTopologyStatus
+}
+
+func RSListToRSStatus(cluster cloudv1.Cluster, rootService cloudv1.RootService, rsList []model.AllVirtualCoreMeta, obServerList []model.AllServer) cloudv1.RootService {
+	zrsList := GenerateZoneRSList(cluster, rsList, obServerList)
+
+	var rsCurrentStatus cloudv1.ClusterRootServiceStatus
+	rsCurrentStatus.Cluster = myconfig.ClusterName
+	rsCurrentStatus.Zone = zrsList
+
+	// multi cluster
+	rsTopologyStatus := GenerateMultiClusterRootServiceStatus(rootService, rsCurrentStatus)
+
+	rootService.Status.Topology = rsTopologyStatus
+	return rootService
+}

pkg/controllers/observer/core/converter/rootservice_job.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"github.com/pkg/errors"
+
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+)
+
+func IsRSJobSuccess(clusterIP, podIP string) (bool, error) {
+	rsJobStatusList := sql.GetRSJobStatus(clusterIP, podIP)
+	if len(rsJobStatusList) == 0 {
+		return false, errors.New("get rs job status faild")
+	}
+	lastJob := rsJobStatusList[len(rsJobStatusList)-1]
+	// job status is not SUCCESS
+	if lastJob.JobStatus != observerconst.RSJobStatusSuccess {
+		return false, nil
+	}
+	return true, nil
+}

pkg/controllers/observer/core/converter/service.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	observerutil "github.com/oceanbase/ob-operator/pkg/controllers/observer/core/util"
+)
+
+func GenerateServiceName(name string) string {
+	statefulAppName := fmt.Sprintf("svc-%s", name)
+	return statefulAppName
+}
+
+func GenerateServiceSpec(statefulAppName string) corev1.ServiceSpec {
+	ports := make([]corev1.ServicePort, 0)
+	var servicePort corev1.ServicePort
+	servicePort.Name = observerconst.MysqlPortName
+	servicePort.Port = observerconst.MysqlPort
+	servicePort.TargetPort = intstr.FromInt(observerconst.MysqlPort)
+	servicePort.Protocol = corev1.ProtocolTCP
+	ports = append(ports, servicePort)
+
+	selector := make(map[string]string)
+	selector["app"] = statefulAppName
+
+	var res corev1.ServiceSpec
+	res.Ports = ports
+	res.Selector = selector
+	res.Type = corev1.ServiceTypeNodePort
+	return res
+}
+
+func GenerateServiceObject(obCluster cloudv1.OBCluster, statefulAppName string) corev1.Service {
+	name := GenerateServiceName(obCluster.Name)
+	objectMeta := observerutil.GenerateObjectMeta(obCluster, name)
+	serviceSpec := GenerateServiceSpec(statefulAppName)
+	service := corev1.Service{
+		ObjectMeta: objectMeta,
+		Spec:       serviceSpec,
+	}
+	return service
+}

pkg/controllers/observer/core/converter/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	observerutil "github.com/oceanbase/ob-operator/pkg/controllers/observer/core/util"
+	statefulappCore "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+)
+
+func GenerateStatefulAppName(name string) string {
+	statefulAppName := fmt.Sprintf("sapp-%s", name)
+	return statefulAppName
+}
+
+func GenerateStatefulAppBootStrapZoneSpec(zone []cloudv1.Subset) []cloudv1.Subset {
+	res := make([]cloudv1.Subset, 0)
+	for _, subset := range zone {
+		tmp := subset
+		tmp.Replicas = 1
+		res = append(res, tmp)
+	}
+	return res
+}
+
+func GenerateImage(version string) string {
+	image := fmt.Sprintf("%s-%s", observerconst.ImgProfix, version)
+	return image
+}
+
+func GeneratePodSpec(obClusterSpec cloudv1.OBClusterSpec) corev1.PodSpec {
+	port := make([]corev1.ContainerPort, 0)
+	cablePort := corev1.ContainerPort{}
+	cablePort.Name = observerconst.CablePortName
+	cablePort.ContainerPort = int32(observerconst.CablePort)
+	cablePort.Protocol = corev1.ProtocolTCP
+	port = append(port, cablePort)
+	mysqlPort := corev1.ContainerPort{}
+	mysqlPort.Name = observerconst.MysqlPortName
+	mysqlPort.ContainerPort = observerconst.MysqlPort
+	mysqlPort.Protocol = corev1.ProtocolTCP
+	port = append(port, mysqlPort)
+	rpcPort := corev1.ContainerPort{}
+	rpcPort.Name = observerconst.RpcPortName
+	rpcPort.ContainerPort = observerconst.RpcPort
+	rpcPort.Protocol = corev1.ProtocolTCP
+	port = append(port, rpcPort)
+
+	requestsResources := corev1.ResourceList{}
+	requestsResources["cpu"] = obClusterSpec.Resources.CPU
+	requestsResources["memory"] = obClusterSpec.Resources.Memory
+	limitResources := corev1.ResourceList{}
+	limitResources["cpu"] = obClusterSpec.Resources.CPU
+	limitResources["memory"] = obClusterSpec.Resources.Memory
+	resources := corev1.ResourceRequirements{
+		Requests: requestsResources,
+		Limits:   limitResources,
+	}
+
+	volumeMountDataFile := corev1.VolumeMount{}
+	volumeMountDataFile.Name = observerconst.DatafileStorageName
+	volumeMountDataFile.MountPath = observerconst.DatafileStoragePath
+	volumeMountDataLog := corev1.VolumeMount{}
+	volumeMountDataLog.Name = observerconst.DatalogStorageName
+	volumeMountDataLog.MountPath = observerconst.DatalogStoragePath
+	volumeMountLog := corev1.VolumeMount{}
+	volumeMountLog.Name = observerconst.LogStorageName
+	volumeMountLog.MountPath = observerconst.LogStoragePath
+	volumeMounts := make([]corev1.VolumeMount, 0)
+	volumeMounts = append(volumeMounts, volumeMountDataFile)
+	volumeMounts = append(volumeMounts, volumeMountDataLog)
+	volumeMounts = append(volumeMounts, volumeMountLog)
+
+	readinessProbeHTTP := corev1.HTTPGetAction{}
+	readinessProbeHTTP.Port = intstr.FromInt(observerconst.CablePort)
+	readinessProbeHTTP.Path = observerconst.CableReadinessUrl
+	readinessProbe := corev1.Probe{}
+	readinessProbe.Handler.HTTPGet = &readinessProbeHTTP
+	readinessProbe.PeriodSeconds = observerconst.CableReadinessPeriod
+
+	container := corev1.Container{
+		Name:            observerconst.ImgOb,
+		Image:           GenerateImage(obClusterSpec.Version),
+		ImagePullPolicy: observerconst.ImgPullPolicy,
+		Ports:           port,
+		Resources:       resources,
+		VolumeMounts:    volumeMounts,
+		ReadinessProbe:  &readinessProbe,
+	}
+	containers := make([]corev1.Container, 0)
+	containers = append(containers, container)
+
+	volumeDataFile := corev1.Volume{}
+	volumeDataFile.Name = observerconst.DatafileStorageName
+	volumeDataFileSource := &corev1.PersistentVolumeClaimVolumeSource{
+		ClaimName: observerconst.DatafileStorageName,
+	}
+	volumeDataFile.VolumeSource.PersistentVolumeClaim = volumeDataFileSource
+	volumeDataLog := corev1.Volume{}
+	volumeDataLog.Name = observerconst.DatalogStorageName
+	volumeDataLogSource := &corev1.PersistentVolumeClaimVolumeSource{
+		ClaimName: observerconst.DatalogStorageName,
+	}
+	volumeDataLog.VolumeSource.PersistentVolumeClaim = volumeDataLogSource
+	volumeLog := corev1.Volume{}
+	volumeLog.Name = observerconst.LogStorageName
+	volumeLogSource := &corev1.PersistentVolumeClaimVolumeSource{
+		ClaimName: observerconst.LogStorageName,
+	}
+	volumeLog.VolumeSource.PersistentVolumeClaim = volumeLogSource
+	volumes := make([]corev1.Volume, 0)
+	volumes = append(volumes, volumeDataFile)
+	volumes = append(volumes, volumeDataLog)
+	volumes = append(volumes, volumeLog)
+
+	podSpec := corev1.PodSpec{
+		Containers: containers,
+		Volumes:    volumes,
+	}
+	return podSpec
+}
+
+func GenerateStorageSpec(obClusterSpec cloudv1.OBClusterSpec) []cloudv1.StorageTemplate {
+	storageTemplates := make([]cloudv1.StorageTemplate, 0)
+	for _, storageSpec := range obClusterSpec.Resources.Storage {
+		storageTemplate := cloudv1.StorageTemplate{}
+		storageTemplate.Name = storageSpec.Name
+		requestsResources := corev1.ResourceList{}
+		requestsResources["storage"] = storageSpec.Size
+		storageTemplate.PVC.StorageClassName = &(storageSpec.StorageClassName)
+		accessModes := make([]corev1.PersistentVolumeAccessMode, 0)
+		accessModes = append(accessModes, corev1.ReadWriteOnce)
+		storageTemplate.PVC.AccessModes = accessModes
+		storageTemplate.PVC.Resources.Requests = requestsResources
+		storageTemplates = append(storageTemplates, storageTemplate)
+	}
+	return storageTemplates
+}
+
+func GenerateStatefulAppSpec(cluster cloudv1.Cluster, obCluster cloudv1.OBCluster) cloudv1.StatefulAppSpec {
+	subset := GenerateStatefulAppBootStrapZoneSpec(cluster.Zone)
+	podSpec := GeneratePodSpec(obCluster.Spec)
+	storage := GenerateStorageSpec(obCluster.Spec)
+	spec := cloudv1.StatefulAppSpec{
+		Cluster:          cluster.Cluster,
+		Subsets:          subset,
+		PodTemplate:      podSpec,
+		StorageTemplates: storage,
+	}
+	return spec
+}
+
+func GenerateStatefulAppObject(cluster cloudv1.Cluster, obCluster cloudv1.OBCluster) cloudv1.StatefulApp {
+	name := GenerateStatefulAppName(obCluster.Name)
+	objectMeta := observerutil.GenerateObjectMeta(obCluster, name)
+	spec := GenerateStatefulAppSpec(cluster, obCluster)
+	statefulApp := cloudv1.StatefulApp{
+		ObjectMeta: objectMeta,
+		Spec:       spec,
+	}
+	return statefulApp
+}
+
+func GetSubsetStatusFromStatefulApp(zoneName string, statefulApp cloudv1.StatefulApp) cloudv1.SubsetStatus {
+	var res cloudv1.SubsetStatus
+	for _, subset := range statefulApp.Status.Subsets {
+		if subset.Name == zoneName {
+			res = subset
+			break
+		}
+	}
+	return res
+}
+
+func UpdateSubsetReplicaForStatefulApp(subset cloudv1.Subset, statefulApp cloudv1.StatefulApp) cloudv1.StatefulApp {
+	zoneList := make([]cloudv1.Subset, 0)
+	for _, zone := range statefulApp.Spec.Subsets {
+		if zone.Name == subset.Name {
+			// one by one
+			if zone.Replicas < subset.Replicas {
+				zone.Replicas = zone.Replicas + 1
+			}
+			if zone.Replicas > subset.Replicas {
+				zone.Replicas = zone.Replicas - 1
+			}
+		}
+		zoneList = append(zoneList, zone)
+	}
+	statefulApp.Spec.Subsets = zoneList
+	return statefulApp
+}
+
+func CheckStatefulAppStatus(statefulApp cloudv1.StatefulApp) bool {
+	if statefulApp.Status.ClusterStatus != statefulappCore.Ready {
+		return false
+	}
+	return true
+}

pkg/controllers/observer/core/event/event.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package event

pkg/controllers/observer/core/judge/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package judge
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+)
+
+func VersionIsModified(version string, statefulApp cloudv1.StatefulApp) (bool, error) {
+	newVersion := fmt.Sprintf("operator-%s", version)
+	var versionCurrent string
+	for _, container := range statefulApp.Spec.PodTemplate.Containers {
+		if container.Name == observerconst.ImgOb {
+			versionCurrent = strings.Split(container.Image, ":")[len(strings.Split(container.Image, ":"))-1]
+			break
+		}
+	}
+	if newVersion == versionCurrent {
+		return false, nil
+	}
+	return true, nil
+}
+
+func ResourcesIsModified(clusterList []cloudv1.Cluster, obCluster cloudv1.OBCluster, statefulApp cloudv1.StatefulApp) (bool, error) {
+	cluster := converter.GetClusterSpecFromOBTopology(clusterList)
+	statefulAppNew := converter.GenerateStatefulAppObject(cluster, obCluster)
+	podTemplateCompareStatus := reflect.DeepEqual(statefulApp.Spec.PodTemplate, statefulAppNew.Spec.PodTemplate)
+	storageTemplatesCompareStatus := reflect.DeepEqual(statefulApp.Spec.StorageTemplates, statefulAppNew.Spec.StorageTemplates)
+	if podTemplateCompareStatus && storageTemplatesCompareStatus {
+		return false, nil
+	}
+	return true, nil
+}
+
+func ZoneNumberIsModified(clusterList []cloudv1.Cluster, statefulApp cloudv1.StatefulApp) (string, error) {
+	cluster := converter.GetClusterSpecFromOBTopology(clusterList)
+	zoneNumberNew := len(cluster.Zone)
+	if zoneNumberNew == 0 {
+		return observerconst.Maintain, kubeerrors.NewServiceUnavailable("can't scale Zone to zero")
+	}
+	zoneNumberCurrent := len(statefulApp.Spec.Subsets)
+	if zoneNumberNew > zoneNumberCurrent {
+		return observerconst.ScaleUP, nil
+	} else if zoneNumberNew < zoneNumberCurrent {
+		return observerconst.ScaleDown, nil
+	} else {
+		return observerconst.Maintain, nil
+	}
+}

pkg/controllers/observer/core/judge/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package judge
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+)
+
+func OBServerScale(clusterList []cloudv1.Cluster, statefulApp cloudv1.StatefulApp) (string, cloudv1.Subset) {
+	var res cloudv1.Subset
+	var scaleState string
+	cluster := converter.GetClusterSpecFromOBTopology(clusterList)
+	for _, zone := range cluster.Zone {
+		subsetStatus := converter.GetSubsetStatusFromStatefulApp(zone.Name, statefulApp)
+		if zone.Replicas > subsetStatus.ExpectedReplicas {
+			scaleState = observerconst.ScaleUP
+			res = zone
+			break
+		} else if zone.Replicas < subsetStatus.ExpectedReplicas {
+			scaleState = observerconst.ScaleDown
+			res = zone
+			break
+		} else {
+			scaleState = observerconst.Maintain
+		}
+	}
+	return scaleState, res
+}

pkg/controllers/observer/core/obcluster_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/judge"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+// OBClusterReconciler reconciles a OBCluster object
+type OBClusterReconciler struct {
+	CRClient client.Client
+	Scheme   *runtime.Scheme
+	// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/events/event.go
+	Recorder record.EventRecorder
+}
+
+type OBClusterCtrl struct {
+	Resource  *resource.Resource
+	OBCluster cloudv1.OBCluster
+}
+
+type OBClusterCtrlOperator interface {
+	OBClusterCoordinator() (ctrl.Result, error)
+}
+
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=obclusters,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=obclusters/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=obclusters/finalizers,verbs=update
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps/finalizers,verbs=update
+// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
+
+func (r *OBClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Fetch the CR instance
+	instance := &cloudv1.OBCluster{}
+	err := r.CRClient.Get(ctx, req.NamespacedName, instance)
+	if err != nil {
+		if kubeerrors.IsNotFound(err) {
+			// Object not found, return.
+			// Created objects are automatically garbage collected.
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object, requeue the request.
+		return reconcile.Result{}, err
+	}
+	// custom logic
+	obClusterCtrl := NewOBServerCtrl(r.CRClient, r.Recorder, *instance)
+	return obClusterCtrl.OBClusterCoordinator()
+}
+
+func NewOBServerCtrl(client client.Client, recorder record.EventRecorder, obCluster cloudv1.OBCluster) OBClusterCtrlOperator {
+	ctrlResource := resource.NewResource(client, recorder)
+	return &OBClusterCtrl{
+		Resource:  ctrlResource,
+		OBCluster: obCluster,
+	}
+}
+
+func (ctrl *OBClusterCtrl) OBClusterCoordinator() (ctrl.Result, error) {
+	var newClusterStatus bool
+	statefulApp := &cloudv1.StatefulApp{}
+	statefulApp, newClusterStatus = ctrl.IsNewCluster(*statefulApp)
+
+	// is new cluster
+	if newClusterStatus {
+		err := ctrl.NewCluster(*statefulApp)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+	}
+
+	// OBCluster control-plan
+	err := ctrl.OBClusterEffector(*statefulApp)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, nil
+}
+
+func (ctrl *OBClusterCtrl) OBClusterEffector(statefulApp cloudv1.StatefulApp) error {
+	var err error
+	obClusterStatus := ctrl.OBCluster.Status.Status
+	switch obClusterStatus {
+	case observerconst.TopologyPrepareing:
+		// OBCluster is not ready
+		err = ctrl.TopologyPrepareingEffector(statefulApp)
+	case observerconst.TopologyNotReady:
+		// OBCluster is not ready
+		err = ctrl.TopologyNotReadyEffector(statefulApp)
+	case observerconst.TopologyReady:
+		// OBCluster is ready
+		err = ctrl.TopologyReadyEffector(statefulApp)
+	}
+	return err
+}
+
+func (ctrl *OBClusterCtrl) TopologyPrepareingEffector(statefulApp cloudv1.StatefulApp) error {
+	var err error
+	for _, clusterStatus := range ctrl.OBCluster.Status.Topology {
+		if clusterStatus.Cluster == myconfig.ClusterName {
+			switch clusterStatus.ClusterStatus {
+			case observerconst.ResourcePrepareing:
+				// StatefulApp is creating
+				err = ctrl.ResourcePrepareingEffectorForBootstrap(statefulApp)
+			case observerconst.ResourceReady:
+				// StatefulApp is ready
+				err = ctrl.ResourceReadyEffectorForBootstrap(statefulApp)
+			case observerconst.OBServerPrepareing:
+				// OBServer is staring
+				err = ctrl.OBServerPrepareingEffectorForBootstrap(statefulApp)
+			case observerconst.OBServerReady:
+				// OBServer is running
+				err = ctrl.OBServerReadyEffectorForBootstrap(statefulApp)
+			case observerconst.OBClusterBootstraping:
+				// OBCluster Bootstraping
+				err = ctrl.OBClusterBootstraping(statefulApp)
+			case observerconst.OBClusterBootstrapReady:
+				// OBCluster Bootstrap ready
+				err = ctrl.OBClusterBootstrapReady(statefulApp)
+			case observerconst.OBClusterReady:
+				// OBCluster bootstrap succeeded
+				err = ctrl.OBClusterReadyForStep(observerconst.StepBootstrap, statefulApp)
+			}
+		}
+	}
+	return err
+}
+
+func (ctrl *OBClusterCtrl) TopologyNotReadyEffector(statefulApp cloudv1.StatefulApp) error {
+	var err error
+	for _, clusterStatus := range ctrl.OBCluster.Status.Topology {
+		if clusterStatus.Cluster == myconfig.ClusterName {
+			switch clusterStatus.ClusterStatus {
+			case observerconst.ScaleUP:
+				// OBServer Scale UP
+				err = ctrl.OBServerScaleUPByZone(statefulApp)
+			case observerconst.ScaleDown:
+				// OBServer Scale Down
+				err = ctrl.OBServerScaleDownByZone(statefulApp)
+			}
+		}
+	}
+	return err
+}
+
+func (ctrl *OBClusterCtrl) TopologyReadyEffector(statefulApp cloudv1.StatefulApp) error {
+	// check version update
+	versionIsModified, err := judge.VersionIsModified(ctrl.OBCluster.Spec.Version, statefulApp)
+	if err != nil {
+		return err
+	}
+	if versionIsModified {
+		// TODO: support version update
+		klog.Errorln("version update is not supported yet")
+		return nil
+	}
+
+	// check resource modified
+	resourcesIsModified, err := judge.ResourcesIsModified(ctrl.OBCluster.Spec.Topology, ctrl.OBCluster, statefulApp)
+	if err != nil {
+		return err
+	}
+	if resourcesIsModified {
+		// TODO: support resource change
+		klog.Errorln("resource changes is not supported yet")
+		return nil
+	}
+
+	// check zone number modified
+	zoneScaleStatus, err := judge.ZoneNumberIsModified(ctrl.OBCluster.Spec.Topology, statefulApp)
+	if err != nil {
+		return err
+	}
+	switch zoneScaleStatus {
+	case observerconst.ScaleUP:
+		// TODO: support Zone Scale UP
+		klog.Errorln("Zone scale up is not supported yet")
+	case observerconst.ScaleDown:
+		// TODO: support Zone Scale Down
+		klog.Errorln("Zone scale down is not supported yet")
+	case observerconst.Maintain:
+		err = ctrl.OBServerCoordinator(statefulApp)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/controllers/observer/core/obcluster_status.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+	"reflect"
+	"runtime"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	observerutil "github.com/oceanbase/ob-operator/pkg/controllers/observer/core/util"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+func (ctrl *OBClusterCtrl) OBClusterReadyForStep(step string, statefulApp cloudv1.StatefulApp) error {
+	// update RootService
+	err := ctrl.UpdateRootServiceStatus(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	// update OBZone
+	err = ctrl.UpdateOBZoneStatus(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	// create service
+	switch step {
+	case observerconst.StepBootstrap:
+		err = ctrl.CreateService(statefulApp.Name)
+		if err != nil {
+			return err
+		}
+	case observerconst.StepMaintain:
+		_, err = ctrl.GetServiceByName(ctrl.OBCluster.Namespace, ctrl.OBCluster.Name)
+		if err != nil {
+			err = ctrl.CreateService(statefulApp.Name)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// update status
+	return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, "", "")
+}
+
+func (ctrl *OBClusterCtrl) UpdateOBClusterAndZoneStatus(clusterStatus, zoneName, zoneStatus string) error {
+	var compareStatus bool
+	obCluster := ctrl.OBCluster
+	obClusterExecuter := resource.NewOBClusterResource(ctrl.Resource)
+	// use retry to update
+	retryErr := retry.RetryOnConflict(
+		retry.DefaultRetry,
+		func() error {
+			// get current OBCluster every time
+			obClusterTemp, err := obClusterExecuter.Get(context.TODO(), obCluster.Namespace, obCluster.Name)
+			if err != nil {
+				return err
+			}
+			// DeepCopy
+			obClusterCurrent := obClusterTemp.(cloudv1.OBCluster)
+			obClusterCurrentDeepCopy := obClusterCurrent.DeepCopy()
+			// assign a value
+			ctrl.OBCluster = *obClusterCurrentDeepCopy
+			// build status
+			obClusterNew, err := ctrl.buildOBClusterStatus(*obClusterCurrentDeepCopy, clusterStatus, zoneName, zoneStatus)
+			if err != nil {
+				return err
+			}
+			// compare status, if Equal don't need update
+			compareStatus = reflect.DeepEqual(obClusterCurrent.Status, obClusterNew.Status)
+			if !compareStatus {
+				// update status
+				err = obClusterExecuter.UpdateStatus(context.TODO(), obClusterNew)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	)
+	if retryErr != nil {
+		klog.Errorln(retryErr)
+		return retryErr
+	}
+	// log
+	if !compareStatus {
+		p, _, _, _ := runtime.Caller(1)
+		tmp := strings.Split(runtime.FuncForPC(p).Name(), "/")
+		funcName := tmp[len(tmp)-1]
+		observerutil.LogForOBClusterStatusConvert(funcName, ctrl.OBCluster.Name, clusterStatus, zoneName, zoneStatus)
+	}
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) buildOBClusterStatus(obCluster cloudv1.OBCluster, clusterStatus, zoneName, zoneStatus string) (cloudv1.OBCluster, error) {
+	statefulAppName := converter.GenerateStatefulAppName(obCluster.Name)
+	statefulApp := &cloudv1.StatefulApp{}
+	statefulAppCtrl := NewStatefulAppCtrl(ctrl, *statefulApp)
+	// TODO: check owner
+	statefulAppCurrent, err := statefulAppCtrl.GetStatefulAppByName(statefulAppName)
+	if err != nil {
+		return obCluster, err
+	}
+
+	cluster := converter.GetClusterSpecFromOBTopology(ctrl.OBCluster.Spec.Topology)
+
+	nodeMap := make(map[string][]cloudv1.OBNode)
+	// get ClusterIP
+	clusterIP, err := ctrl.GetServiceClusterIPByName(ctrl.OBCluster.Namespace, ctrl.OBCluster.Name)
+	if err == nil {
+		// get nodeMap from DB
+		nodeMap = getNodeMapFromDB(clusterIP)
+	}
+
+	zoneList := buildZoneStatusList(cluster, statefulAppCurrent, nodeMap, zoneName, zoneStatus)
+
+	// old cluster status
+	var lastTransitionTime metav1.Time
+	oldClusterStatus := converter.GetClusterStatusFromOBTopologyStatus(ctrl.OBCluster.Status.Topology)
+	// old cluster status != now cluster status, need update lastTransitionTime & status
+	if oldClusterStatus.ClusterStatus != clusterStatus {
+		lastTransitionTime = metav1.Now()
+	} else {
+		lastTransitionTime = oldClusterStatus.LastTransitionTime
+	}
+
+	// new cluster status
+	var clusterCurrentStatus cloudv1.ClusterStatus
+	clusterCurrentStatus.Cluster = myconfig.ClusterName
+	clusterCurrentStatus.ClusterStatus = clusterStatus
+	clusterCurrentStatus.LastTransitionTime = lastTransitionTime
+	clusterCurrentStatus.Zone = zoneList
+
+	// topology status, multi cluster
+	topologyStatus := buildMultiClusterStatus(obCluster, clusterCurrentStatus)
+
+	if clusterStatus == observerconst.ClusterReady {
+		obCluster.Status.Status = observerconst.TopologyReady
+	} else if clusterStatus == observerconst.ScaleUP || clusterStatus == observerconst.ScaleDown {
+		obCluster.Status.Status = observerconst.TopologyNotReady
+	} else {
+		obCluster.Status.Status = observerconst.TopologyPrepareing
+	}
+	obCluster.Status.Topology = topologyStatus
+	return obCluster, nil
+}
+
+func buildZoneStatusList(cluster cloudv1.Cluster, statefulAppCurrent cloudv1.StatefulApp, nodeMap map[string][]cloudv1.OBNode, name, status string) []cloudv1.ZoneStatus {
+	zoneList := make([]cloudv1.ZoneStatus, 0)
+	for _, zone := range cluster.Zone {
+		zoneStatus := buildZoneStatus(statefulAppCurrent, nodeMap, name, status, zone)
+		zoneList = append(zoneList, zoneStatus)
+	}
+	return zoneList
+}
+
+func buildZoneStatus(statefulAppCurrent cloudv1.StatefulApp, nodeMap map[string][]cloudv1.OBNode, name, status string, zone cloudv1.Subset) cloudv1.ZoneStatus {
+	subsetStatus := converter.GetSubsetStatusFromStatefulApp(zone.Name, statefulAppCurrent)
+	var zoneStatus cloudv1.ZoneStatus
+	zoneStatus.Name = zone.Name
+	zoneStatus.Region = zone.Region
+	zoneStatus.ExpectedReplicas = zone.Replicas
+	// real AvailableReplicas from OB
+	nodeList := nodeMap[zone.Name]
+	zoneStatus.AvailableReplicas = int32(len(nodeList))
+	// StatefulApp is not ready
+	if subsetStatus.ExpectedReplicas != subsetStatus.AvailableReplicas {
+		zoneStatus.ZoneStatus = observerconst.OBZonePrepareing
+	} else {
+		if zoneStatus.ExpectedReplicas > zoneStatus.AvailableReplicas {
+			zoneStatus.ZoneStatus = observerconst.ScaleUP
+		} else if zoneStatus.ExpectedReplicas < zoneStatus.AvailableReplicas {
+			zoneStatus.ZoneStatus = observerconst.ScaleDown
+		} else {
+			zoneStatus.ZoneStatus = observerconst.OBZoneReady
+		}
+	}
+	// use custom status
+	if name == zone.Name && status != "" {
+		zoneStatus.ZoneStatus = status
+	}
+	return zoneStatus
+}
+
+func buildMultiClusterStatus(obCluster cloudv1.OBCluster, clusterCurrentStatus cloudv1.ClusterStatus) []cloudv1.ClusterStatus {
+	topologyStatus := make([]cloudv1.ClusterStatus, 0)
+	if len(obCluster.Status.Topology) > 0 {
+		for _, otherClusterStatus := range obCluster.Status.Topology {
+			if otherClusterStatus.Cluster != myconfig.ClusterName {
+				topologyStatus = append(topologyStatus, otherClusterStatus)
+			}
+		}
+	}
+	topologyStatus = append(topologyStatus, clusterCurrentStatus)
+	return topologyStatus
+}
+
+func getNodeMapFromDB(clusterIP string) map[string][]cloudv1.OBNode {
+	nodeMap := make(map[string][]cloudv1.OBNode)
+	obServerList := sql.GetOBServer(clusterIP)
+	if len(obServerList) > 0 {
+		nodeMap = converter.GenerateNodeMapByOBServerList(obServerList)
+	}
+	return nodeMap
+}

pkg/controllers/observer/core/observer_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/judge"
+)
+
+func (ctrl *OBClusterCtrl) OBServerCoordinator(statefulApp cloudv1.StatefulApp) error {
+	var err error
+	scaleState, subset := judge.OBServerScale(ctrl.OBCluster.Spec.Topology, statefulApp)
+	switch scaleState {
+	case observerconst.ScaleUP:
+		err = ctrl.UpdateOBServerReplica(subset, statefulApp, observerconst.ScaleUP)
+	case observerconst.ScaleDown:
+		err = ctrl.UpdateOBServerReplica(subset, statefulApp, observerconst.ScaleDown)
+	case observerconst.Maintain:
+		err = ctrl.OBServerMaintain(statefulApp)
+	}
+	return err
+}
+
+func (ctrl *OBClusterCtrl) UpdateOBServerReplica(subset cloudv1.Subset, statefulApp cloudv1.StatefulApp, status string) error {
+	// generate new StatefulApp for new replica
+	newStatefulApp := converter.UpdateSubsetReplicaForStatefulApp(subset, statefulApp)
+	// update StatefulApp replica
+	statefulAppCtrl := NewStatefulAppCtrl(ctrl, newStatefulApp)
+	err := statefulAppCtrl.UpdateStatefulApp()
+	if err != nil {
+		return err
+	}
+
+	// generate new OBZone for new replica
+	obZoneName := converter.GenerateOBZoneName(ctrl.OBCluster.Name)
+	obZoneCtrl := NewOBZoneCtrl(ctrl)
+	obZoneCurrent, err := obZoneCtrl.GetOBZoneByName(ctrl.OBCluster.Namespace, obZoneName)
+	if err != nil {
+		return err
+	}
+	newOBZone := converter.UpdateOBZoneSpec(obZoneCurrent, ctrl.OBCluster.Spec.Topology)
+	// update OBZone replica
+	err = obZoneCtrl.UpdateOBZone(newOBZone)
+	if err != nil {
+		return err
+	}
+
+	// update status
+	return ctrl.UpdateOBClusterAndZoneStatus(status, "", "")
+}
+
+func (ctrl *OBClusterCtrl) OBServerScaleUPByZone(statefulApp cloudv1.StatefulApp) error {
+	var clusterStatus string
+	var zoneStatus string
+
+	// get ClusterIP
+	clusterIP, err := ctrl.GetServiceClusterIPByName(ctrl.OBCluster.Namespace, ctrl.OBCluster.Name)
+	if err != nil {
+		return err
+	}
+
+	// get info for add server
+	err, zoneName, podIP := converter.GetInfoForAddServerByZone(clusterIP, statefulApp)
+	// nil need to add server
+	if err == nil {
+		clusterStatus = observerconst.ScaleUP
+		// add server
+		err = ctrl.AddOBServer(clusterIP, zoneName, podIP, statefulApp)
+		if err != nil {
+			return err
+		}
+		zoneStatus = observerconst.OBServerAdd
+		// update status
+		return ctrl.UpdateOBClusterAndZoneStatus(clusterStatus, zoneName, zoneStatus)
+	}
+
+	// need fix status
+	return ctrl.FixStatus()
+}
+
+func (ctrl *OBClusterCtrl) OBServerScaleDownByZone(statefulApp cloudv1.StatefulApp) error {
+	var clusterStatus string
+	var zoneStatus string
+
+	// get ClusterIP
+	clusterIP, err := ctrl.GetServiceClusterIPByName(ctrl.OBCluster.Namespace, ctrl.OBCluster.Name)
+	if err != nil {
+		return err
+	}
+
+	// get info for del server
+	clusterSpec := converter.GetClusterSpecFromOBTopology(ctrl.OBCluster.Spec.Topology)
+	err, zoneName, podIP := converter.GetInfoForDelServerByZone(clusterIP, clusterSpec, statefulApp)
+	// nil need to del server
+	if err == nil {
+		clusterStatus = observerconst.ScaleDown
+		// del server
+		err = ctrl.DelOBServer(clusterIP, zoneName, podIP)
+		if err != nil {
+			return err
+		}
+		zoneStatus = observerconst.OBServerDel
+		// update status
+		return ctrl.UpdateOBClusterAndZoneStatus(clusterStatus, zoneName, zoneStatus)
+	}
+
+	// need fix status
+	return ctrl.FixStatus()
+}
+
+func (ctrl *OBClusterCtrl) OBServerMaintain(statefulApp cloudv1.StatefulApp) error {
+	// get ClusterIP
+	clusterIP, err := ctrl.GetServiceClusterIPByName(ctrl.OBCluster.Namespace, ctrl.OBCluster.Name)
+	if err != nil {
+		return err
+	}
+
+	// get info for add server
+	err, zoneName, podIP := converter.GetInfoForAddServerByZone(clusterIP, statefulApp)
+	// nil is need to add server
+	if err == nil {
+		// add server
+		return ctrl.AddOBServer(clusterIP, zoneName, podIP, statefulApp)
+	}
+
+	// get info for del server
+	clusterSpec := converter.GetClusterSpecFromOBTopology(ctrl.OBCluster.Spec.Topology)
+	err, zoneName, podIP = converter.GetInfoForDelServerByZone(clusterIP, clusterSpec, statefulApp)
+	// nil need to del server
+	if err == nil {
+		// del server
+		return ctrl.DelOBServer(clusterIP, zoneName, podIP)
+	}
+
+	return ctrl.OBClusterReadyForStep(observerconst.StepMaintain, statefulApp)
+}
+
+func (ctrl *OBClusterCtrl) FixStatus() error {
+	var zoneName string
+	var zoneStatus string
+	var clusterStatus string
+	oldClusterStatus := converter.GetClusterStatusFromOBTopologyStatus(ctrl.OBCluster.Status.Topology)
+	for _, oldZoneStatus := range oldClusterStatus.Zone {
+		if oldZoneStatus.ZoneStatus != observerconst.OBZoneReady {
+			zoneName = oldZoneStatus.Name
+			zoneStatus = observerconst.OBZoneReady
+			clusterStatus = observerconst.ClusterReady
+			break
+		}
+	}
+	return ctrl.UpdateOBClusterAndZoneStatus(clusterStatus, zoneName, zoneStatus)
+}

pkg/controllers/observer/core/obzone_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type OBZoneCtrl struct {
+	OBClusterCtrl OBClusterCtrl
+}
+
+type OBZoneCtrlOperator interface {
+	CreateOBZone() (cloudv1.OBZone, error)
+	GetOBZoneByName(namespace, name string) (cloudv1.OBZone, error)
+	UpdateOBZone(zone cloudv1.OBZone) error
+	UpdateOBZoneStatus(zone cloudv1.OBZone) error
+	DeleteOBZone(zone cloudv1.OBZone) error
+}
+
+func NewOBZoneCtrl(obClusterCtrl *OBClusterCtrl) OBZoneCtrlOperator {
+	return &OBZoneCtrl{
+		OBClusterCtrl: *obClusterCtrl,
+	}
+}
+
+func (ctrl *OBZoneCtrl) CreateOBZone() (cloudv1.OBZone, error) {
+	obZone := converter.GenerateOBZoneObject(ctrl.OBClusterCtrl.OBCluster)
+	obZoneExecuter := resource.NewOBZoneResource(ctrl.OBClusterCtrl.Resource)
+	err := obZoneExecuter.Create(context.TODO(), obZone)
+	if err != nil {
+		return obZone, err
+	}
+	return obZone, nil
+}
+
+func (ctrl *OBZoneCtrl) GetOBZoneByName(namespace, name string) (cloudv1.OBZone, error) {
+	obZoneExecuter := resource.NewOBZoneResource(ctrl.OBClusterCtrl.Resource)
+	rs, err := obZoneExecuter.Get(context.TODO(), namespace, name)
+	if err != nil {
+		return rs.(cloudv1.OBZone), err
+	}
+	return rs.(cloudv1.OBZone), nil
+}
+
+func (ctrl *OBZoneCtrl) UpdateOBZone(zone cloudv1.OBZone) error {
+	obZoneExecuter := resource.NewOBZoneResource(ctrl.OBClusterCtrl.Resource)
+	err := obZoneExecuter.Update(context.TODO(), zone)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctrl *OBZoneCtrl) UpdateOBZoneStatus(zone cloudv1.OBZone) error {
+	obZoneExecuter := resource.NewOBZoneResource(ctrl.OBClusterCtrl.Resource)
+	err := obZoneExecuter.UpdateStatus(context.TODO(), zone)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctrl *OBZoneCtrl) DeleteOBZone(zone cloudv1.OBZone) error {
+	obZoneExecuter := resource.NewOBZoneResource(ctrl.OBClusterCtrl.Resource)
+	err := obZoneExecuter.Delete(context.TODO(), zone)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/controllers/observer/core/obzone_status.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"reflect"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+func (ctrl *OBClusterCtrl) UpdateOBZoneStatus(statefulApp cloudv1.StatefulApp) error {
+	subsets := statefulApp.Status.Subsets
+	// TODO: check owner
+	obZoneName := converter.GenerateOBZoneName(ctrl.OBCluster.Name)
+	obZoneCtrl := NewOBZoneCtrl(ctrl)
+	obZoneCurrent, err := obZoneCtrl.GetOBZoneByName(ctrl.OBCluster.Namespace, obZoneName)
+	if err != nil {
+		return err
+	}
+	obServerList := sql.GetOBServer(subsets[0].Pods[0].PodIP)
+	cluster := converter.GetClusterSpecFromOBTopology(ctrl.OBCluster.Spec.Topology)
+	obZoneStatus := converter.OBServerListToOBZoneStatus(cluster, obZoneCurrent, obServerList)
+	status := reflect.DeepEqual(obZoneCurrent.Status, obZoneStatus.Status)
+	if !status {
+		err = obZoneCtrl.UpdateOBZoneStatus(obZoneStatus)
+		if err != nil {
+			return err
+		}
+		kube.LogForAppActionStatus(obZoneStatus.Kind, obZoneName, "update status", obZoneStatus)
+	}
+	return nil
+}

pkg/controllers/observer/core/pod_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+func (ctrl *OBClusterCtrl) DelPodFromStatefulAppByIP(zoneName, podIP string, statefulApp cloudv1.StatefulApp) error {
+	subsetStatus := converter.GetSubsetStatusFromStatefulApp(zoneName, statefulApp)
+	for _, pod := range subsetStatus.Pods {
+		if pod.PodIP == podIP {
+			podExecuter := resource.NewPodResource(ctrl.Resource)
+			podObject, err := podExecuter.Get(context.TODO(), ctrl.OBCluster.Namespace, pod.Name)
+			if err != nil {
+				return err
+			}
+			err = podExecuter.Delete(context.TODO(), podObject)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/controllers/observer/core/rootservice_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type RootServiceCtrl struct {
+	OBClusterCtrl OBClusterCtrl
+}
+
+type RootServiceCtrlOperator interface {
+	CreateRootService() (cloudv1.RootService, error)
+	GetRootServiceByName(namespace, name string) (cloudv1.RootService, error)
+	UpdateRootServiceStatus(rs cloudv1.RootService) error
+	DeleteRootService(rs cloudv1.RootService) error
+}
+
+func NewRootServiceCtrl(obClusterCtrl *OBClusterCtrl) RootServiceCtrlOperator {
+	return &RootServiceCtrl{
+		OBClusterCtrl: *obClusterCtrl,
+	}
+}
+
+func (ctrl *RootServiceCtrl) CreateRootService() (cloudv1.RootService, error) {
+	rootService := converter.GenerateRootServiceObject(ctrl.OBClusterCtrl.OBCluster)
+	rootServiceExecuter := resource.NewRootServiceResource(ctrl.OBClusterCtrl.Resource)
+	err := rootServiceExecuter.Create(context.TODO(), rootService)
+	if err != nil {
+		return rootService, err
+	}
+	return rootService, nil
+}
+
+func (ctrl *RootServiceCtrl) GetRootServiceByName(namespace, name string) (cloudv1.RootService, error) {
+	rootServiceExecuter := resource.NewRootServiceResource(ctrl.OBClusterCtrl.Resource)
+	rs, err := rootServiceExecuter.Get(context.TODO(), namespace, name)
+	if err != nil {
+		return rs.(cloudv1.RootService), err
+	}
+	return rs.(cloudv1.RootService), nil
+}
+
+func (ctrl *RootServiceCtrl) UpdateRootServiceStatus(rs cloudv1.RootService) error {
+	rootServiceExecuter := resource.NewRootServiceResource(ctrl.OBClusterCtrl.Resource)
+	err := rootServiceExecuter.UpdateStatus(context.TODO(), rs)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctrl *RootServiceCtrl) DeleteRootService(rs cloudv1.RootService) error {
+	rootServiceExecuter := resource.NewRootServiceResource(ctrl.OBClusterCtrl.Resource)
+	err := rootServiceExecuter.Delete(context.TODO(), rs)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/controllers/observer/core/rootservice_status.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"reflect"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+func (ctrl *OBClusterCtrl) UpdateRootServiceStatus(statefulApp cloudv1.StatefulApp) error {
+	subsets := statefulApp.Status.Subsets
+	// TODO: check owner
+	rsName := converter.GenerateRootServiceName(ctrl.OBCluster.Name)
+	rsCtrl := NewRootServiceCtrl(ctrl)
+	rsCurrent, err := rsCtrl.GetRootServiceByName(ctrl.OBCluster.Namespace, rsName)
+	if err != nil {
+		return err
+	}
+	rsList := sql.GetRootService(subsets[0].Pods[0].PodIP)
+	obServerList := sql.GetOBServer(subsets[0].Pods[0].PodIP)
+	cluster := converter.GetClusterSpecFromOBTopology(ctrl.OBCluster.Spec.Topology)
+	rsStatus := converter.RSListToRSStatus(cluster, rsCurrent, rsList, obServerList)
+	status := reflect.DeepEqual(rsCurrent.Status, rsStatus.Status)
+	if !status {
+		err = rsCtrl.UpdateRootServiceStatus(rsStatus)
+		if err != nil {
+			return err
+		}
+		kube.LogForAppActionStatus(rsStatus.Kind, rsName, "update", rsStatus)
+	}
+	return nil
+}

pkg/controllers/observer/core/service_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+func (ctrl *OBClusterCtrl) CreateService(statefulAppName string) error {
+	service := converter.GenerateServiceObject(ctrl.OBCluster, statefulAppName)
+	serviceExecuter := resource.NewServiceResource(ctrl.Resource)
+	err := serviceExecuter.Create(context.TODO(), service)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) GetServiceByName(namespace, name string) (corev1.Service, error) {
+	svcName := converter.GenerateServiceName(name)
+	serviceExecuter := resource.NewServiceResource(ctrl.Resource)
+	svc, err := serviceExecuter.Get(context.TODO(), namespace, svcName)
+	if err != nil {
+		return svc.(corev1.Service), err
+	}
+	return svc.(corev1.Service), nil
+}
+
+func (ctrl *OBClusterCtrl) GetServiceClusterIPByName(namespace, name string) (string, error) {
+	svcName := converter.GenerateServiceName(name)
+	serviceExecuter := resource.NewServiceResource(ctrl.Resource)
+	svc, err := serviceExecuter.Get(context.TODO(), namespace, svcName)
+	if err != nil {
+		return "", err
+	}
+	return svc.(corev1.Service).Spec.ClusterIP, nil
+}

pkg/controllers/observer/core/statefulapp_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type StatefulAppCtrl struct {
+	OBClusterCtrl OBClusterCtrl
+	StatefulApp   cloudv1.StatefulApp
+}
+
+type StatefulAppCtrlOperator interface {
+	CreateStatefulApp() (cloudv1.StatefulApp, error)
+	GetStatefulAppByName(name string) (cloudv1.StatefulApp, error)
+	UpdateStatefulApp() error
+	DeleteStatefulApp() error
+}
+
+func NewStatefulAppCtrl(obClusterCtrl *OBClusterCtrl, statefulApp cloudv1.StatefulApp) StatefulAppCtrlOperator {
+	return &StatefulAppCtrl{
+		OBClusterCtrl: *obClusterCtrl,
+		StatefulApp:   statefulApp,
+	}
+}
+
+func (ctrl *StatefulAppCtrl) CreateStatefulApp() (cloudv1.StatefulApp, error) {
+	var cluster cloudv1.Cluster
+	for _, cluster = range ctrl.OBClusterCtrl.OBCluster.Spec.Topology {
+		if cluster.Cluster == myconfig.ClusterName {
+			break
+		}
+	}
+	statefulApp := converter.GenerateStatefulAppObject(cluster, ctrl.OBClusterCtrl.OBCluster)
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrl.OBClusterCtrl.Resource)
+	err := statefulAppExecuter.Create(context.TODO(), statefulApp)
+	if err != nil {
+		return statefulApp, err
+	}
+	return statefulApp, nil
+}
+
+func (ctrl *StatefulAppCtrl) GetStatefulAppByName(name string) (cloudv1.StatefulApp, error) {
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrl.OBClusterCtrl.Resource)
+	statefulApp, err := statefulAppExecuter.Get(context.TODO(), ctrl.OBClusterCtrl.OBCluster.Namespace, name)
+	if err != nil {
+		return cloudv1.StatefulApp{}, err
+	}
+	return statefulApp.(cloudv1.StatefulApp), nil
+}
+
+func (ctrl *StatefulAppCtrl) UpdateStatefulApp() error {
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrl.OBClusterCtrl.Resource)
+	err := statefulAppExecuter.Update(context.TODO(), ctrl.StatefulApp)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctrl *StatefulAppCtrl) DeleteStatefulApp() error {
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrl.OBClusterCtrl.Resource)
+	err := statefulAppExecuter.Delete(context.TODO(), ctrl.StatefulApp)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/controllers/observer/core/task_obcluster_bootstrap.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"time"
+
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/klog/v2"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/cable"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	statefulappCore "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func (ctrl *OBClusterCtrl) IsNewCluster(statefulApp cloudv1.StatefulApp) (*cloudv1.StatefulApp, bool) {
+	var err error
+	for _, cluster := range ctrl.OBCluster.Spec.Topology {
+		if cluster.Cluster == myconfig.ClusterName {
+			statefulAppCtrl := NewStatefulAppCtrl(ctrl, statefulApp)
+			// TODO: check owner
+			statefulAppName := converter.GenerateStatefulAppName(ctrl.OBCluster.Name)
+			statefulApp, err = statefulAppCtrl.GetStatefulAppByName(statefulAppName)
+			if err != nil {
+				if kubeerrors.IsNotFound(err) {
+					return &statefulApp, true
+				}
+			}
+		}
+	}
+	return &statefulApp, false
+}
+
+func (ctrl *OBClusterCtrl) NewCluster(statefulApp cloudv1.StatefulApp) error {
+	// create StatefulApp
+	statefulAppCtrl := NewStatefulAppCtrl(ctrl, statefulApp)
+	statefulApp, err := statefulAppCtrl.CreateStatefulApp()
+	if err != nil {
+		return err
+	}
+
+	// create RootService
+	rsCtrl := NewRootServiceCtrl(ctrl)
+	_, err = rsCtrl.CreateRootService()
+	if err != nil {
+		return err
+	}
+
+	// create OBZone
+	obZoneCtrl := NewOBZoneCtrl(ctrl)
+	_, err = obZoneCtrl.CreateOBZone()
+	if err != nil {
+		return err
+	}
+
+	// update status
+	return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ResourcePrepareing, "", "")
+}
+
+func (ctrl *OBClusterCtrl) ResourcePrepareingEffectorForBootstrap(statefulApp cloudv1.StatefulApp) error {
+	var err error
+	if statefulApp.Status.ClusterStatus == statefulappCore.Ready {
+		// update status
+		return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ResourceReady, "", "")
+	} else {
+		// time out, delete StatefulApp
+		_, err = ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// ResourceReadyEffectorForBootstrap will get iplsit by zone, and send iplsit to cable server, cable will start ob server
+func (ctrl *OBClusterCtrl) ResourceReadyEffectorForBootstrap(statefulApp cloudv1.StatefulApp) error {
+	// time out, delete StatefulApp
+	status, err := ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	if !status {
+		// check StatefulApp status
+		err = ctrl.StatefulAppStatusCheckForBootstrap(statefulApp)
+		if err != nil {
+			return err
+		}
+
+		subsets := statefulApp.Status.Subsets
+
+		// check cable status
+		err = cable.CableStatusCheck(subsets)
+		if err != nil {
+			// TODO: need to check, why the cable get ip failed
+			klog.Errorln("cable get ip failed, ", "OBCluster status ", util.CovertToJSON(ctrl.OBCluster.Status), "StatefulApp status", util.CovertToJSON(statefulApp.Status))
+			return err
+		}
+
+		// generate rsList
+		rsList := cable.GenerateRSListFromSubset(subsets)
+
+		// start observer
+		go cable.OBServerStart(ctrl.OBCluster, subsets, rsList)
+
+		// update status
+		return ctrl.UpdateOBClusterAndZoneStatus(observerconst.OBServerPrepareing, "", "")
+	}
+
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) OBServerPrepareingEffectorForBootstrap(statefulApp cloudv1.StatefulApp) error {
+	// time out, delete StatefulApp
+	status, err := ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	if !status {
+		// check StatefulApp status
+		err = ctrl.StatefulAppStatusCheckForBootstrap(statefulApp)
+		if err != nil {
+			return err
+		}
+
+		subsets := statefulApp.Status.Subsets
+
+		// check observer status
+		err = cable.OBServerStatusCheck(ctrl.OBCluster.Name, subsets)
+		if err != nil {
+			return nil
+		}
+
+		// update status
+		return ctrl.UpdateOBClusterAndZoneStatus(observerconst.OBServerReady, "", "")
+	}
+
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) OBServerReadyEffectorForBootstrap(statefulApp cloudv1.StatefulApp) error {
+	// time out, delete StatefulApp
+	status, err := ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	if !status {
+		// check StatefulApp status
+		err = ctrl.StatefulAppStatusCheckForBootstrap(statefulApp)
+		if err != nil {
+			return err
+		}
+
+		subsets := statefulApp.Status.Subsets
+
+		// make obcluster bootstrap args
+		obclusterBootstrapArgs, err := cable.GenerateOBClusterBootstrapArgs(subsets)
+		if err != nil {
+			// delete StatefulApp
+			statefulAppCtrl := NewStatefulAppCtrl(ctrl, statefulApp)
+			err = statefulAppCtrl.DeleteStatefulApp()
+			if err != nil {
+				return err
+			}
+			klog.Errorln("generate OBCluster bootstrap args failed, delete StatefulApp", statefulApp.Name, "to recreate")
+		}
+
+		// update status
+		err = ctrl.UpdateOBClusterAndZoneStatus(observerconst.OBClusterBootstraping, "", "")
+		if err != nil {
+			return err
+		}
+
+		// run bootstrap SQL
+		go ctrl.BootstrapForOB(subsets[0].Pods[0].PodIP, obclusterBootstrapArgs)
+	}
+
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) BootstrapForOB(IP, SQL string) {
+	sql.BootstrapForOB(IP, SQL)
+	klog.Infoln("OBCluster", ctrl.OBCluster.Name, "run bootstrap sql finish")
+	_ = ctrl.UpdateOBClusterStatusBootstrapReady()
+}
+
+func (ctrl *OBClusterCtrl) UpdateOBClusterStatusBootstrapReady() error {
+	// update status
+	return ctrl.UpdateOBClusterAndZoneStatus(observerconst.OBClusterBootstrapReady, "", "")
+}
+
+func (ctrl *OBClusterCtrl) OBClusterBootstraping(statefulApp cloudv1.StatefulApp) error {
+	// time out, delete StatefulApp
+	_, err := ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+	return err
+}
+
+func (ctrl *OBClusterCtrl) OBClusterBootstrapReady(statefulApp cloudv1.StatefulApp) error {
+	// time out, delete StatefulApp
+	status, err := ctrl.CheckTimeoutAndKillForBootstrap(statefulApp)
+	if err != nil {
+		return err
+	}
+
+	if !status {
+		subsets := statefulApp.Status.Subsets
+		obServerList := sql.GetOBServer(subsets[0].Pods[0].PodIP)
+
+		obServerBootstrapSucceed := converter.IsAllOBServerActive(obServerList, ctrl.OBCluster.Spec.Topology)
+		if obServerBootstrapSucceed {
+			// update OBServer Pod Readiness
+			err = cable.CableReadinessUpdate(subsets)
+			if err != nil {
+				return err
+			}
+			// update status
+			return ctrl.UpdateOBClusterAndZoneStatus(observerconst.OBClusterReady, "", "")
+		}
+
+		klog.Infoln("wait for OBCluster", ctrl.OBCluster.Name, "Bootstraping finish")
+	}
+
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) StatefulAppStatusCheckForBootstrap(statefulApp cloudv1.StatefulApp) error {
+	if statefulApp.Status.ClusterStatus != statefulappCore.Ready {
+		// update status
+		err := ctrl.UpdateOBClusterAndZoneStatus(observerconst.ResourcePrepareing, "", "")
+		if err != nil {
+			return err
+		}
+		klog.Infoln("StatefulApp is Prepareing, update OBCluster", ctrl.OBCluster.Name, "to", observerconst.ResourcePrepareing)
+	}
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) CheckTimeoutAndKillForBootstrap(statefulApp cloudv1.StatefulApp) (bool, error) {
+	creationTimestamp := statefulApp.CreationTimestamp.Unix()
+	nowTimestamp := time.Now().Unix()
+	// 5 min
+	if nowTimestamp-creationTimestamp > observerconst.BootstrapTimeout {
+		klog.Infoln("OBCluster Bootstraping timeout, now time", nowTimestamp, "create time", creationTimestamp)
+		statefulAppCtrl := NewStatefulAppCtrl(ctrl, statefulApp)
+		// delete StatefulApp
+		err := statefulAppCtrl.DeleteStatefulApp()
+		if err != nil {
+			return true, err
+		}
+		// delete RootService
+		rsName := converter.GenerateRootServiceName(ctrl.OBCluster.Name)
+		rsCtrl := NewRootServiceCtrl(ctrl)
+		rsCurrent, _ := rsCtrl.GetRootServiceByName(ctrl.OBCluster.Namespace, rsName)
+		_ = rsCtrl.DeleteRootService(rsCurrent)
+		// delete OBZone
+		obZoneName := converter.GenerateOBZoneName(ctrl.OBCluster.Name)
+		obZoneCtrl := NewOBZoneCtrl(ctrl)
+		obZoneCurrent, _ := obZoneCtrl.GetOBZoneByName(ctrl.OBCluster.Namespace, obZoneName)
+		_ = obZoneCtrl.DeleteOBZone(obZoneCurrent)
+		// update status
+		err = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, "", "")
+		if err != nil {
+			return true, err
+		}
+		return true, nil
+	}
+	return false, nil
+}

pkg/controllers/observer/core/task_observer_add.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/cable"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+)
+
+func (ctrl *OBClusterCtrl) AddOBServer(clusterIP, zoneName, podIP string, statefulApp cloudv1.StatefulApp) error {
+	clusterStatus := converter.GetClusterStatusFromOBTopologyStatus(ctrl.OBCluster.Status.Topology)
+	for _, zone := range clusterStatus.Zone {
+		if zone.ZoneStatus == observerconst.OBServerAdd {
+			// judge timeout
+			lastTransitionTimestamp := clusterStatus.LastTransitionTime.Unix()
+			nowTimestamp := time.Now().Unix()
+			if nowTimestamp-lastTransitionTimestamp > observerconst.AddServerTimeout {
+				klog.Infoln("add server timeout, need delete", zoneName, podIP)
+				err := ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+				if err != nil {
+					return err
+				}
+				return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+			}
+			return nil
+		}
+	}
+	for _, zone := range clusterStatus.Zone {
+		// an IP is already in the process of being added
+		// execute serially, one IP at a time
+		if zone.Name == zoneName && zone.ZoneStatus != observerconst.OBServerAdd {
+			// add server
+			go ctrl.AddOBServerExecuter(clusterIP, zoneName, podIP, statefulApp)
+			// update status
+			return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ScaleUP, zoneName, observerconst.OBServerAdd)
+		}
+	}
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) AddOBServerExecuter(clusterIP, zoneName, podIP string, statefulApp cloudv1.StatefulApp) {
+	klog.Infoln("begin add OBServer", zoneName, podIP)
+
+	// check cable status
+	err := cable.CableStatusCheckExecuter(podIP)
+	if err != nil {
+		// kill pod
+		_ = ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+		_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+	}
+
+	// get rs
+	rsName := converter.GenerateRootServiceName(ctrl.OBCluster.Name)
+	rsCtrl := NewRootServiceCtrl(ctrl)
+	rsCurrent, err := rsCtrl.GetRootServiceByName(ctrl.OBCluster.Namespace, rsName)
+	if err != nil {
+		// kill pod
+		_ = ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+		_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+	}
+
+	// generate rsList
+	rsList := cable.GenerateRSListFromRootServiceStatus(rsCurrent.Status.Topology)
+
+	// generate start args
+	obServerStartArgs := cable.GenerateOBServerStartArgs(ctrl.OBCluster, zoneName, rsList)
+
+	// check OBServer is already running, for OBServer Scale UP
+	err = cable.OBServerStatusCheckExecuter(ctrl.OBCluster.Name, podIP)
+	// nil is OBServer is already running
+	if err != nil {
+		cable.OBServerStartExecuter(podIP, obServerStartArgs)
+		err = TickerOBServerStatusCheck(ctrl.OBCluster.Name, podIP)
+		if err != nil {
+			// kill pod
+			_ = ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+			_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+		}
+	}
+
+	// add server
+	err = sql.AddServer(clusterIP, zoneName, podIP)
+	if err != nil {
+		// kill pod
+		_ = ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+		_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+	}
+
+	// update OBServer Pod Readiness
+	err = cable.CableReadinessUpdateExecuter(podIP)
+	if err != nil {
+		// kill pod
+		_ = ctrl.DelPodFromStatefulAppByIP(zoneName, podIP, statefulApp)
+		_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+	}
+
+	klog.Infoln("add OBServer finish", zoneName, podIP)
+
+	// update status
+	_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+}
+
+func TickerOBServerStatusCheck(clusterName, podIP string) error {
+	tick := time.Tick(observerconst.TickPeriodForOBServerStatusCheck)
+	var num int
+	var err error
+	for {
+		select {
+		case <-tick:
+			if num > observerconst.TickNumForOBServerStatusCheck {
+				return errors.New("observer starting timeout")
+			}
+			num = num + 1
+			err = cable.OBServerStatusCheckExecuter(clusterName, podIP)
+			if err == nil {
+				return err
+			}
+		}
+	}
+}

pkg/controllers/observer/core/task_observer_del.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"runtime"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+)
+
+func (ctrl *OBClusterCtrl) DelOBServer(clusterIP, zoneName, podIP string) error {
+	clusterStatus := converter.GetClusterStatusFromOBTopologyStatus(ctrl.OBCluster.Status.Topology)
+	for _, zone := range clusterStatus.Zone {
+		if zone.ZoneStatus == observerconst.OBServerDel {
+			// judge timeout
+			lastTransitionTimestamp := clusterStatus.LastTransitionTime.Unix()
+			nowTimestamp := time.Now().Unix()
+			if nowTimestamp-lastTransitionTimestamp > observerconst.DelServerTimeout {
+				klog.Infoln("OBCluster delete server timeout, now time", nowTimestamp, "create time", lastTransitionTimestamp)
+				return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+			}
+			return nil
+		}
+	}
+	for _, zone := range clusterStatus.Zone {
+		// an IP is already in the process of being deleted
+		// execute serially, one IP at a time
+		if zone.Name == zoneName && zone.ZoneStatus != observerconst.OBServerDel {
+			// del server
+			go ctrl.DelOBServerExecuter(clusterIP, zoneName, podIP)
+			// update status
+			return ctrl.UpdateOBClusterAndZoneStatus(observerconst.ScaleDown, zoneName, observerconst.OBServerDel)
+		}
+	}
+	return nil
+}
+
+func (ctrl *OBClusterCtrl) DelOBServerExecuter(clusterIP, zoneName, podIP string) {
+	klog.Infoln("begin delete OBServer", zoneName, podIP)
+
+	// update server server_permanent_offline_time
+	err := sql.SetServerOfflineTime(clusterIP, 20)
+	if err != nil {
+		klog.Errorln("set server_permanent_offline_time error", zoneName, podIP)
+		runtime.Goexit()
+	}
+
+	// delete server
+	err = sql.DelServer(clusterIP, podIP)
+	if err != nil {
+		klog.Errorln("delete server error", zoneName, podIP)
+		runtime.Goexit()
+	}
+
+	// check delete finish
+	err = TickerRSJobStatusCheck(clusterIP, podIP)
+	if err != nil {
+		klog.Errorln("check rs job status error", zoneName, podIP)
+		runtime.Goexit()
+	}
+
+	// check server is not in db
+	status := OBServerDeletedCheck(clusterIP, podIP)
+	if status {
+		klog.Infoln("delete OBServer finish", zoneName, podIP)
+
+		// update status
+		_ = ctrl.UpdateOBClusterAndZoneStatus(observerconst.ClusterReady, zoneName, observerconst.OBZoneReady)
+	}
+}
+
+func TickerRSJobStatusCheck(clusterIP, podIP string) error {
+	tick := time.Tick(observerconst.TickPeriodForRSJobStatusCheck)
+	var num int
+	for {
+		select {
+		case <-tick:
+			if num > observerconst.TickNumForRSJobStatusCheck {
+				return errors.New("observer starting timeout")
+			}
+			num = num + 1
+			status, err := converter.IsRSJobSuccess(clusterIP, podIP)
+			if err == nil {
+				return err
+			}
+			if status {
+				err = sql.SetServerOfflineTime(clusterIP, 3600)
+				if err != nil {
+					return err
+				}
+				return nil
+			}
+		}
+	}
+}
+
+func OBServerDeletedCheck(clusterIP, podIP string) bool {
+	status := converter.IsOBServerDeleted(clusterIP, podIP)
+	if status {
+		return true
+	}
+	return false
+}

pkg/controllers/observer/core/util/log.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"runtime"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+func LogForOBClusterStatusConvert(funcName, clusterName, status, zoneName, zoneStatus string) {
+	if funcName == "" {
+		p, _, _, _ := runtime.Caller(2)
+		tmp := strings.Split(runtime.FuncForPC(p).Name(), "/")
+		funcName = tmp[len(tmp)-1]
+	}
+	klog.Infoln(funcName, "update OBCluster", clusterName, "to", status, "Zone", zoneName, "Status", zoneStatus)
+}

pkg/controllers/observer/core/util/util.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func GenerateOwnerReference(obCluster cloudv1.OBCluster) metav1.OwnerReference {
+	ownerReference := metav1.OwnerReference{
+		APIVersion: obCluster.APIVersion,
+		Kind:       obCluster.Kind,
+		Name:       obCluster.Name,
+		UID:        obCluster.UID,
+	}
+	return ownerReference
+}
+
+func GenerateLabels(app string) map[string]string {
+	labels := make(map[string]string)
+	labels["app"] = app
+	return labels
+}
+
+func GenerateObjectMeta(obCluster cloudv1.OBCluster, name string) metav1.ObjectMeta {
+	ownerReference := GenerateOwnerReference(obCluster)
+	labels := GenerateLabels(obCluster.Name)
+	objectMeta := metav1.ObjectMeta{
+		Name:            name,
+		Namespace:       obCluster.Namespace,
+		OwnerReferences: []metav1.OwnerReference{ownerReference},
+		Labels:          labels,
+	}
+	return objectMeta
+}

pkg/controllers/observer/event_handler.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func isRefController(controllerRef *metav1.OwnerReference) bool {
+	refGV, err := schema.ParseGroupVersion(controllerRef.APIVersion)
+	if err != nil {
+		klog.Errorf("could not parse OwnerReference %v APIVersion: %v", controllerRef, err)
+		return false
+	}
+	return controllerRef.Kind == controllerKind.Kind && refGV.Group == controllerKind.Group
+}
+
+type statefulAppEventHandler struct {
+	enqueueHandler handler.EnqueueRequestForOwner
+}
+
+func (p *statefulAppEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+	statefulApp := evt.Object.(*cloudv1.StatefulApp)
+	if statefulApp.DeletionTimestamp != nil {
+		p.Delete(event.DeleteEvent{Object: evt.Object}, q)
+		return
+	}
+	controllerRef := metav1.GetControllerOf(statefulApp)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Create(evt, q)
+	}
+}
+
+func (p *statefulAppEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	oldStatefulApp := evt.ObjectOld.(*cloudv1.StatefulApp)
+	newStatefulApp := evt.ObjectNew.(*cloudv1.StatefulApp)
+	if newStatefulApp.ResourceVersion == oldStatefulApp.ResourceVersion {
+		return
+	}
+	p.enqueueHandler.Update(evt, q)
+}
+
+func (p *statefulAppEventHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	statefulApp := evt.Object.(*cloudv1.StatefulApp)
+	controllerRef := metav1.GetControllerOf(statefulApp)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Delete(evt, q)
+	}
+}
+
+func (p *statefulAppEventHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}

pkg/controllers/observer/init.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+	"github.com/oceanbase/ob-operator/pkg/kubeclient"
+)
+
+var (
+	controllerKind = cloudv1.SchemeGroupVersion.WithKind("OBCluster")
+)
+
+// Add creates a new Controller and adds it to the Manager with default RBAC.
+// The Manager will set fields on the Controller and Start it when the Manager is Started.
+func Add(mgr manager.Manager) error {
+	if !kube.DiscoverGVK(controllerKind) {
+		return nil
+	}
+	return add(mgr, newReconciler(mgr))
+}
+
+func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+	return &core.OBClusterReconciler{
+		CRClient: kubeclient.NewClientFromManager(mgr),
+		Scheme:   mgr.GetScheme(),
+		Recorder: mgr.GetEventRecorderFor(observerconst.ControllerName),
+	}
+}
+
+// add a new Controller to mgr with r
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New(
+		observerconst.ControllerName,
+		mgr,
+		controller.Options{
+			Reconciler:              r,
+			MaxConcurrentReconciles: observerconst.ConcurrentReconciles,
+		},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	// Watch for changes to OBCluster
+	err = c.Watch(
+		&source.Kind{Type: &cloudv1.OBCluster{}},
+		&handler.EnqueueRequestForObject{},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	// Watch for changes to StatefulApp
+	err = c.Watch(
+		&source.Kind{Type: &cloudv1.StatefulApp{}},
+		&statefulAppEventHandler{
+			enqueueHandler: handler.EnqueueRequestForOwner{
+				IsController: true,
+				OwnerType:    &cloudv1.OBCluster{},
+			},
+		},
+		&statefulAppPredicate{},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	return nil
+}

pkg/controllers/observer/model/model.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package model
+
+type AllServer struct {
+	ID              int64
+	Zone            string
+	SvrIP           string
+	SvrPort         int64
+	InnerPort       int64
+	WithRootService int64
+	WithPartition   int64
+	Status          string
+}
+
+type AllVirtualCoreMeta struct {
+	Zone         string
+	SvrIP        string
+	SvrPort      int64
+	Role         int64
+	PartitionIP  int64
+	PartitionCnt int64
+}
+
+type RSJobStatus struct {
+	JobStatus  string
+	ReturnCode int64
+	Progress   int64
+}

pkg/controllers/observer/predicate.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type statefulAppPredicate struct {
+}
+
+func (p statefulAppPredicate) Create(e event.CreateEvent) bool {
+	return true
+}
+
+func (p statefulAppPredicate) Delete(e event.DeleteEvent) bool {
+	return true
+}
+
+func (p statefulAppPredicate) Update(e event.UpdateEvent) bool {
+	return true
+}
+
+func (p statefulAppPredicate) Generic(e event.GenericEvent) bool {
+	return true
+}

pkg/controllers/observer/sql/conn.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+import (
+	"fmt"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+	"k8s.io/klog/v2"
+)
+
+func ConnOB(IP, port, dbName string, timeout int) *gorm.DB {
+	connInfo := fmt.Sprintf("root:@tcp(%s:%s)/%s?timeout=%ds&charset=utf8&parseTime=True&loc=Local", IP, port, dbName, timeout)
+	client, err := gorm.Open("mysql", connInfo)
+	if err != nil {
+		errNum, errMsg := covertErrToMySQLError(err)
+		klog.Errorln(errNum, errMsg)
+		return nil
+	}
+	return client
+}

pkg/controllers/observer/sql/const.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+const (
+	DatabaseOb = "oceanbase"
+)
+
+const (
+	SetTimeoutSQL                   = "SET ob_query_timeout = 600000000"
+	SetServerOfflineTimeSQLTemplate = "ALTER SYSTEM SET server_permanent_offline_time=${OFFLINE_TIME};"
+
+	GetOBServerSQL       = "SELECT id, zone, svr_ip, svr_port, inner_port, with_rootserver, with_partition, status FROM __all_server;"
+	AddServerSQLTemplate = "ALTER SYSTEM ADD SERVER '${SERVER_IP}' ZONE '${ZONE_NAME}';"
+	DelServerSQLTemplate = "ALTER SYSTEM DELETE SERVER '${SERVER_IP}';"
+
+	GetRootServiceSQL = "SELECT zone, svr_ip, svr_port, role, partition_id, partition_cnt FROM __all_virtual_core_meta_table;"
+
+	GetRSJobStatusSQL = "SELECT job_status, return_code, progress FROM __all_rootservice_job WHERE job_type = 'DELETE_SERVER' AND svr_ip = '${DELETE_SERVER_IP}' AND svr_port = '${DELETE_SERVER_PORT}';"
+)

pkg/controllers/observer/sql/error.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+import (
+	"github.com/go-sql-driver/mysql"
+	"k8s.io/klog/v2"
+)
+
+func covertErrToMySQLError(err error) (uint16, string) {
+	mysqlErr, ok := err.(*mysql.MySQLError)
+	if ok {
+		return mysqlErr.Number, mysqlErr.Message
+	}
+	klog.Errorln(err)
+	return 0, ""
+}

pkg/controllers/observer/sql/sql_builder.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+import (
+	"strconv"
+	"strings"
+)
+
+func ReplaceAll(template string, replacers ...*strings.Replacer) string {
+	s := template
+	for _, replacer := range replacers {
+		s = replacer.Replace(s)
+	}
+	return s
+}
+
+func SetServerOfflineTimeSQLReplacer(offlineTime int) *strings.Replacer {
+	return strings.NewReplacer("${OFFLINE_TIME}", strconv.Itoa(offlineTime))
+}
+
+func AddServerSQLReplacer(zoneName, serverIP string) *strings.Replacer {
+	return strings.NewReplacer("${SERVER_IP}", serverIP, "${ZONE_NAME}", zoneName)
+}
+
+func DelServerSQLReplacer(serverIP string) *strings.Replacer {
+	return strings.NewReplacer("${SERVER_IP}", serverIP)
+}
+
+func GetRSJobStatusSQLReplacer(serverIP, port string) *strings.Replacer {
+	return strings.NewReplacer("${DELETE_SERVER_IP}", serverIP, "${DELETE_SERVER_PORT}", port)
+}

pkg/controllers/observer/sql/sql_executer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func ExecSQL(ip, port, db, SQL string, timeout int) error {
+	klog.Infoln(SQL)
+	client := ConnOB(ip, port, db, timeout)
+	if client != nil {
+		defer client.Close()
+		res := client.Exec(SQL)
+		if res.Error != nil {
+			errNum, errMsg := covertErrToMySQLError(res.Error)
+			klog.Errorln(errNum, errMsg)
+			return errors.New(errMsg)
+		}
+	}
+	return nil
+}
+
+func GetOBServerFromDB(ip, port, db, SQL string) []model.AllServer {
+	client := ConnOB(ip, port, db, 5)
+	res := make([]model.AllServer, 0)
+	if client != nil {
+		defer client.Close()
+		rows, err := client.Model(&model.AllServer{}).Raw(SQL).Rows()
+		defer rows.Close()
+		var rowData model.AllServer
+		for rows.Next() {
+			err = client.ScanRows(rows, &rowData)
+			if err == nil {
+				res = append(res, rowData)
+			}
+		}
+	}
+	return res
+}
+
+func GetRootServiceFromDB(ip, port, db, SQL string) []model.AllVirtualCoreMeta {
+	client := ConnOB(ip, port, db, 5)
+	res := make([]model.AllVirtualCoreMeta, 0)
+	if client != nil {
+		defer client.Close()
+		rows, err := client.Model(&model.AllVirtualCoreMeta{}).Raw(SQL).Rows()
+		defer rows.Close()
+		var rowData model.AllVirtualCoreMeta
+		for rows.Next() {
+			err = client.ScanRows(rows, &rowData)
+			if err == nil {
+				res = append(res, rowData)
+			}
+		}
+	}
+	return res
+}
+
+func GetRSJobStatusFromDB(ip, port, db, SQL string) []model.RSJobStatus {
+	client := ConnOB(ip, port, db, 5)
+	res := make([]model.RSJobStatus, 0)
+	if client != nil {
+		defer client.Close()
+		rows, err := client.Model(&model.RSJobStatus{}).Raw(SQL).Rows()
+		defer rows.Close()
+		var rowData model.RSJobStatus
+		for rows.Next() {
+			err = client.ScanRows(rows, &rowData)
+			if err == nil {
+				res = append(res, rowData)
+			}
+		}
+	}
+	return res
+}

pkg/controllers/observer/sql/sql_operator.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package sql
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/ob"
+)
+
+func SetServerOfflineTime(clusterIP string, offlineTime int) error {
+	sql := ReplaceAll(SetServerOfflineTimeSQLTemplate, SetServerOfflineTimeSQLReplacer(offlineTime))
+	return ExecSQL(clusterIP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, sql, 5)
+}
+
+func BootstrapForOB(IP, SQL string) {
+	setTimeOutRes := ExecSQL(IP, ob.OBSERVER_MYSQL_PORT, "", SetTimeoutSQL, 5)
+	if setTimeOutRes != nil {
+		klog.Errorln("set ob_query_timeout error", setTimeOutRes)
+	}
+	bootstrapRes := ExecSQL(IP, ob.OBSERVER_MYSQL_PORT, "", SQL, 300)
+	if bootstrapRes != nil {
+		klog.Errorln("run bootstrap sql error", bootstrapRes)
+	}
+}
+
+func GetOBServer(IP string) []model.AllServer {
+	return GetOBServerFromDB(IP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, GetOBServerSQL)
+}
+
+func AddServer(clusterIP, zoneName, podIP string) error {
+	serverIP := fmt.Sprintf("%s:%s", podIP, ob.OBSERVER_RPC_PORT)
+	sql := ReplaceAll(AddServerSQLTemplate, AddServerSQLReplacer(zoneName, serverIP))
+	return ExecSQL(clusterIP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, sql, 60)
+}
+
+func DelServer(clusterIP, podIP string) error {
+	serverIP := fmt.Sprintf("%s:%s", podIP, ob.OBSERVER_RPC_PORT)
+	sql := ReplaceAll(DelServerSQLTemplate, DelServerSQLReplacer(serverIP))
+	return ExecSQL(clusterIP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, sql, 60)
+}
+
+func GetRootService(IP string) []model.AllVirtualCoreMeta {
+	return GetRootServiceFromDB(IP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, GetRootServiceSQL)
+}
+
+func GetRSJobStatus(clusterIP, podIP string) []model.RSJobStatus {
+	sql := ReplaceAll(GetRSJobStatusSQL, GetRSJobStatusSQLReplacer(podIP, ob.OBSERVER_RPC_PORT))
+	return GetRSJobStatusFromDB(clusterIP, ob.OBSERVER_MYSQL_PORT, DatabaseOb, sql)
+}

pkg/controllers/statefulapp/const/controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulappconst
+
+// controller
+const (
+	ControllerName       = "statefulapp-controller"
+	ConcurrentReconciles = 3
+)

pkg/controllers/statefulapp/const/pipeline.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulappconst
+
+// scale state machine
+const (
+	ScaleUP   = "Scale UP"
+	ScaleDown = "Scale Down"
+	Maintain  = "Maintain"
+)

pkg/controllers/statefulapp/const/pod.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulappconst
+
+// Pod status
+const (
+	PodStatusRunning = "Running"
+)

pkg/controllers/statefulapp/const/pvc.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulappconst
+
+// PVC status
+const (
+	Bound = "Bound"
+)

pkg/controllers/statefulapp/const/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulappconst
+
+// StatefulApp Status
+const (
+	Prepareing = "Prepareing"
+	Ready      = "Ready"
+)

pkg/controllers/statefulapp/core/converter/pod.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+
+	corev1 "k8s.io/api/core/v1"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	statefulapputil "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/util"
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func GeneratePodName(statefulAppName, clusterName, subsetName string, podIndex int) string {
+	return fmt.Sprintf("%s-%s-%s-%d", statefulAppName, clusterName, subsetName, podIndex)
+}
+
+func GeneratePodsIndexList(expectedReplicas int) []int {
+	res := make([]int, 0)
+	for i := 0; i < expectedReplicas; i++ {
+		res = append(res, i)
+	}
+	return res
+}
+
+func GetPodsIndexList(pods []corev1.Pod) []int {
+	res := make([]int, 0)
+	for _, pod := range pods {
+		index, _ := strconv.Atoi(pod.Labels["index"])
+		res = append(res, index)
+	}
+	return res
+}
+
+func FindMissingIndex(subset cloudv1.Subset, pod []corev1.Pod) int {
+	podsCurrentIndexList := GetPodsIndexList(pod)
+	podsExpectedIndexList := GeneratePodsIndexList(int(subset.Replicas))
+	index := util.CompareSlice(podsCurrentIndexList, podsExpectedIndexList)
+	return index
+}
+
+func GetDeleteIndex(statefulApp cloudv1.StatefulApp, subset cloudv1.Subset) int {
+	// TODO: support specify the index
+	index := subset.Replicas
+	return int(index)
+}
+
+func GeneratePodObject(statefulApp cloudv1.StatefulApp, subset cloudv1.Subset, podsCurrent []corev1.Pod) (string, int, corev1.Pod) {
+	var podIndex int
+	// get pod index
+	if len(podsCurrent) == 0 {
+		// new subset
+		podIndex = 0
+	} else {
+		// find the missing index
+		podIndex = FindMissingIndex(subset, podsCurrent)
+	}
+	// get pod name
+	podName := GeneratePodName(statefulApp.Name, myconfig.ClusterName, subset.Name, podIndex)
+	// generate
+	podObject := GeneratePodObjectPcress(subset.Name, podName, podIndex, statefulApp, subset.NodeSelector)
+	return podName, podIndex, podObject
+}
+
+func GeneratePodObjectPcress(subsetName, podName string, podIndex int, statefulApp cloudv1.StatefulApp, nodeSelect map[string]string) corev1.Pod {
+	objectMeta := statefulapputil.GenerateObjectMeta(subsetName, podName, podIndex, statefulApp)
+	// TODO: support PodSpecial
+	// DeepCopy
+	podTemplate := statefulApp.DeepCopy().Spec.PodTemplate
+
+	podTemplate.NodeSelector = nodeSelect
+
+	// pvc rewrite
+	volumes := podTemplate.Volumes
+	if len(volumes) > 0 {
+		podTemplate.Volumes = PVCRewrite(podName, volumes)
+	}
+
+	pod := corev1.Pod{
+		ObjectMeta: objectMeta,
+		Spec:       podTemplate,
+	}
+	return pod
+}
+
+func PVCRewrite(podName string, volumes []corev1.Volume) []corev1.Volume {
+	newVolumes := make([]corev1.Volume, 0)
+	for _, volume := range volumes {
+		name := volume.PersistentVolumeClaim.ClaimName
+		newName := GeneratePVCName(podName, name)
+		volume.PersistentVolumeClaim.ClaimName = newName
+		newVolumes = append(newVolumes, volume)
+	}
+	return newVolumes
+}
+
+func PodListToPods(podList corev1.PodList) []corev1.Pod {
+	res := make([]corev1.Pod, 0)
+	if len(podList.Items) > 0 {
+		res = podList.Items
+	}
+	return res
+}
+
+func SortPodsDesc(pods []corev1.Pod) []corev1.Pod {
+	sort.Slice(pods, func(i, j int) bool {
+		indexI, _ := strconv.Atoi(pods[i].Labels["index"])
+		indexJ, _ := strconv.Atoi(pods[j].Labels["index"])
+		if indexI > indexJ {
+			return true
+		}
+		return false
+	})
+	return pods
+}
+
+func PodCurrentStatusToPodStatus(pod corev1.Pod) cloudv1.PodStatus {
+	var podStatus cloudv1.PodStatus
+	podStatus.Name = pod.Name
+	podStatus.Index, _ = strconv.Atoi(pod.Labels["index"])
+	podStatus.PodPhase = pod.Status.Phase
+	podStatus.PodIP = pod.Status.PodIP
+	podStatus.NodeIP = pod.Status.HostIP
+	return podStatus
+}
+
+func SortPodsStatus(podsStatus []cloudv1.PodStatus) []cloudv1.PodStatus {
+	sort.Slice(podsStatus, func(i, j int) bool {
+		if podsStatus[i].Index < podsStatus[j].Index {
+			return true
+		}
+		return false
+	})
+	return podsStatus
+}

pkg/controllers/statefulapp/core/converter/pvc.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulapputil "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/util"
+)
+
+// pvc name: podname-pvcname
+func GeneratePVCName(podName, volumeName string) string {
+	pvcName := fmt.Sprintf("%s-%s", podName, volumeName)
+	return pvcName
+}
+
+func GeneratePVCsObject(subsetName, podName string, podIndex int, statefulApp cloudv1.StatefulApp) []corev1.PersistentVolumeClaim {
+	// TODO: support PVCSpecial
+	res := make([]corev1.PersistentVolumeClaim, 0)
+	pvcTemplates := statefulApp.Spec.StorageTemplates
+	for _, pvcTemplate := range pvcTemplates {
+		pvcName := GeneratePVCName(podName, pvcTemplate.Name)
+		objectMeta := statefulapputil.GenerateObjectMeta(subsetName, pvcName, podIndex, statefulApp)
+		pvc := corev1.PersistentVolumeClaim{
+			ObjectMeta: objectMeta,
+			Spec:       pvcTemplate.PVC,
+		}
+		res = append(res, pvc)
+	}
+	return res
+}
+
+func PVCCurrentStatusToPVCStatus(pvc corev1.PersistentVolumeClaim) cloudv1.PVCStatus {
+	var pvcStatus cloudv1.PVCStatus
+	pvcStatus.Name = pvc.Name
+	pvcStatus.Phase = corev1.PersistentVolumePhase(pvc.Status.Phase)
+	return pvcStatus
+}

pkg/controllers/statefulapp/core/converter/subset.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func GenerateSubsetStatus(subsetName, regionName string, expectedReplicas, availableReplicas int32, podsStatus []cloudv1.PodStatus) cloudv1.SubsetStatus {
+	var subsetStatus cloudv1.SubsetStatus
+	subsetStatus.Name = subsetName
+	if regionName != "" {
+		subsetStatus.Region = regionName
+	}
+	subsetStatus.ExpectedReplicas = expectedReplicas
+	subsetStatus.AvailableReplicas = availableReplicas
+	subsetStatus.Pods = podsStatus
+	return subsetStatus
+}
+
+func FindElementNotInSubsetsCurrentNameList(subsetsSpec []cloudv1.Subset, subsetsCurrentNameList []string) string {
+	var res string
+	for _, subsetSpec := range subsetsSpec {
+		var matchStatus bool
+		matchStatus = false
+		for _, subsetName := range subsetsCurrentNameList {
+			if subsetSpec.Name == subsetName {
+				matchStatus = true
+			}
+		}
+		if !matchStatus {
+			res = subsetSpec.Name
+			break
+		}
+	}
+	return res
+}
+
+func FindElementNotInSubsetsSpec(subsetsSpec []cloudv1.Subset, subsetsCurrentNameList []string) string {
+	var res string
+	for _, subsetName := range subsetsCurrentNameList {
+		var matchStatus bool
+		matchStatus = false
+		for _, subsetSpec := range subsetsSpec {
+			if subsetSpec.Name == subsetName {
+				matchStatus = true
+			}
+		}
+		if !matchStatus {
+			res = subsetName
+			break
+		}
+	}
+	return res
+}
+
+func GetSubsetMapFromPods(pods []corev1.Pod) map[string]bool {
+	subsetMap := make(map[string]bool)
+	for _, pod := range pods {
+		key := pod.Labels["subset"]
+		subsetMap[key] = true
+	}
+	return subsetMap
+}

pkg/controllers/statefulapp/core/ctrl.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+type UniversalCtrl struct {
+	Client      client.Client
+	StatefulApp cloudv1.StatefulApp
+}
+
+func NewUniversalCtrl(client client.Client, statefulApp cloudv1.StatefulApp) UniversalCtrl {
+	return UniversalCtrl{
+		Client:      client,
+		StatefulApp: statefulApp,
+	}
+}

pkg/controllers/statefulapp/core/judge/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package judge
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/converter"
+)
+
+func SubsetScaleJundge(subsetsSpec []cloudv1.Subset, subsetsCurrentNameList []string) (string, string) {
+	var scaleState string
+	var subsetName string
+
+	// new cr
+	if len(subsetsCurrentNameList) == 0 {
+		return statefulappconst.ScaleUP, subsetsSpec[0].Name
+	}
+
+	if len(subsetsSpec) > len(subsetsCurrentNameList) {
+		scaleState = statefulappconst.ScaleUP
+		// find which subsets need to add
+		subsetName = converter.FindElementNotInSubsetsCurrentNameList(subsetsSpec, subsetsCurrentNameList)
+	} else if len(subsetsSpec) < len(subsetsCurrentNameList) {
+		scaleState = statefulappconst.ScaleDown
+		// find which subsets need to delete
+		subsetName = converter.FindElementNotInSubsetsSpec(subsetsSpec, subsetsCurrentNameList)
+	} else {
+		scaleState = statefulappconst.Maintain
+	}
+
+	return scaleState, subsetName
+}

pkg/controllers/statefulapp/core/judge/subset.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package judge
+
+import (
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+)
+
+func PodScale(expectedReplicas, availableReplicas int) string {
+	var scaleState string
+	if expectedReplicas > availableReplicas {
+		scaleState = statefulappconst.ScaleUP
+	} else if expectedReplicas < availableReplicas {
+		scaleState = statefulappconst.ScaleDown
+	} else {
+		scaleState = statefulappconst.Maintain
+	}
+	return scaleState
+}

pkg/controllers/statefulapp/core/pod_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	myconfig "github.com/oceanbase/ob-operator/pkg/config"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type PodCtrl struct {
+	Resource    *resource.Resource
+	StatefulApp cloudv1.StatefulApp
+}
+
+type PodCtrlOperator interface {
+	PodsCoordinator(subset cloudv1.Subset, subsetPodsCurrent []corev1.Pod) (bool, error)
+	CreatePod(subset cloudv1.Subset) error
+	GetPodsByLables(namespace string, listOption client.ListOption) []corev1.Pod
+	GetPodsByApp(namespace, name string) []corev1.Pod
+	GetPodsBySubset(namespace, name, subsetName string) []corev1.Pod
+	GetPodsStatusBySubset(namespace, name, subsetName string) []cloudv1.PodStatus
+	DeletePod(subset cloudv1.Subset) error
+	DeletePodList(pods []corev1.Pod) error
+}
+
+func NewPodCtrl(client client.Client, recorder record.EventRecorder, statefulApp cloudv1.StatefulApp) PodCtrlOperator {
+	ctrlResource := resource.NewResource(client, recorder)
+	return &PodCtrl{
+		Resource:    ctrlResource,
+		StatefulApp: statefulApp,
+	}
+}
+
+func (ctrl *PodCtrl) PodsCoordinator(subset cloudv1.Subset, subsetPodsCurrent []corev1.Pod) (bool, error) {
+	var podNeedUpgrade bool
+	podNeedUpgrade = false
+
+	// index from large to small
+	pods := converter.SortPodsDesc(subsetPodsCurrent)
+	for _, pod := range pods {
+		// TODO: support upgrade
+		// compare spec to determine whether an upgrade is required
+		if podNeedUpgrade {
+			klog.Errorln("Not support upgrade pod yet", pod)
+		}
+	}
+
+	return UpdateStatusWithEqual(ctrl.Resource, ctrl.StatefulApp)
+}
+
+func (ctrl *PodCtrl) CreatePod(subset cloudv1.Subset) error {
+	var err error
+
+	podsCurrent := ctrl.GetPodsBySubset(ctrl.StatefulApp.Namespace, ctrl.StatefulApp.Name, subset.Name)
+
+	podName, podIndex, podObject := converter.GeneratePodObject(ctrl.StatefulApp, subset, podsCurrent)
+
+	// create pod
+	podExecuter := resource.NewPodResource(ctrl.Resource)
+	err = podExecuter.Create(context.TODO(), podObject)
+	if err != nil {
+		if kubeerrors.IsAlreadyExists(err) {
+			klog.Errorln("pod is already exist, need recreate")
+			// delete pod
+			err = podExecuter.Delete(context.TODO(), podObject)
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	}
+
+	// create pvc
+	pvcCtrl := NewPVCCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+	err = pvcCtrl.CreatePVCs(subset, podName, podIndex)
+	if err != nil {
+		klog.Errorln("pvc is already exist, need recreate")
+		// delete pvc
+		err = pvcCtrl.DeletePVCs(podObject)
+		if err != nil {
+			return err
+		}
+		// delete pod
+		err = podExecuter.Delete(context.TODO(), podObject)
+		if err != nil {
+			return err
+		}
+		return err
+	}
+
+	// update status
+	if err == nil {
+		err = UpdateStatus(ctrl.Resource, ctrl.StatefulApp)
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+func (ctrl *PodCtrl) GetPodsByLables(namespace string, listOption client.ListOption) []corev1.Pod {
+	podExecuter := resource.NewPodResource(ctrl.Resource)
+	podList := podExecuter.List(context.TODO(), namespace, listOption)
+	return converter.PodListToPods(podList.(corev1.PodList))
+}
+
+func (ctrl *PodCtrl) GetPodsByApp(namespace, name string) []corev1.Pod {
+	listOption := client.MatchingLabels{
+		"app": name,
+	}
+	return ctrl.GetPodsByLables(namespace, listOption)
+}
+
+func (ctrl *PodCtrl) GetPodsBySubset(namespace, name, subsetName string) []corev1.Pod {
+	listOption := client.MatchingLabels{
+		"app":    name,
+		"subset": subsetName,
+	}
+	return ctrl.GetPodsByLables(namespace, listOption)
+}
+
+func (ctrl *PodCtrl) GetPodsStatusBySubset(namespace, name, subsetName string) []cloudv1.PodStatus {
+	res := make([]cloudv1.PodStatus, 0)
+	podList := ctrl.GetPodsBySubset(namespace, name, subsetName)
+	if len(podList) > 0 {
+		for _, pod := range podList {
+			podStatus := converter.PodCurrentStatusToPodStatus(pod)
+			res = append(res, podStatus)
+		}
+	}
+	return res
+}
+
+func (ctrl *PodCtrl) DeletePod(subset cloudv1.Subset) error {
+	var err error
+	var podIndex int
+	var podName string
+
+	podIndex = converter.GetDeleteIndex(ctrl.StatefulApp, subset)
+	podName = converter.GeneratePodName(ctrl.StatefulApp.Name, myconfig.ClusterName, subset.Name, podIndex)
+	pods := ctrl.GetPodsBySubset(ctrl.StatefulApp.Namespace, ctrl.StatefulApp.Name, subset.Name)
+
+	// zero
+	if len(pods) == 1 {
+		return errors.New("can't scale pods to zero")
+	}
+
+	// delete pod
+	podExecuter := resource.NewPodResource(ctrl.Resource)
+	pvcCtrl := NewPVCCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+	for _, pod := range pods {
+		if pod.Name == podName {
+			// TODO: support PVReclaimPolicy
+			err = podExecuter.Delete(context.TODO(), pod)
+			if err == nil {
+				err = pvcCtrl.DeletePVCs(pod)
+			}
+			break
+		}
+	}
+
+	// update status
+	if err == nil {
+		err = UpdateStatus(ctrl.Resource, ctrl.StatefulApp)
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+func (ctrl *PodCtrl) DeletePodList(pods []corev1.Pod) error {
+	podExecuter := resource.NewPodResource(ctrl.Resource)
+	var err error
+	for _, pod := range pods {
+		err = podExecuter.Delete(context.TODO(), pod)
+	}
+	return err
+}

pkg/controllers/statefulapp/core/pvc_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type PVCCtrl struct {
+	Resource    *resource.Resource
+	StatefulApp cloudv1.StatefulApp
+}
+
+type PVCCtrlOperator interface {
+	CreatePVCs(subset cloudv1.Subset, podName string, podIndex int) error
+	GetPVCsStatus(pod corev1.Pod) []cloudv1.PVCStatus
+	DeletePVCs(pod corev1.Pod) error
+}
+
+func NewPVCCtrl(client client.Client, recorder record.EventRecorder, statefulApp cloudv1.StatefulApp) PVCCtrlOperator {
+	ctrlResource := resource.NewResource(client, recorder)
+	return &PVCCtrl{
+		Resource:    ctrlResource,
+		StatefulApp: statefulApp,
+	}
+}
+
+func (ctrl *PVCCtrl) CreatePVCs(subset cloudv1.Subset, podName string, podIndex int) error {
+	pvcs := converter.GeneratePVCsObject(subset.Name, podName, podIndex, ctrl.StatefulApp)
+	pvcExecuter := resource.NewPVCResource(ctrl.Resource)
+	err := pvcExecuter.Create(context.TODO(), pvcs)
+	return err
+}
+
+func (ctrl *PVCCtrl) GetPVCsStatus(pod corev1.Pod) []cloudv1.PVCStatus {
+	res := make([]cloudv1.PVCStatus, 0)
+	pvcExecuter := resource.NewPVCResource(ctrl.Resource)
+	for _, volume := range pod.Spec.Volumes {
+		// pvc type
+		if volume.PersistentVolumeClaim != nil {
+			pvcName := volume.PersistentVolumeClaim.ClaimName
+			pvcCurrent, err := pvcExecuter.Get(context.TODO(), pod.Namespace, pvcName)
+			if err == nil {
+				pvcStatus := converter.PVCCurrentStatusToPVCStatus(pvcCurrent.(corev1.PersistentVolumeClaim))
+				res = append(res, pvcStatus)
+			}
+		}
+	}
+	return res
+}
+
+func (ctrl *PVCCtrl) DeletePVCs(pod corev1.Pod) error {
+	var pvc corev1.PersistentVolumeClaim
+	var pvcInterface interface{}
+	var err error
+	pvcExecuter := resource.NewPVCResource(ctrl.Resource)
+	for _, volume := range pod.Spec.Volumes {
+		// pvc type
+		if volume.PersistentVolumeClaim != nil {
+			pvcName := volume.PersistentVolumeClaim.ClaimName
+			pvcInterface, err = pvcExecuter.Get(context.TODO(), pod.Namespace, pvcName)
+			pvc = pvcInterface.(corev1.PersistentVolumeClaim)
+			if err == nil {
+				err = pvcExecuter.Delete(context.TODO(), pvc)
+				if err != nil {
+					break
+				}
+			}
+		}
+	}
+	return err
+}

pkg/controllers/statefulapp/core/statefulapp_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/judge"
+)
+
+// StatefulAppReconciler reconciles a StatefulApp object
+type StatefulAppReconciler struct {
+	CRClient client.Client
+	Scheme   *runtime.Scheme
+	// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/events/event.go
+	Recorder record.EventRecorder
+}
+
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=cloud.oceanbase.com,resources=statefulapps/sfinalizers,verbs=update
+// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=update
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
+
+func (r *StatefulAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Fetch the CR instance
+	instance := &cloudv1.StatefulApp{}
+	err := r.CRClient.Get(ctx, req.NamespacedName, instance)
+	if err != nil {
+		if kubeerrors.IsNotFound(err) {
+			// Object not found, return.
+			// Created objects are automatically garbage collected.
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object, requeue the request.
+		return reconcile.Result{}, err
+	}
+	// custom logic
+	return r.StatefulAppCoordinator(*instance)
+}
+
+// StatefulAppCoordinator is the entry function for control-plan logic
+func (r *StatefulAppReconciler) StatefulAppCoordinator(statefulApp cloudv1.StatefulApp) (ctrl.Result, error) {
+	var err error
+	subsetCtrl := NewSubsetCtrl(r.CRClient, r.Recorder, statefulApp)
+
+	subsetsSpec := statefulApp.Spec.Subsets
+	subsetsCurrentNameList := subsetCtrl.GetSubsetsNameList()
+
+	// ScaleUP, add subset
+	// ScaleDown, delete subsets
+	// Maintain, update cluster status
+	scaleState, subsetSpecName := judge.SubsetScaleJundge(subsetsSpec, subsetsCurrentNameList)
+	switch scaleState {
+	case statefulappconst.ScaleUP:
+		// create subset
+		err = subsetCtrl.CreateSubset(subsetSpecName, subsetsSpec)
+	case statefulappconst.ScaleDown:
+		// delete subset
+		err = subsetCtrl.DeleteSubset(subsetSpecName, subsetsCurrentNameList)
+	case statefulappconst.Maintain:
+		err = r.SubsetsCoordinator(statefulApp, subsetsSpec)
+	}
+
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	return reconcile.Result{}, nil
+}

pkg/controllers/statefulapp/core/status.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"context"
+	"reflect"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+func UpdateStatus(ctrlResource *resource.Resource, statefulApp cloudv1.StatefulApp) error {
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrlResource)
+	// use retry to update
+	retryErr := retry.RetryOnConflict(
+		retry.DefaultRetry,
+		func() error {
+			// get current StatefulApp every time
+			statefulAppTemp, err := statefulAppExecuter.Get(context.TODO(), statefulApp.Namespace, statefulApp.Name)
+			if err != nil {
+				return err
+			}
+			// DeepCopy
+			statefulAppCurrent := statefulAppTemp.(cloudv1.StatefulApp)
+			statefulAppCurrentDeepCopy := statefulAppCurrent.DeepCopy()
+			// build status
+			statefulAppNew := buildStatus(ctrlResource, *statefulAppCurrentDeepCopy)
+			// update status
+			err = statefulAppExecuter.UpdateStatus(context.TODO(), statefulAppNew)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+	if retryErr != nil {
+		klog.Errorln(retryErr)
+		return retryErr
+	}
+	kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "update status", statefulApp.Status)
+	return nil
+}
+
+func UpdateStatusWithEqual(ctrlResource *resource.Resource, statefulApp cloudv1.StatefulApp) (bool, error) {
+	statefulAppExecuter := resource.NewStatefulAppResource(ctrlResource)
+	// get current
+	statefulAppTemp, err := statefulAppExecuter.Get(context.TODO(), statefulApp.Namespace, statefulApp.Name)
+	if err != nil {
+		// this one is broken, try next
+		return true, err
+	}
+	// DeepCopy
+	statefulAppCurrent := statefulAppTemp.(cloudv1.StatefulApp)
+	statefulAppCurrentDeepCopy := statefulAppCurrent.DeepCopy()
+	// build status
+	statefulAppNew := buildStatus(ctrlResource, *statefulAppCurrentDeepCopy)
+	// compare status, if Equal don't need update
+	compareStatus := reflect.DeepEqual(statefulAppCurrent.Status, statefulAppNew.Status)
+	if compareStatus {
+		return compareStatus, nil
+	}
+	// update status
+	err = statefulAppExecuter.UpdateStatus(context.TODO(), statefulAppNew)
+	if err != nil {
+		klog.Errorln(err)
+		// this one is broken, try next
+		return true, err
+	}
+	kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "update status", statefulApp.Status)
+	// do one thing at a time
+	return false, nil
+}
+
+// update all status every time
+func buildStatus(ctrlResource *resource.Resource, statefulApp cloudv1.StatefulApp) cloudv1.StatefulApp {
+	subsetsStatus := make([]cloudv1.SubsetStatus, 0)
+	podCtrl := NewPodCtrl(ctrlResource.Client, ctrlResource.Recorder, statefulApp)
+	pvcCtrl := NewPVCCtrl(ctrlResource.Client, ctrlResource.Recorder, statefulApp)
+	for _, subset := range statefulApp.Spec.Subsets {
+		subsetStatus := buildSubsetStatus(podCtrl, pvcCtrl, statefulApp, subset)
+		subsetsStatus = append(subsetsStatus, subsetStatus)
+	}
+	// check cluster status
+	clusterStatus := checkClusterStatus(statefulApp)
+	statefulApp = generateStatefulAppStatus(statefulApp.Spec.Cluster, clusterStatus, subsetsStatus, statefulApp)
+	return statefulApp
+}
+
+func buildSubsetStatus(podCtrl PodCtrlOperator, pvcCtrl PVCCtrlOperator, statefulApp cloudv1.StatefulApp, subset cloudv1.Subset) cloudv1.SubsetStatus {
+	// pod status
+	var replicas int
+	podsStatus := make([]cloudv1.PodStatus, 0)
+	pods := podCtrl.GetPodsBySubset(statefulApp.Namespace, statefulApp.Name, subset.Name)
+	if len(pods) > 0 {
+		for _, pod := range pods {
+			podStatus := buildPodStatus(pvcCtrl, pod)
+			podsStatus = append(podsStatus, podStatus)
+			if podStatus.PodPhase == statefulappconst.PodStatusRunning {
+				replicas += 1
+			}
+		}
+	}
+	podsStatus = converter.SortPodsStatus(podsStatus)
+	subsetStatus := converter.GenerateSubsetStatus(subset.Name, subset.Region, subset.Replicas, int32(replicas), podsStatus)
+	return subsetStatus
+}
+
+func buildPodStatus(pvcCtrl PVCCtrlOperator, pod corev1.Pod) cloudv1.PodStatus {
+	podStatus := converter.PodCurrentStatusToPodStatus(pod)
+	pvcs := pvcCtrl.GetPVCsStatus(pod)
+	if len(pvcs) > 0 {
+		podStatus.PVCs = pvcs
+	}
+	return podStatus
+}
+
+func checkClusterStatus(statefulApp cloudv1.StatefulApp) string {
+	var status string
+	status = statefulappconst.Ready
+	// subset number is right
+	if len(statefulApp.Status.Subsets) == len(statefulApp.Spec.Subsets) {
+		for _, subset := range statefulApp.Status.Subsets {
+			// replica number is right
+			if subset.ExpectedReplicas != subset.AvailableReplicas {
+				status = statefulappconst.Prepareing
+				break
+			}
+			// pod status is Running
+			for _, pod := range subset.Pods {
+				if pod.PodPhase != statefulappconst.PodStatusRunning {
+					status = statefulappconst.Prepareing
+					break
+				}
+				// pvc status is Bound
+				for _, pv := range pod.PVCs {
+					if pv.Phase != statefulappconst.Bound {
+						status = statefulappconst.Prepareing
+						break
+					}
+				}
+			}
+		}
+	}
+	return status
+}
+
+func generateStatefulAppStatus(clusterName, clusterStatus string, topology []cloudv1.SubsetStatus, statefulApp cloudv1.StatefulApp) cloudv1.StatefulApp {
+	statefulApp.Status.Cluster = clusterName
+	statefulApp.Status.ClusterStatus = clusterStatus
+	statefulApp.Status.Subsets = topology
+	return statefulApp
+}

pkg/controllers/statefulapp/core/subset_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core/judge"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube/resource"
+)
+
+type SubsetCtrl struct {
+	Resource    *resource.Resource
+	StatefulApp cloudv1.StatefulApp
+}
+
+type SubsetCtrlOperator interface {
+	SubsetCoordinator(ubset cloudv1.Subset) (bool, error)
+	CreateSubset(subsetSpecName string, subsetsSpec []cloudv1.Subset) error
+	GetSubsetsNameList() []string
+	DeleteSubset(subsetSpecName string, subsetsCurrent []string) error
+}
+
+func NewSubsetCtrl(client client.Client, recorder record.EventRecorder, statefulApp cloudv1.StatefulApp) SubsetCtrlOperator {
+	ctrlResource := resource.NewResource(client, recorder)
+	return &SubsetCtrl{
+		Resource:    ctrlResource,
+		StatefulApp: statefulApp,
+	}
+}
+
+func (ctrl *SubsetCtrl) SubsetCoordinator(subset cloudv1.Subset) (bool, error) {
+	var compareStatus bool
+	var err error
+
+	podCtrl := NewPodCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+
+	compareStatus = false
+
+	subsetPodsCurrent := podCtrl.GetPodsBySubset(ctrl.StatefulApp.Namespace, ctrl.StatefulApp.Name, subset.Name)
+
+	scaleState := judge.PodScale(int(subset.Replicas), len(subsetPodsCurrent))
+	switch scaleState {
+	case statefulappconst.ScaleUP:
+		err = podCtrl.CreatePod(subset)
+	case statefulappconst.ScaleDown:
+		err = podCtrl.DeletePod(subset)
+	case statefulappconst.Maintain:
+		compareStatus, err = podCtrl.PodsCoordinator(subset, subsetPodsCurrent)
+	}
+
+	return compareStatus, err
+}
+
+func (ctrl *SubsetCtrl) CreateSubset(subsetSpecName string, subsetsSpec []cloudv1.Subset) error {
+	var err error
+	podCtrl := NewPodCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+	for _, subsetSpec := range subsetsSpec {
+		if subsetSpec.Name == subsetSpecName {
+			// create one pod, then use Maintain to create more pods
+			err = podCtrl.CreatePod(subsetSpec)
+		}
+	}
+	return err
+}
+
+func (ctrl *SubsetCtrl) GetSubsetsNameList() []string {
+	res := make([]string, 0)
+	podCtrl := NewPodCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+	pods := podCtrl.GetPodsByApp(ctrl.StatefulApp.Namespace, ctrl.StatefulApp.Name)
+	if len(pods) == 0 {
+		return res
+	}
+	subsetMap := converter.GetSubsetMapFromPods(pods)
+	for k := range subsetMap {
+		res = append(res, k)
+	}
+	return res
+}
+
+func (ctrl *SubsetCtrl) DeleteSubset(subsetSpecName string, subsetsCurrent []string) error {
+	var err error
+	podCtrl := NewPodCtrl(ctrl.Resource.Client, ctrl.Resource.Recorder, ctrl.StatefulApp)
+	// zero
+	if len(subsetsCurrent) == 1 {
+		return errors.New("can't scale subsets to zero")
+	}
+	for _, subsetName := range subsetsCurrent {
+		if subsetName == subsetSpecName {
+			pods := podCtrl.GetPodsBySubset(ctrl.StatefulApp.Namespace, ctrl.StatefulApp.Name, subsetName)
+			err = podCtrl.DeletePodList(pods)
+		}
+	}
+	return err
+}

pkg/controllers/statefulapp/core/subsets_controller.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package core
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func (r *StatefulAppReconciler) SubsetsCoordinator(statefulApp cloudv1.StatefulApp, subsetsSpec []cloudv1.Subset) error {
+	var err error
+	var status bool
+	subsetCtrl := NewSubsetCtrl(r.CRClient, r.Recorder, statefulApp)
+	for _, subset := range subsetsSpec {
+		status, err = subsetCtrl.SubsetCoordinator(subset)
+		// compare status
+		// if true continue, don't need to update, but it is not ok
+		// if false beark, do one thing at a time
+		if !status || err != nil {
+			break
+		}
+	}
+	return err
+}

pkg/controllers/statefulapp/core/util/util.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"strconv"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+func GenerateOwnerReference(statefulApp cloudv1.StatefulApp) metav1.OwnerReference {
+	ownerReference := metav1.OwnerReference{
+		APIVersion: statefulApp.APIVersion,
+		Kind:       statefulApp.Kind,
+		Name:       statefulApp.Name,
+		UID:        statefulApp.UID,
+	}
+	return ownerReference
+}
+
+func GenerateLabels(app, subset, index string) map[string]string {
+	labels := make(map[string]string)
+	labels["app"] = app
+	labels["subset"] = subset
+	labels["index"] = index
+	return labels
+}
+
+func GenerateObjectMeta(subsetName string, name string, index int, statefulApp cloudv1.StatefulApp) metav1.ObjectMeta {
+	ownerReference := GenerateOwnerReference(statefulApp)
+	labels := GenerateLabels(statefulApp.Name, subsetName, strconv.Itoa(index))
+	objectMeta := metav1.ObjectMeta{
+		Name:            name,
+		Namespace:       statefulApp.Namespace,
+		OwnerReferences: []metav1.OwnerReference{ownerReference},
+		Labels:          labels,
+	}
+	return objectMeta
+}

pkg/controllers/statefulapp/event_handler.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulapp
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+)
+
+func isRefController(controllerRef *metav1.OwnerReference) bool {
+	refGV, err := schema.ParseGroupVersion(controllerRef.APIVersion)
+	if err != nil {
+		klog.Errorf("could not parse OwnerReference %v APIVersion: %v", controllerRef, err)
+		return false
+	}
+	return controllerRef.Kind == controllerKind.Kind && refGV.Group == controllerKind.Group
+}
+
+type podEventHandler struct {
+	enqueueHandler handler.EnqueueRequestForOwner
+}
+
+func (p *podEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+	pod := evt.Object.(*v1.Pod)
+	if pod.DeletionTimestamp != nil {
+		p.Delete(event.DeleteEvent{Object: evt.Object}, q)
+		return
+	}
+	controllerRef := metav1.GetControllerOf(pod)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Create(evt, q)
+	}
+}
+
+func (p *podEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	oldPod := evt.ObjectOld.(*v1.Pod)
+	newPod := evt.ObjectNew.(*v1.Pod)
+	if newPod.ResourceVersion == oldPod.ResourceVersion {
+		return
+	}
+	p.enqueueHandler.Update(evt, q)
+}
+
+func (p *podEventHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	pod := evt.Object.(*v1.Pod)
+	controllerRef := metav1.GetControllerOf(pod)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Delete(evt, q)
+	}
+}
+
+func (p *podEventHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}
+
+type pvcEventHandler struct {
+	enqueueHandler handler.EnqueueRequestForOwner
+}
+
+func (p *pvcEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+	pvc := evt.Object.(*v1.PersistentVolumeClaim)
+	if pvc.DeletionTimestamp != nil {
+		p.Delete(event.DeleteEvent{Object: evt.Object}, q)
+		return
+	}
+	controllerRef := metav1.GetControllerOf(pvc)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Create(evt, q)
+	}
+}
+
+func (p *pvcEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	oldPVC := evt.ObjectOld.(*v1.PersistentVolumeClaim)
+	newPVC := evt.ObjectNew.(*v1.PersistentVolumeClaim)
+	if newPVC.ResourceVersion == oldPVC.ResourceVersion {
+		return
+	}
+	p.enqueueHandler.Update(evt, q)
+}
+
+func (p *pvcEventHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	pvc := evt.Object.(*v1.PersistentVolumeClaim)
+	controllerRef := metav1.GetControllerOf(pvc)
+	if controllerRef != nil && isRefController(controllerRef) {
+		p.enqueueHandler.Delete(evt, q)
+	}
+}
+
+func (p *pvcEventHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}

pkg/controllers/statefulapp/init.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulapp
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappconst "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/core"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+	"github.com/oceanbase/ob-operator/pkg/kubeclient"
+)
+
+var (
+	controllerKind = cloudv1.SchemeGroupVersion.WithKind("StatefulApp")
+)
+
+// Add creates a new Controller and adds it to the Manager with default RBAC.
+// The Manager will set fields on the Controller and Start it when the Manager is Started.
+func Add(mgr manager.Manager) error {
+	if !kube.DiscoverGVK(controllerKind) {
+		return nil
+	}
+	return add(mgr, newReconciler(mgr))
+}
+
+func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+	return &core.StatefulAppReconciler{
+		CRClient: kubeclient.NewClientFromManager(mgr),
+		Scheme:   mgr.GetScheme(),
+		Recorder: mgr.GetEventRecorderFor(statefulappconst.ControllerName),
+	}
+}
+
+// add a new Controller to mgr with r
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New(
+		statefulappconst.ControllerName,
+		mgr,
+		controller.Options{
+			Reconciler:              r,
+			MaxConcurrentReconciles: statefulappconst.ConcurrentReconciles,
+		},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	// Watch for changes to StatefulApp
+	err = c.Watch(
+		&source.Kind{Type: &cloudv1.StatefulApp{}},
+		&handler.EnqueueRequestForObject{},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	// Wathc for changes to Pod
+	err = c.Watch(
+		&source.Kind{Type: &corev1.Pod{}},
+		&podEventHandler{
+			enqueueHandler: handler.EnqueueRequestForOwner{
+				IsController: true,
+				OwnerType:    &cloudv1.StatefulApp{},
+			},
+		},
+		&podPredicate{},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	// Wathc for changes to pvc
+	err = c.Watch(
+		&source.Kind{Type: &corev1.PersistentVolumeClaim{}},
+		&pvcEventHandler{
+			enqueueHandler: handler.EnqueueRequestForOwner{
+				IsController: true,
+				OwnerType:    &cloudv1.StatefulApp{},
+			},
+		},
+		&pvcPredicate{},
+	)
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	return nil
+}

pkg/controllers/statefulapp/predicate.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulapp
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type podPredicate struct {
+}
+
+func (p podPredicate) Create(e event.CreateEvent) bool {
+	return true
+}
+
+func (p podPredicate) Delete(e event.DeleteEvent) bool {
+	return true
+}
+
+func (p podPredicate) Update(e event.UpdateEvent) bool {
+	return true
+}
+
+func (p podPredicate) Generic(e event.GenericEvent) bool {
+	return true
+}
+
+type pvcPredicate struct {
+}
+
+func (p pvcPredicate) Create(e event.CreateEvent) bool {
+	return true
+}
+
+func (p pvcPredicate) Delete(e event.DeleteEvent) bool {
+	return true
+}
+
+func (p pvcPredicate) Update(e event.UpdateEvent) bool {
+	return true
+}
+
+func (p pvcPredicate) Generic(e event.GenericEvent) bool {
+	return true
+}

pkg/infrastructure/kube/crd.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+)
+
+func Established(crd *v1.CustomResourceDefinition) bool {
+	for _, condition := range crd.Status.Conditions {
+		if condition.Type == v1.Established && condition.Status == v1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}

pkg/infrastructure/kube/discovery.go

+/*
+Copyright (c) 2021 OceanBase
+Copyright 2021 The Kruise Authors.
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+
+	"github.com/oceanbase/ob-operator/apis"
+	"github.com/oceanbase/ob-operator/pkg/kubeclient"
+)
+
+var (
+	internalScheme = runtime.NewScheme()
+
+	errKindNotFound = errors.Errorf("kind not found in group version resources")
+
+	backOff = wait.Backoff{
+		Steps:    4,
+		Duration: 500 * time.Millisecond,
+		Factor:   5.0,
+		Jitter:   0.1,
+	}
+)
+
+func init() {
+	err := apis.AddToScheme(internalScheme)
+	if err != nil {
+		klog.Errorln(err)
+	}
+}
+
+func DiscoverGVK(gvk schema.GroupVersionKind) bool {
+	genericClient := kubeclient.GetGenericClient()
+	if genericClient == nil {
+		return true
+	}
+	discoveryClient := genericClient.DiscoveryClient
+
+	err := retry.OnError(
+		backOff,
+		func(err error) bool {
+			return true
+		},
+		func() error {
+			resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String())
+			if err != nil {
+				return err
+			}
+			for _, r := range resourceList.APIResources {
+				if r.Kind == gvk.Kind {
+					return nil
+				}
+			}
+			return errKindNotFound
+		},
+	)
+
+	if err != nil {
+		if err == errKindNotFound {
+			klog.Errorf("not found kind %s in group version %s, waiting time %s", gvk.Kind, gvk.GroupVersion().String())
+			return false
+		}
+		// This might be caused by abnormal apiserver or etcd, ignore it
+		klog.Errorf("failed to find resources in group version %s: %v, waiting time %s", gvk.GroupVersion().String(), err)
+	}
+	return true
+}

pkg/infrastructure/kube/finalizer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func AddFinalizer(meta *metav1.ObjectMeta, finalizer string) {
+	if !HasFinalizer(meta, finalizer) {
+		meta.Finalizers = append(meta.Finalizers, finalizer)
+	}
+}
+
+func HasFinalizer(meta *metav1.ObjectMeta, finalizer string) bool {
+	return containsString(meta.Finalizers, finalizer)
+}
+
+func RemoveFinalizer(meta *metav1.ObjectMeta, finalizer string) {
+	meta.Finalizers = removeString(meta.Finalizers, finalizer)
+}
+
+func containsString(slice []string, s string) bool {
+	for _, item := range slice {
+		if item == s {
+			return true
+		}
+	}
+	return false
+}
+
+func removeString(slice []string, s string) []string {
+	var result []string
+	for _, item := range slice {
+		if item == s {
+			continue
+		}
+		result = append(result, item)
+	}
+	return result
+}

pkg/infrastructure/kube/label.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+func GetLabelSelector(key string, value string) labels.Selector {
+	// Key is empty
+	if len(key) == 0 {
+		return labels.SelectorFromSet(map[string]string{})
+	}
+	return labels.SelectorFromSet(labels.Set{key: value})
+}
+
+func AddLabel(labels map[string]string, key string, value string) map[string]string {
+	if key == "" {
+		return labels
+	}
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+	labels[key] = value
+	return labels
+}

pkg/infrastructure/kube/log.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"k8s.io/klog/v2"
+
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func LogForUniversal(msg string, obj interface{}) {
+	klog.Infoln(msg, util.CovertToJSON(obj))
+}
+
+func LogForAppActionStatus(kind, appName, action string, obj interface{}) {
+	klog.Infoln(action, kind, appName, util.CovertToJSON(obj))
+}

pkg/infrastructure/kube/namespace.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var readNamespace = func() ([]byte, error) {
+	return ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+}
+
+func GetOperatorNamespace() (string, error) {
+	nsBytes, err := readNamespace()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", errors.New("cannot find namespace of the operator")
+		}
+		return "", err
+	}
+	ns := strings.TrimSpace(string(nsBytes))
+	return ns, nil
+}

pkg/infrastructure/kube/object.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"context"
+
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func GetObjectList(ctx context.Context, c client.Client, obj client.ObjectList, listOption ...client.ListOption) (client.ObjectList, error) {
+	err := c.List(ctx, obj, listOption...)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return obj, err
+}
+
+func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) {
+	return controllerutil.CreateOrUpdate(ctx, c, obj, func() error {
+		original := obj.DeepCopyObject()
+		if err := f(); err != nil {
+			return err
+		}
+		generateObjectDiff(original, obj)
+		return nil
+	})
+}
+
+func generateObjectDiff(original runtime.Object, modified runtime.Object) {
+	diff := cmp.Diff(original, modified)
+	if len(diff) != 0 {
+		klog.Infoln(diff)
+	}
+}

pkg/infrastructure/kube/reference.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	ref "k8s.io/client-go/tools/reference"
+)
+
+func getRef(object runtime.Object) (*corev1.ObjectReference, error) {
+	return ref.GetReference(scheme.Scheme, object)
+}

pkg/infrastructure/kube/resource/event.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+// Container event reason list
+const (
+	CreatedContainer        = "Created"
+	FailedToCreateContainer = "Failed"
+	StartedContainer        = "Started"
+	FailedToStartContainer  = "Failed"
+	BackOffStartContainer   = "BackOff"
+	KillingContainer        = "Killing"
+	ExceededGracePeriod     = "ExceededGracePeriod"
+)
+
+// Pod event reason list
+const (
+	CreatedPod        = "CreatedPod"
+	FailedToCreatePod = "FailedCreatePod"
+	DeletedPod        = "DeletedPod"
+	FailedToKillPod   = "FailedKillPod"
+)
+
+// Storage event reason list
+const (
+	CreatedPVC        = "CreatedPVC"
+	FailedToCreatePVC = "FailedCreatePVC"
+	DeletedPVC        = "DeletedPVC"
+	FailedToDeletePVC = "FailedDeletePVC"
+)
+
+// Service event reason list
+const (
+	CreatedService        = "CreatedService"
+	FailedToCreateService = "FailedCreateService"
+	DeletedService        = "DeletedService"
+	FailedToDeleteService = "FailedDeleteService"
+)
+
+// StatefulApp event reason list
+const (
+	CreatedStatefulApp        = "CreatedStatefulApp"
+	FailedToCreateStatefulApp = "FailedCreateStatefulApp"
+	DeletedStatefulApp        = "DeletedPVC"
+	FailedToDeleteStatefulApp = "FailedDeleteStatefulApp"
+)
+
+// OBCluster event reason list
+const (
+	CreatedOBCluster        = "CreatedOBCluster"
+	FailedToCreateOBCluster = "FailedCreateOBCluster"
+	DeletedOBCluster        = "DeletedOBCluster"
+	FailedToDeleteOBCluster = "FailedDeleteOBCluster"
+)
+
+// Storage event reason list
+const (
+	CreatedRootService        = "CreatedRootService"
+	FailedToCreateRootService = "FailedCreateRootService"
+	DeletedRootService        = "DeletedRootService"
+	FailedToDeleteRootService = "FailedDeleteRootService"
+)
+
+// Storage event reason list
+const (
+	CreatedOBZone        = "CreatedOBZone"
+	FailedToCreateOBZone = "FailedCreateOBZone"
+	DeletedOBZone        = "DeletedOBZone"
+	FailedToDeleteOBZone = "FailedDeleteOBZone"
+)
+
+// Probe event reason list
+const (
+	ContainerUnhealthy = "Unhealthy"
+)

pkg/infrastructure/kube/resource/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type OBClusterResource struct {
+	*Resource
+}
+
+func NewOBClusterResource(resource *Resource) ResourceOperator {
+	return &OBClusterResource{resource}
+}
+
+func (r *OBClusterResource) Create(ctx context.Context, obj interface{}) error {
+	return nil
+}
+
+func (r *OBClusterResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	var obClusterCurrent cloudv1.OBCluster
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), &obClusterCurrent)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return obClusterCurrent, err
+}
+
+func (r *OBClusterResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	var res interface{}
+	return res
+}
+
+func (r *OBClusterResource) Update(ctx context.Context, obj interface{}) error {
+	return nil
+}
+
+func (r *OBClusterResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	obCluster := obj.(cloudv1.OBCluster)
+	err := r.Client.Status().Update(ctx, &obCluster)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *OBClusterResource) Delete(ctx context.Context, obj interface{}) error {
+	obCluster := obj.(cloudv1.OBCluster)
+	// kube.LogForAppActionStatus(obCluster.Kind, obCluster.Name, "delete", obCluster)
+	err := r.Client.Delete(ctx, &obCluster)
+	if err != nil {
+		r.Recorder.Eventf(&obCluster, corev1.EventTypeWarning, FailedToDeleteOBCluster, "delete OBCluster"+obCluster.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(obCluster.Kind, obCluster.Name, "delete", "succeed")
+	r.Recorder.Event(&obCluster, corev1.EventTypeNormal, DeletedOBCluster, "delete OBCluster"+obCluster.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/obzone.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type OBZoneResource struct {
+	*Resource
+}
+
+func NewOBZoneResource(resource *Resource) ResourceOperator {
+	return &OBZoneResource{resource}
+}
+
+func (r *OBZoneResource) Create(ctx context.Context, obj interface{}) error {
+	obZone := obj.(cloudv1.OBZone)
+	// kube.LogForAppActionStatus(obZone.Kind, obZone.Name, "create", obZone)
+	err := r.Client.Create(ctx, &obZone)
+	if err != nil {
+		r.Recorder.Eventf(&obZone, corev1.EventTypeWarning, FailedToCreateOBZone, "create OBZone"+obZone.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(obZone.Kind, obZone.Name, "create", "succeed")
+	r.Recorder.Event(&obZone, corev1.EventTypeNormal, CreatedOBZone, "create OBZone"+obZone.Name)
+	return nil
+}
+
+func (r *OBZoneResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	var obZoneCurrent cloudv1.OBZone
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), &obZoneCurrent)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return obZoneCurrent, err
+}
+
+func (r *OBZoneResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	var res interface{}
+	return res
+}
+
+func (r *OBZoneResource) Update(ctx context.Context, obj interface{}) error {
+	obZone := obj.(cloudv1.OBZone)
+	err := r.Client.Update(ctx, &obZone)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *OBZoneResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	obZone := obj.(cloudv1.OBZone)
+	err := r.Client.Status().Update(ctx, &obZone)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *OBZoneResource) Delete(ctx context.Context, obj interface{}) error {
+	obZone := obj.(cloudv1.OBZone)
+	// kube.LogForAppActionStatus(obZone.Kind, obZone.Name, "delete", obZone)
+	err := r.Client.Delete(ctx, &obZone)
+	if err != nil {
+		r.Recorder.Eventf(&obZone, corev1.EventTypeWarning, FailedToDeleteOBZone, "delete OBZone CR"+obZone.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(obZone.Kind, obZone.Name, "delete", "succeed")
+	r.Recorder.Event(&obZone, corev1.EventTypeNormal, DeletedOBZone, "delete OBZone CR"+obZone.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/pod.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type PodResource struct {
+	*Resource
+}
+
+func NewPodResource(resource *Resource) ResourceOperator {
+	return &PodResource{resource}
+}
+
+func (r *PodResource) Create(ctx context.Context, obj interface{}) error {
+	pod := obj.(corev1.Pod)
+	// kube.LogForAppActionStatus(pod.Kind, pod.Name, "create", pod)
+	err := r.Client.Create(ctx, &pod)
+	if err != nil {
+		r.Recorder.Eventf(&pod, corev1.EventTypeWarning, FailedToCreatePod, "create Pod"+pod.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(pod.Kind, pod.Name, "create", "succeed")
+	r.Recorder.Event(&pod, corev1.EventTypeNormal, CreatedPod, "create Pod"+pod.Name)
+	return nil
+}
+
+func (r *PodResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	pod := &corev1.Pod{}
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), pod)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return *pod, err
+}
+
+func (r *PodResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	podList := &corev1.PodList{}
+	err := r.Client.List(ctx, podList, client.InNamespace(namespace), listOption)
+	if err != nil {
+		// can definitely get a value, so errors are not returned
+		klog.Errorln(err)
+	}
+	return *podList
+}
+
+func (r *PodResource) Update(ctx context.Context, obj interface{}) error {
+	return nil
+}
+
+func (r *PodResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	var res error
+	return res
+}
+
+func (r *PodResource) Delete(ctx context.Context, obj interface{}) error {
+	pod := obj.(corev1.Pod)
+	// kube.LogForAppActionStatus(pod.Kind, pod.Name, "delete", pod)
+	err := r.Client.Delete(ctx, &pod)
+	if err != nil {
+		r.Recorder.Eventf(&pod, corev1.EventTypeWarning, FailedToKillPod, "delete Pod"+pod.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(pod.Kind, pod.Name, "delete", "succeed")
+	r.Recorder.Event(&pod, corev1.EventTypeNormal, DeletedPod, "delete Pod"+pod.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/pvc.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type PVCResource struct {
+	*Resource
+}
+
+func NewPVCResource(resource *Resource) ResourceOperator {
+	return &PVCResource{resource}
+}
+
+func (r *PVCResource) Create(ctx context.Context, obj interface{}) error {
+	pvcs := obj.([]corev1.PersistentVolumeClaim)
+	var err error
+	for _, pvc := range pvcs {
+		// kube.LogForAppActionStatus(pvc.Kind, pvc.Name, "create", pvc)
+		err = r.Client.Create(ctx, &pvc)
+		if err != nil {
+			r.Recorder.Eventf(&pvc, corev1.EventTypeWarning, FailedToCreatePVC, "create PVC"+pvc.Name)
+			klog.Errorln(err)
+			break
+		}
+		kube.LogForAppActionStatus(pvc.Kind, pvc.Name, "create", "succeed")
+		r.Recorder.Event(&pvc, corev1.EventTypeNormal, CreatedPVC, "create PVC"+pvc.Name)
+	}
+	return err
+}
+
+func (r *PVCResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	pvc := &corev1.PersistentVolumeClaim{}
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), pvc)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return *pvc, err
+}
+
+func (r *PVCResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	var res interface{}
+	return res
+}
+
+func (r *PVCResource) Update(ctx context.Context, obj interface{}) error {
+	return nil
+}
+
+func (r *PVCResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	var res error
+	return res
+}
+
+func (r *PVCResource) Delete(ctx context.Context, obj interface{}) error {
+	pvc := obj.(corev1.PersistentVolumeClaim)
+	// kube.LogForAppActionStatus(pvc.Kind, pvc.Name, "delete", pvc)
+	err := r.Client.Delete(context.TODO(), &pvc)
+	if err != nil {
+		r.Recorder.Eventf(&pvc, corev1.EventTypeWarning, FailedToDeletePVC, "delete PVC"+pvc.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(pvc.Kind, pvc.Name, "delete", "succeed")
+	r.Recorder.Event(&pvc, corev1.EventTypeNormal, DeletedPVC, "delete PVC"+pvc.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/resource.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type Resource struct {
+	Client   client.Client
+	Recorder record.EventRecorder
+}
+
+func NewResource(client client.Client, record record.EventRecorder) *Resource {
+	return &Resource{
+		Client:   client,
+		Recorder: record,
+	}
+}
+
+type ResourceOperator interface {
+	Create(ctx context.Context, obj interface{}) error
+	Get(ctx context.Context, namespace, name string) (interface{}, error)
+	List(ctx context.Context, namespace string, listOption client.ListOption) interface{}
+	Update(ctx context.Context, obj interface{}) error
+	UpdateStatus(ctx context.Context, obj interface{}) error
+	Delete(ctx context.Context, obj interface{}) error
+}

pkg/infrastructure/kube/resource/rootservice.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type RootServiceResource struct {
+	*Resource
+}
+
+func NewRootServiceResource(resource *Resource) ResourceOperator {
+	return &RootServiceResource{resource}
+}
+
+func (r *RootServiceResource) Create(ctx context.Context, obj interface{}) error {
+	rootService := obj.(cloudv1.RootService)
+	// kube.LogForAppActionStatus(rootService.Kind, rootService.Name, "create", rootService)
+	err := r.Client.Create(ctx, &rootService)
+	if err != nil {
+		r.Recorder.Eventf(&rootService, corev1.EventTypeWarning, FailedToCreateRootService, "create RootService"+rootService.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(rootService.Kind, rootService.Name, "create", "succeed")
+	r.Recorder.Event(&rootService, corev1.EventTypeNormal, CreatedRootService, "create RootService"+rootService.Name)
+	return nil
+}
+
+func (r *RootServiceResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	var rootServiceCurrent cloudv1.RootService
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), &rootServiceCurrent)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return rootServiceCurrent, err
+}
+
+func (r *RootServiceResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	var res interface{}
+	return res
+}
+
+func (r *RootServiceResource) Update(ctx context.Context, obj interface{}) error {
+	return nil
+}
+
+func (r *RootServiceResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	rootService := obj.(cloudv1.RootService)
+	err := r.Client.Status().Update(ctx, &rootService)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *RootServiceResource) Delete(ctx context.Context, obj interface{}) error {
+	rootService := obj.(cloudv1.RootService)
+	// kube.LogForAppActionStatus(rootService.Kind, rootService.Name, "delete", rootService)
+	err := r.Client.Delete(ctx, &rootService)
+	if err != nil {
+		r.Recorder.Eventf(&rootService, corev1.EventTypeWarning, FailedToDeleteRootService, "delete RootService CR"+rootService.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(rootService.Kind, rootService.Name, "delete", "succeed")
+	r.Recorder.Event(&rootService, corev1.EventTypeNormal, DeletedRootService, "delete RootService CR"+rootService.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/service.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type ServiceResource struct {
+	*Resource
+}
+
+func NewServiceResource(resource *Resource) ResourceOperator {
+	return &ServiceResource{resource}
+}
+
+func (r *ServiceResource) Create(ctx context.Context, obj interface{}) error {
+	service := obj.(corev1.Service)
+	// kube.LogForAppActionStatus(service.Kind, service.Name, "create", service)
+	err := r.Client.Create(ctx, &service)
+	if err != nil {
+		r.Recorder.Eventf(&service, corev1.EventTypeWarning, FailedToCreateService, "create Service"+service.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(service.Kind, service.Name, "create", "succeed")
+	r.Recorder.Event(&service, corev1.EventTypeNormal, CreatedService, "create Service"+service.Name)
+	return nil
+}
+
+func (r *ServiceResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	service := &corev1.Service{}
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), service)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return *service, err
+}
+
+func (r *ServiceResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	return nil
+}
+
+func (r *ServiceResource) Update(ctx context.Context, obj interface{}) error {
+	service := obj.(corev1.Service)
+	err := r.Client.Update(ctx, &service)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *ServiceResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	var res error
+	return res
+}
+
+func (r *ServiceResource) Delete(ctx context.Context, obj interface{}) error {
+	service := obj.(corev1.Service)
+	// kube.LogForAppActionStatus(service.Kind, service.Name, "delete", service)
+	err := r.Client.Delete(ctx, &service)
+	if err != nil {
+		r.Recorder.Eventf(&service, corev1.EventTypeWarning, FailedToDeleteService, "delete Service"+service.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(service.Kind, service.Name, "delete", "succeed")
+	r.Recorder.Event(&service, corev1.EventTypeNormal, DeletedService, "delete Service"+service.Name)
+	return nil
+}

pkg/infrastructure/kube/resource/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/infrastructure/kube"
+)
+
+type StatefulAppResource struct {
+	*Resource
+}
+
+func NewStatefulAppResource(resource *Resource) ResourceOperator {
+	return &StatefulAppResource{resource}
+}
+
+func (r *StatefulAppResource) Create(ctx context.Context, obj interface{}) error {
+	statefulApp := obj.(cloudv1.StatefulApp)
+	// kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "create", statefulApp)
+	err := r.Client.Create(ctx, &statefulApp)
+	if err != nil {
+		r.Recorder.Eventf(&statefulApp, corev1.EventTypeWarning, FailedToCreateStatefulApp, "create StatefulApp"+statefulApp.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "create", "succeed")
+	r.Recorder.Event(&statefulApp, corev1.EventTypeNormal, CreatedStatefulApp, "create StatefulApp"+statefulApp.Name)
+	return nil
+}
+
+func (r *StatefulAppResource) Get(ctx context.Context, namespace, name string) (interface{}, error) {
+	var statefulAppCurrent cloudv1.StatefulApp
+	err := r.Client.Get(ctx, kube.GenerateNamespacedName(namespace, name), &statefulAppCurrent)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return statefulAppCurrent, err
+}
+
+func (r *StatefulAppResource) List(ctx context.Context, namespace string, listOption client.ListOption) interface{} {
+	var res interface{}
+	return res
+}
+
+func (r *StatefulAppResource) Update(ctx context.Context, obj interface{}) error {
+	statefulAppNew := obj.(cloudv1.StatefulApp)
+	err := r.Client.Update(ctx, &statefulAppNew)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *StatefulAppResource) UpdateStatus(ctx context.Context, obj interface{}) error {
+	statefulAppNew := obj.(cloudv1.StatefulApp)
+	err := r.Client.Status().Update(ctx, &statefulAppNew)
+	if err != nil {
+		klog.Errorln(err)
+	}
+	return err
+}
+
+func (r *StatefulAppResource) Delete(ctx context.Context, obj interface{}) error {
+	statefulApp := obj.(cloudv1.StatefulApp)
+	// kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "delete", statefulApp)
+	err := r.Client.Delete(ctx, &statefulApp)
+	if err != nil {
+		r.Recorder.Eventf(&statefulApp, corev1.EventTypeWarning, FailedToDeleteStatefulApp, "delete StatefulApp"+statefulApp.Name)
+		klog.Errorln(err)
+		return err
+	}
+	kube.LogForAppActionStatus(statefulApp.Kind, statefulApp.Name, "delete", "succeed")
+	r.Recorder.Event(&statefulApp, corev1.EventTypeNormal, DeletedStatefulApp, "delete StatefulApp"+statefulApp.Name)
+	return nil
+}

pkg/infrastructure/kube/secret.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func LoadSecretData(apiReader client.Reader, namespace, secretName, dataKey string) (string, error) {
+	secret := &corev1.Secret{}
+	err := apiReader.Get(context.TODO(), GenerateNamespacedName(namespace, secretName), secret)
+	if err != nil {
+		return "", err
+	}
+	retStr, ok := secret.Data[dataKey]
+	if !ok {
+		return "", errors.Errorf("secret %s did not contain key %s", secretName, dataKey)
+	}
+	return string(retStr), nil
+}
+
+func LoadSecretDataUsingClient(c client.Client, namespace, secretName, dataKey string) (string, error) {
+	secret := &corev1.Secret{}
+	err := c.Get(context.TODO(), GenerateNamespacedName(namespace, secretName), secret)
+	if err != nil {
+		return "", err
+	}
+	retStr, ok := secret.Data[dataKey]
+	if !ok {
+		return "", errors.Errorf("secret %s did not contain key %s", secretName, dataKey)
+	}
+	return string(retStr), nil
+}

pkg/infrastructure/kube/util.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kube
+
+import (
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func GenerateNamespacedName(namespace, name string) types.NamespacedName {
+	var namespacedName types.NamespacedName
+	namespacedName.Namespace = namespace
+	namespacedName.Name = name
+	return namespacedName
+}

pkg/infrastructure/ob/obcluster_bootstrap.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package ob
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ClusterType string
+
+const (
+	PRIMARY ClusterType = "PRIMARY"
+	STANDBY ClusterType = "STANDBY"
+)
+
+const (
+	bootstrapPrimarySql         = "ALTER SYSTEM BOOTSTRAP %v"
+	bootstrapStandbySql         = "ALTER SYSTEM BOOTSTRAP CLUSTER STANDBY %v"
+	bootstrapPrimaryInfoSqlPart = " PRIMARY_CLUSTER_ID %v PRIMARY_ROOTSERVICE_LIST '%v'"
+	bootstrapRSInfoSqlPart      = "REGION '%v' ZONE '%v' SERVER '%v:%v'"
+)
+
+type BootstrapParam struct {
+	ClusterType ClusterType  `json:"clusterType"` // cluster type, primary or standby
+	PrimaryInfo *PrimaryInfo `json:"primaryInfo"` // provided when bootstrap standby clusters, nil if not needed
+	RSInfoList  []RSInfo     `json:"rsInfoList"`  // root server info
+}
+
+type PrimaryInfo struct {
+	PrimaryClusterId       int    `json:"primaryClusterId"`
+	PrimaryRootServiceList string `json:"primaryRootServiceList"`
+}
+
+type RSInfo struct {
+	Region string     `json:"region"`
+	Zone   string     `json:"zone"`
+	Server ServerInfo `json:"server"`
+}
+
+type ServerInfo struct {
+	Ip   string `json:"ip"`
+	Port int    `json:"port"`
+}
+
+func GenerateBootstrapSQL(param BootstrapParam) string {
+	var rsInfos []string
+	for _, rsInfo := range param.RSInfoList {
+		rsInfoString := fmt.Sprintf(bootstrapRSInfoSqlPart, rsInfo.Region, rsInfo.Zone, rsInfo.Server.Ip, rsInfo.Server.Port)
+		rsInfos = append(rsInfos, rsInfoString)
+	}
+	rsInfoList := strings.Join(rsInfos, ", ")
+	var sql string
+	if param.ClusterType == PRIMARY {
+		sql = fmt.Sprintf(bootstrapPrimarySql, rsInfoList)
+	} else if param.ClusterType == STANDBY {
+		sql = fmt.Sprintf(bootstrapStandbySql, rsInfoList)
+		if param.PrimaryInfo != nil {
+			sql += fmt.Sprintf(bootstrapPrimaryInfoSqlPart, param.PrimaryInfo.PrimaryClusterId, param.PrimaryInfo.PrimaryRootServiceList)
+		}
+	} else {
+		return ""
+	}
+	return sql
+}

pkg/infrastructure/ob/observer_start.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package ob
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+
+	"github.com/oceanbase/ob-operator/pkg/util/shell"
+)
+
+const (
+	CPU_COUNT                       = 16
+	MEMORY_LIMIT                    = 10
+	MEMORY_LOW                      = "8"
+	MEMORY_SIMPLE                   = 64
+	NIC                             = "eth0"
+	OBSERVER_MYSQL_PORT             = "2881"
+	OBSERVER_RPC_PORT               = "2882"
+	OBSERVER_START_COMMAND_TEMPLATE = "cd /home/admin/oceanbase; ulimit -s 10240; ulimit -c unlimited; LD_LIBRARY_PATH=/home/admin/oceanbase/lib:$LD_LIBRARY_PATH LD_PRELOAD='' /home/admin/oceanbase/bin/observer --appname ${OB_CLUSTER_NAME} --cluster_id ${OB_CLUSTER_ID} --zone ${ZONE_NAME} --devname ${DEV_NAME} -p 2881 -P 2882 -d /home/admin/oceanbase/store/ -l info -o 'rootservice_list=${RS_LIST},config_additional_dir=/home/admin/oceanbase/etc2,/home/admin/oceanbase/etc3,${OPTION}'"
+)
+
+type StartObServerProcessArguments struct {
+	ClusterName string `json:"clusterName" binding:"required"`
+	ClusterId   int    `json:"clusterId" binding:"required"`
+	ZoneName    string `json:"zoneName" binding:"required"`
+	RsList      string `json:"rsList" binding:"required"`
+	CpuLimit    int    `json:"cpuLimit" binding:"required"`
+	MemoryLimit int    `json:"memoryLimit" binding:"required"`
+}
+
+func StartOBServerProcess(param StartObServerProcessArguments) {
+	cpu := getCPU(param.CpuLimit)
+	memory := getMemory(param.MemoryLimit)
+	systemMemory := getSystemMemory(param.MemoryLimit)
+	datafileSize := getDatafileSize(memory)
+	var cmd string
+	var option string
+	obClusterName := param.ClusterName
+	obClusterId := param.ClusterId
+	zoneName := param.ZoneName
+	rsList := param.RsList
+	deviceName := NIC
+	memoryInt, err := strconv.Atoi(memory)
+	if memoryInt <= MEMORY_SIMPLE {
+		// 2C, 10G
+		option = fmt.Sprintf("cpu_count=%s,memory_limit=%sG,system_memory=%sG,__min_full_resource_pool_memory=268435456,datafile_size=%sG,net_thread_count=%d,stack_size=512K,cache_wash_threshold=1G,schema_history_expire_time=1d,enable_separate_sys_clog=false,enable_merge_by_turn=false,enable_syslog_recycle=true,enable_syslog_wf=false,max_syslog_file_count=4", cpu, memory, systemMemory, datafileSize, param.CpuLimit)
+	} else {
+		// 16C, 64G
+		option = fmt.Sprintf("cpu_count=%s,memory_limit=%sG,system_memory=%sG,__min_full_resource_pool_memory=1073741824,datafile_size=%sG,net_thread_count=%d", cpu, memory, systemMemory, datafileSize, param.CpuLimit)
+	}
+	cmd = replaceAll(OBSERVER_START_COMMAND_TEMPLATE, startObServerParamReplacer(obClusterName, obClusterId, zoneName, deviceName, rsList, option))
+	_, err = shell.NewCommand(cmd).WithContext(context.TODO()).WithUser(shell.AdminUser).Execute()
+	if err != nil {
+		log.Println("cmd exec error", err)
+	}
+}
+
+func getCPU(cpuLimit int) string {
+	if CPU_COUNT+1 > cpuLimit {
+		return strconv.Itoa(CPU_COUNT)
+	}
+	return strconv.Itoa(cpuLimit - 1)
+}
+
+func getMemory(memoryLimit int) string {
+	if memoryLimit == MEMORY_LIMIT {
+		log.Println("memoryLimit is low")
+		return MEMORY_LOW
+	}
+	return strconv.Itoa(int(float32(memoryLimit) * 0.9))
+}
+
+func getSystemMemory(memory int) string {
+	tmp := float32(memory) * 0.9
+	var coefficient float32
+	if tmp >= 150 {
+		coefficient = 0.3
+	} else if tmp >= 64 && tmp < 150 {
+		coefficient = 0.4
+	} else {
+		coefficient = 0.5
+	}
+	return strconv.Itoa(int(tmp * coefficient))
+}
+
+func getDatafileSize(memoryLimit string) string {
+	tmp, _ := strconv.Atoi(memoryLimit)
+	return strconv.Itoa(tmp * 3)
+}
+
+func replaceAll(template string, replacers ...*strings.Replacer) string {
+	s := template
+	for _, replacer := range replacers {
+		s = replacer.Replace(s)
+	}
+	return s
+}
+
+func startObServerParamReplacer(obClusterName string, obClusterId int, zoneName, deviceName, rsList, option string) *strings.Replacer {
+	return strings.NewReplacer("${OB_CLUSTER_NAME}", obClusterName, "${OB_CLUSTER_ID}", strconv.Itoa(obClusterId), "${ZONE_NAME}", zoneName, "${DEV_NAME}", deviceName, "${RS_LIST}", rsList, "${OPTION}", option)
+}

pkg/kubeclient/client.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kubeclient
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+func NewClientFromManager(mgr manager.Manager) client.Client {
+	cfg := *mgr.GetConfig()
+
+	// cfg.Burst = 10
+	// cfg.QPS = float32(10)
+
+	c, err := client.New(
+		&cfg,
+		client.Options{Scheme: mgr.GetScheme(), Mapper: mgr.GetRESTMapper()},
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	delegatingClient, _ := client.NewDelegatingClient(
+		client.NewDelegatingClientInput{
+			CacheReader: mgr.GetCache(),
+			Client:      c,
+		},
+	)
+
+	return delegatingClient
+}

pkg/kubeclient/generic_client.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kubeclient
+
+import (
+	"k8s.io/client-go/discovery"
+	kubeclientset "k8s.io/client-go/kubernetes"
+)
+
+// GenericClientset defines a generic client
+type GenericClientset struct {
+	DiscoveryClient discovery.DiscoveryInterface
+	KubeClient      kubeclientset.Interface
+}

pkg/kubeclient/registry.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package kubeclient
+
+var (
+	defaultGenericClient *GenericClientset
+)
+
+// GetGenericClient returns default clientset
+func GetGenericClient() *GenericClientset {
+	return defaultGenericClient
+}

pkg/util/func.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"fmt"
+	"time"
+)
+
+// TimeElapsed prints time it takes to execute a function
+// Usage: defer TimeElapsed("function-name")()
+func TimeElapsed(functionName string) func() {
+	start := time.Now()
+	return func() {
+		fmt.Printf("%s took %v\n", functionName, time.Since(start))
+	}
+}

pkg/util/http.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+func HTTPGET(reqURL string) (int, map[string]interface{}) {
+	req, _ := http.NewRequest("GET", reqURL, nil)
+	c := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	res, perr := c.Do(req)
+	if perr != nil {
+		log.Println(perr)
+		return 0, nil
+	}
+	resBody, berr := ioutil.ReadAll(res.Body)
+	_ = res.Body.Close()
+	if berr != nil {
+		log.Println(berr)
+		return 0, nil
+	}
+	responeDate := make(map[string]interface{})
+	jerr := json.Unmarshal(resBody, &responeDate)
+	if jerr != nil {
+		log.Println(jerr)
+		return res.StatusCode, nil
+	}
+	return res.StatusCode, responeDate
+}
+
+func HTTPPOST(reqURL, reqData string) (int, map[string]interface{}) {
+	req, _ := http.NewRequest("POST", reqURL, strings.NewReader(reqData))
+	req.Header.Add("Content-Type", "application/json")
+	c := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	res, perr := c.Do(req)
+	if perr != nil {
+		log.Println(perr)
+		return 0, nil
+	}
+	resBody, berr := ioutil.ReadAll(res.Body)
+	_ = res.Body.Close()
+	if berr != nil {
+		log.Println(berr)
+		return 0, nil
+	}
+	responeDate := make(map[string]interface{})
+	jerr := json.Unmarshal(resBody, &responeDate)
+	if jerr != nil {
+		log.Println(jerr)
+		return res.StatusCode, nil
+	}
+	return res.StatusCode, responeDate
+}
+
+func HTTPPUT(reqURL, reqData string) map[string]interface{} {
+	req, _ := http.NewRequest("PUT", reqURL, strings.NewReader(reqData))
+	req.Header.Add("Content-Type", "application/json")
+	c := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	res, perr := c.Do(req)
+	if perr != nil {
+		log.Println(perr)
+		return nil
+	}
+	resBody, berr := ioutil.ReadAll(res.Body)
+	_ = res.Body.Close()
+	if berr != nil {
+		log.Println(berr)
+		return nil
+	}
+	responeDate := make(map[string]interface{})
+	jerr := json.Unmarshal(resBody, &responeDate)
+	if jerr != nil {
+		log.Println(jerr)
+		return nil
+	}
+	return responeDate
+}
+
+func HTTPDELETE(reqURL, reqData string) map[string]interface{} {
+	req, _ := http.NewRequest("DELETE", reqURL, nil)
+	c := &http.Client{
+		Timeout: 1 * time.Second,
+	}
+	res, perr := c.Do(req)
+	if perr != nil {
+		log.Println(perr)
+		return nil
+	}
+	resBody, berr := ioutil.ReadAll(res.Body)
+	_ = res.Body.Close()
+	if berr != nil {
+		log.Println(berr)
+		return nil
+	}
+	responeDate := make(map[string]interface{})
+	jerr := json.Unmarshal(resBody, &responeDate)
+	if jerr != nil {
+		log.Println(jerr)
+		return nil
+	}
+	return responeDate
+}

pkg/util/json.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"encoding/json"
+)
+
+func CovertToJSON(element interface{}) string {
+	tempJSON, _ := json.Marshal(element)
+	return string(tempJSON)
+}

pkg/util/shell/command.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package shell
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+type Program string
+
+const (
+	Sh        = "sh"
+	RootUser  = "root"
+	AdminUser = "admin"
+)
+
+const DefaultProgram = Sh
+const DefaultTimeout = 10 * time.Second
+
+type Command interface {
+	Cmd() string
+	User() string
+	Program() Program
+	Timeout() time.Duration
+	WithContext(ctx context.Context) Command
+	WithUser(user string) Command
+	WithProgram(program Program) Command
+	WithTimeout(timeout time.Duration) Command
+	Execute() (*ExecuteResult, error)
+	ExecuteWithDebug() (*ExecuteResult, error)
+	ExecuteAllowFailure() (*ExecuteResult, error)
+}
+
+func NewCommand(cmd string) Command {
+	return &command{
+		program: DefaultProgram,
+		cmd:     cmd,
+		timeout: DefaultTimeout,
+	}
+}
+
+type command struct {
+	user    string  // Run command as this user, if not provided, run command as current process's user
+	program Program // Shell program to execute command, e.g. sh, bash
+	cmd     string
+	timeout time.Duration
+	context context.Context
+}
+
+func (c *command) Cmd() string {
+	return c.cmd
+}
+
+func (c *command) User() string {
+	return c.user
+}
+
+func (c *command) Program() Program {
+	return c.program
+}
+
+func (c *command) Timeout() time.Duration {
+	return c.timeout
+}
+
+func (c *command) WithContext(ctx context.Context) Command {
+	c.context = ctx
+	return c
+}
+
+func (c *command) WithUser(user string) Command {
+	c.user = user
+	return c
+}
+
+func (c *command) WithProgram(program Program) Command {
+	c.program = program
+	return c
+}
+
+func (c *command) WithTimeout(timeout time.Duration) Command {
+	c.timeout = timeout
+	return c
+}
+
+func (c *command) String() string {
+	return fmt.Sprintf("Command{user=%s, program=%s, cmd=%s, timeout=%s}", c.user, c.program, c.cmd, c.timeout)
+}

pkg/util/shell/exec.go

+/*
+Copyright (c) 2021 OceanBase
+Copyright (c) 2015-2020 InfluxData Inc.
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package shell
+
+import (
+	"bytes"
+	"context"
+	"os/exec"
+	"os/user"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	info  = 0b1
+	debug = 0b10
+)
+
+type ExecuteResult struct {
+	ExitCode int
+	Output   string
+}
+
+func (r ExecuteResult) IsSuccessful() bool {
+	return r.ExitCode == 0
+}
+
+func (r ExecuteResult) AsError() error {
+	if r.IsSuccessful() {
+		return nil
+	}
+	return errors.Errorf("failed to execute command, exitCode: %d, output: %s", r.ExitCode, r.Output)
+}
+
+func (r ExecuteResult) Lines() []string {
+	if len(r.Output) == 0 {
+		return []string{}
+	}
+	if !strings.Contains(r.Output, "\n") {
+		return []string{r.Output}
+	}
+	lines := strings.Split(strings.Trim(r.Output, "\n"), "\n")
+	return lines
+}
+
+// Execute the given command and expect the command to succeed (exits with 0)
+// If the command exits with a non-zero code, return an error
+func (c *command) Execute() (*ExecuteResult, error) {
+	executeResult, err := c.execute(info)
+	if err != nil {
+		return executeResult, err
+	}
+	return executeResult, executeResult.AsError()
+}
+
+// Execute the given command, allow the command to failed (exits with non-zero code).
+func (c *command) ExecuteAllowFailure() (*ExecuteResult, error) {
+	return c.execute(info)
+}
+
+// 正常日志用 debug 打印
+func (c *command) ExecuteWithDebug() (*ExecuteResult, error) {
+	executeResult, err := c.execute(debug)
+	if err != nil {
+		return executeResult, err
+	}
+	return executeResult, executeResult.AsError()
+}
+
+// Execute the given command with the given program and timeout
+// It returns:
+// 1. the exit code
+// 2. the combined output of stdout and stderr
+// 3. the error
+func (c *command) execute(flag int) (*ExecuteResult, error) {
+	if c.context == nil {
+		c.context = context.Background()
+	}
+	ctx := context.WithValue(c.context, "agentlog.StartTimeKey", time.Now())
+	if flag&debug != 0 {
+		log.WithContext(ctx).Debugf("execute shell command start, command=%s", c.String())
+	} else {
+		log.WithContext(ctx).Infof("execute shell command start, command=%s", c.String())
+	}
+	var runCmd *exec.Cmd
+	currentUser := getCurrentUser()
+	if c.user == "" || c.user == currentUser {
+		runCmd = exec.Command(string(c.program), "-c", c.cmd)
+	} else if currentUser == RootUser {
+		runCmd = exec.Command("runuser", "-l", c.user, "-c", c.cmd)
+	} else if c.user == RootUser {
+		runCmd = exec.Command("sudo", string(c.program), "-c", c.cmd)
+	} else {
+		runCmd = exec.Command("sudo", "-u", c.user, string(c.program), "-c", c.cmd)
+	}
+	b, err := CombinedOutputTimeout(runCmd, c.timeout)
+	output := string(b)
+	log.WithContext(ctx).Debugf("execute shell command %s, output=%s", c.String(), output)
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			exitCode := exitError.ExitCode()
+			log.WithContext(ctx).Infof("execute shell command failed, command=%s, exitCode=%d", c.String(), exitCode)
+			return &ExecuteResult{
+				ExitCode: exitCode,
+				Output:   output,
+			}, nil
+		} else {
+			log.WithContext(ctx).Errorf("execute shell command error, command=%s, error=%s", c.String(), err)
+			return nil, errors.Errorf("error when execute shell command %s: %s", c.cmd, err)
+		}
+	} else {
+		if flag&debug != 0 {
+			log.WithContext(ctx).Debugf("execute shell command end, command=%s", c.String())
+		} else {
+			log.WithContext(ctx).Infof("execute shell command end, command=%s", c.String())
+		}
+		return &ExecuteResult{
+			ExitCode: 0,
+			Output:   output,
+		}, nil
+	}
+}
+
+// CombinedOutputTimeout runs the given command with the given timeout and returns the output of stdout and stderr
+// If the command times out, it attempts to kill the process
+func CombinedOutputTimeout(c *exec.Cmd, timeout time.Duration) ([]byte, error) {
+	var b bytes.Buffer
+	c.Stdout = &b
+	c.Stderr = &b
+	if err := c.Start(); err != nil {
+		return nil, err
+	}
+	err := WaitTimeout(c, timeout)
+	return b.Bytes(), err
+}
+
+// StdoutOutputTimeout runs the given command with the given timeout and returns the output of stdout
+// If the command times out, it attempts to kill the process
+func StdoutOutputTimeout(c *exec.Cmd, timeout time.Duration) ([]byte, error) {
+	var b bytes.Buffer
+	c.Stdout = &b
+	c.Stderr = nil
+	if err := c.Start(); err != nil {
+		return nil, err
+	}
+	err := WaitTimeout(c, timeout)
+	return b.Bytes(), err
+}
+
+// RunTimeout runs the given command with the given timeout
+// If the command times out, it attempts to kill the process
+func RunTimeout(c *exec.Cmd, timeout time.Duration) error {
+	if err := c.Start(); err != nil {
+		return err
+	}
+	return WaitTimeout(c, timeout)
+}
+
+func getCurrentUser() string {
+	currentUser, err := user.Current()
+	if err != nil {
+		return ""
+	}
+	return currentUser.Username
+}

pkg/util/shell/exec_test.go

+/*
+Copyright (c) 2021 OceanBase
+Copyright (c) 2015-2020 InfluxData Inc.
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package shell
+
+import (
+	"bytes"
+	"os/exec"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	. "github.com/smartystreets/goconvey/convey"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	sleepbin, _ = exec.LookPath("sleep")
+	echobin, _  = exec.LookPath("echo")
+	shell, _    = exec.LookPath("sh")
+)
+
+func TestExecuteCommand(t *testing.T) {
+	type args struct {
+		cmd string
+	}
+	type want struct {
+		successful bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want want
+	}{
+		{
+			name: "normal command",
+			args: args{cmd: "echo a"},
+			want: want{successful: true},
+		},
+		{
+			name: "command not exist",
+			args: args{cmd: "command_not_exist"},
+			want: want{successful: false},
+		},
+	}
+	for _, tt := range tests {
+		Convey(tt.name, t, func() {
+			executeResult, _ := NewCommand(tt.args.cmd).Execute()
+			So(executeResult, ShouldNotBeNil)
+			So(executeResult.IsSuccessful(), ShouldEqual, tt.want.successful)
+		})
+	}
+}
+
+func TestRunTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test due to random failures.")
+	}
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(sleepbin, "10")
+	start := time.Now()
+	err := RunTimeout(cmd, time.Millisecond*20)
+	elapsed := time.Since(start)
+
+	assert.Equal(t, TimeoutErr, err)
+	// Verify that command gets killed in 20ms, with some breathing room
+	assert.True(t, elapsed < time.Millisecond*75)
+}
+
+// Verifies behavior of a command that doesn't get killed.
+func TestRunTimeoutFastExit(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test due to random failures.")
+	}
+	if echobin == "" {
+		t.Skip("'echo' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(echobin)
+	start := time.Now()
+	err := RunTimeout(cmd, time.Millisecond*20)
+	buf := &bytes.Buffer{}
+	log.SetOutput(buf)
+	elapsed := time.Since(start)
+
+	require.NoError(t, err)
+	// Verify that command gets killed in 20ms, with some breathing room
+	assert.True(t, elapsed < time.Millisecond*75)
+
+	// Verify "process already finished" log doesn't occur.
+	time.Sleep(time.Millisecond * 75)
+	require.Equal(t, "", buf.String())
+}
+
+func TestCombinedOutputTimeout(t *testing.T) {
+	t.Skip("Test failing too often, skip for now and revisit later.")
+	if sleepbin == "" {
+		t.Skip("'sleep' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(sleepbin, "10")
+	start := time.Now()
+	_, err := CombinedOutputTimeout(cmd, time.Millisecond*20)
+	elapsed := time.Since(start)
+	assert.Equal(t, TimeoutErr, err)
+	// Verify that command gets killed in 20ms, with some breathing room
+	assert.True(t, elapsed < time.Millisecond*75)
+}
+
+func TestCombinedOutput(t *testing.T) {
+	if echobin == "" {
+		t.Skip("'echo' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(echobin, "foo")
+	out, err := CombinedOutputTimeout(cmd, time.Second)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "foo\n", string(out))
+}
+
+// test that CombinedOutputTimeout and exec.Cmd.CombinedOutput return
+// the same output from a failed command.
+func TestCombinedOutputError(t *testing.T) {
+	if shell == "" {
+		t.Skip("'sh' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(shell, "-c", "false")
+	expected, err := cmd.CombinedOutput()
+
+	cmd2 := exec.Command(shell, "-c", "false")
+	actual, err := CombinedOutputTimeout(cmd2, time.Second)
+
+	assert.Error(t, err)
+	assert.Equal(t, expected, actual)
+}
+
+func TestRunError(t *testing.T) {
+	if shell == "" {
+		t.Skip("'sh' binary not available on OS, skipping.")
+	}
+	cmd := exec.Command(shell, "-c", "false")
+	err := RunTimeout(cmd, time.Second)
+
+	assert.Error(t, err)
+}

pkg/util/shell/exec_unix.go

+//go:build !windows
+// +build !windows
+
+/*
+Copyright (c) 2021 OceanBase
+Copyright (c) 2015-2020 InfluxData Inc.
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package shell
+
+import (
+	"errors"
+	"os/exec"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+var TimeoutErr = errors.New("Command timed out.")
+
+// KillGrace is the amount of time we allow a process to shutdown before sending a SIGKILL
+const KillGrace = 5 * time.Second
+
+// WaitTimeout waits for the given command to finish with a timeout
+// It assumes the command has already been started
+// If the command times out, it attempts to kill the process
+func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
+	var kill *time.Timer
+	term := time.AfterFunc(timeout, func() {
+		err := c.Process.Signal(syscall.SIGTERM)
+		if err != nil {
+			log.Errorf("[agent] Error terminating process: %s", err)
+			return
+		}
+		kill = time.AfterFunc(KillGrace, func() {
+			err := c.Process.Kill()
+			if err != nil {
+				log.Errorf("[agent] Error killing process: %s", err)
+				return
+			}
+		})
+	})
+
+	err := c.Wait()
+
+	// Shutdown all timers
+	if kill != nil {
+		kill.Stop()
+	}
+	termSent := !term.Stop()
+
+	// If the process exited without error treat it as success
+	// This allows a process to do a clean shutdown on signal
+	if err == nil {
+		return nil
+	}
+
+	// If SIGTERM was sent then treat any process error as a timeout
+	if termSent {
+		return TimeoutErr
+	}
+
+	// Otherwise there was an error unrelated to termination
+	return err
+}

pkg/util/signal.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+var FuncList []func()
+var onlyOneSignalHandler = make(chan struct{})
+var shutdownSignals = []os.Signal{syscall.SIGTERM, syscall.SIGINT}
+
+// SignalHandler registers for SIGTERM and SIGINT
+func SignalHandler(funcList []func()) context.Context {
+	close(onlyOneSignalHandler)
+	ctx, cancel := context.WithCancel(context.Background())
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, shutdownSignals...)
+	go func() {
+		<-c
+		klog.Infoln("stoping server")
+		cancel()
+		// custom closing logic
+		if len(funcList) > 0 {
+			for _, f := range funcList {
+				f()
+			}
+		}
+		time.Sleep(1 * time.Second)
+		os.Exit(0)
+	}()
+	return ctx
+}

pkg/util/slice.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+func CompareSlice(currentList, expectedList []int) int {
+	for _, i := range currentList {
+		for index, j := range expectedList {
+			if i == j {
+				expectedList = append(expectedList[:index], expectedList[index+1:]...)
+				break
+			}
+		}
+	}
+	return expectedList[0]
+}

pkg/util/string.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package util
+
+func TrimSuffixLastest(s string) string {
+	s = s[:len(s)-1]
+	return s
+}

pkg/util/system/exit.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package system
+
+import (
+	"os"
+	"time"
+
+	"github.com/oceanbase/ob-operator/pkg/util"
+)
+
+func Exit() {
+	if len(util.FuncList) > 0 {
+		for _, f := range util.FuncList {
+			f()
+		}
+	}
+	time.Sleep(1 * time.Second)
+	os.Exit(0)
+}

pkg/util/system/network.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package system
+
+import (
+	"net"
+)
+
+func GetNICInfo(name string) (map[string]string, error) {
+	res := make(map[string]string)
+	netInterfaces, err := net.Interfaces()
+	if err != nil {
+		return res, err
+	}
+	for _, netInterface := range netInterfaces {
+		if netInterface.Name == name {
+			addrs, _ := netInterface.Addrs()
+			res["nic"] = name
+			res["ip"] = addrs[0].String()
+		}
+	}
+	return res, nil
+}

pkg/util/system/process.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package system
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/oceanbase/ob-operator/pkg/util/shell"
+	"github.com/pkg/errors"
+	psutil "github.com/shirou/gopsutil/v3/process"
+)
+
+type ProcessManager struct {
+}
+
+type ProcessInfo struct {
+	Pid               int32     `json:"pid"`
+	Name              string    `json:"name"`              // process name
+	StartCommand      string    `json:"startCommand"`      // process command line, with arguments
+	Username          string    `json:"username"`          // username of the process
+	Ports             []int     `json:"ports"`             // the host ports the process occupied
+	CreateTime        time.Time `json:"createTime"`        // process create time
+	ElapsedTimeMillis int64     `json:"elapsedTimeMillis"` // elapsed time since process created
+}
+
+const GetProcessTcpListenCommand = "netstat -tunlp | { grep '%d/' || true; }"
+
+func (p ProcessManager) listProcesses() ([]*psutil.Process, error) {
+	processes, err := psutil.Processes()
+	if err != nil {
+		return nil, errors.Errorf("failed to list processes: %s", err)
+	}
+	return processes, nil
+}
+
+func (p ProcessManager) findProcessByName(name string) ([]*psutil.Process, error) {
+	processes, err := p.listProcesses()
+	if err != nil {
+		return nil, errors.Wrapf(err, "find process by name %s", name)
+	}
+	var result []*psutil.Process
+	for _, proc := range processes {
+		if procName, err := proc.Name(); err == nil && procName == name {
+			result = append(result, proc)
+		}
+	}
+	return result, nil
+}
+
+func (p ProcessManager) FindProcessInfoByName(name string) ([]*ProcessInfo, error) {
+	processes, err := p.findProcessByName(name)
+	if err != nil {
+		return nil, errors.Wrapf(err, "find process info by name %s", name)
+	}
+	result := make([]*ProcessInfo, 0)
+	for _, proc := range processes {
+		result = append(result, processToProcessInfo(proc))
+	}
+	return result, nil
+}
+
+func (p ProcessManager) ProcessExists(name string) (bool, error) {
+	processes, err := p.findProcessByName(name)
+	if err != nil {
+		return false, errors.Wrapf(err, "check process exists by name %s", name)
+	}
+	return len(processes) > 0, nil
+}
+
+func (p ProcessManager) ProcessIsRunningByName(name string) bool {
+	processes, err := p.findProcessByName(name)
+	if err != nil {
+		log.Printf("find process info by name %s %s", name, err)
+		return false
+	}
+	var status bool
+	for _, proc := range processes {
+		procName, _ := proc.Name()
+		procStatus, _ := proc.Status()
+		log.Println(procName, proc.Pid, procStatus)
+		for _, ps := range procStatus {
+			if ps == "zombie" {
+				return false
+			}
+		}
+		status, err = proc.IsRunning()
+		if err != nil {
+			return false
+		}
+	}
+	return status
+}
+
+func (p ProcessManager) terminateProcess(proc *psutil.Process) error {
+	err := proc.Terminate()
+	if err != nil {
+		return errors.Errorf("failed to terminate process with pid %d: %s", proc.Pid, err)
+	}
+	return nil
+}
+
+func (p ProcessManager) TerminateProcessByName(name string) error {
+	processes, err := p.findProcessByName(name)
+	if err != nil {
+		return errors.Wrapf(err, "terminate process by name %s", name)
+	}
+	for _, proc := range processes {
+		err = p.terminateProcess(proc)
+		if err != nil {
+			return errors.Wrapf(err, "terminate process by name %s", name)
+		}
+	}
+	return nil
+}
+
+func (p ProcessManager) killProcess(proc *psutil.Process) error {
+	err := proc.Kill()
+	if err != nil {
+		return errors.Errorf("failed to kill process with pid %d, %s", proc.Pid, err)
+	}
+	return nil
+}
+
+func (p ProcessManager) KillProcessByName(name string) error {
+	processes, err := p.findProcessByName(name)
+	if err != nil {
+		return errors.Wrapf(err, "kill process by name %s", name)
+	}
+	for _, proc := range processes {
+		err = p.killProcess(proc)
+		if err != nil {
+			return errors.Wrapf(err, "kill process by name %s", name)
+		}
+	}
+	return nil
+}
+
+func processToProcessInfo(p *psutil.Process) *ProcessInfo {
+	pi := ProcessInfo{
+		Ports: []int{},
+	}
+	pi.Pid = p.Pid
+	if createTimeMillis, err := p.CreateTime(); err == nil {
+		createTime := time.Unix(createTimeMillis/1000, (createTimeMillis%1000)*1000)
+		pi.CreateTime = createTime
+		elapsedTime := time.Now().Sub(createTime)
+		pi.ElapsedTimeMillis = int64(elapsedTime / time.Millisecond)
+	}
+	if name, err := p.Name(); err == nil {
+		pi.Name = name
+	}
+	if cmdline, err := p.Cmdline(); err == nil {
+		pi.StartCommand = cmdline
+	}
+	if username, err := p.Username(); err == nil {
+		pi.Username = username
+	}
+	if ports, err := getProcessOccupiedPorts(p.Pid); err == nil {
+		pi.Ports = ports
+	}
+	return &pi
+}
+
+func getProcessOccupiedPorts(pid int32) ([]int, error) {
+	cmd := fmt.Sprintf(GetProcessTcpListenCommand, pid)
+	executeResult, err := shell.NewCommand(cmd).WithUser(shell.RootUser).Execute()
+	if err != nil {
+		return nil, errors.Wrap(err, "get process occupied ports")
+	}
+	output := strings.TrimSpace(executeResult.Output)
+	if output == "" {
+		return []int{}, nil
+	}
+	occupiedPorts := make([]int, 0)
+	tcpLines := strings.Split(output, "\n")
+	for _, tcpLine := range tcpLines {
+		port, ok := parseNetstatLine(tcpLine)
+		if !ok {
+			return nil, errors.Errorf("failed to get process occupied ports, invalid output line: %s", tcpLine)
+		}
+		occupiedPorts = append(occupiedPorts, port)
+	}
+	// One occupied port can corresponds to multiple netstat lines, so remove duplicate ports.
+	return removeDuplicate(occupiedPorts), nil
+}
+
+// Example:
+// Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
+// tcp        0      0 0.0.0.0:62888           0.0.0.0:*               LISTEN      104668/pos_proxy
+// tcp6       0      0 :::62888                :::*                    LISTEN      104668/pos_proxy
+func parseNetstatLine(line string) (int, bool) {
+	fields := strings.Fields(line)
+	if len(fields) != 7 {
+		return 0, false
+	}
+	field := fields[3]
+	if len(field) == 0 || !strings.Contains(field, ":") {
+		return 0, false
+	}
+	i := strings.LastIndex(field, ":")
+	if i == -1 {
+		return 0, false
+	}
+	portStr := field[i+1:]
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return 0, false
+	}
+	return port, true
+}
+
+func removeDuplicate(ports []int) []int {
+	if len(ports) == 0 {
+		return []int{}
+	}
+	result := make([]int, 0)
+	var seen = make(map[int]bool, len(ports))
+	for _, port := range ports {
+		if _, exists := seen[port]; !exists {
+			result = append(result, port)
+			seen[port] = true
+		}
+	}
+	return result
+}

scripts/observer/Dockerfile

+# build builder
+FROM golang:1.16 as builder
+
+WORKDIR /workspace
+
+# Copy the Go Modules manifests
+COPY . .
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOPROXY=https://mirrors.aliyun.com/goproxy/ go build -a -o cable cmd/cable/app.go
+
+# build server
+FROM openanolis/anolisos:8.4-x86_64
+
+RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+WORKDIR /app
+
+COPY --from=builder /workspace/cable .
+RUN chmod +x /app/cable
+
+RUN useradd -m admin
+ENV OB_VERSION 3.1.2-10000392021123010
+RUN rpm -ivh https://mirrors.aliyun.com/oceanbase/community/stable/el/8/x86_64/oceanbase-ce-libs-${OB_VERSION}.el8.x86_64.rpm
+RUN rpm -ivh https://mirrors.aliyun.com/oceanbase/community/stable/el/8/x86_64/oceanbase-ce-${OB_VERSION}.el8.x86_64.rpm
+RUN mkdir -p /home/admin/oceanbase/run
+RUN mkdir -p /home/admin/oceanbase/store/sort_dir
+RUN mkdir -p /home/admin/oceanbase/store/sstable
+RUN mkdir -p /home/admin/oceanbase/store/clog
+RUN mkdir -p /home/admin/oceanbase/store/ilog
+RUN mkdir -p /home/admin/oceanbase/store/slog
+RUN mkdir -p /home/admin/oceanbase/log
+RUN chown -R admin:admin /home/admin/oceanbase/
+
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--"]
+CMD ["/app/cable"]

scripts/observer/Makefile

+# Ensure Make is run with bash shell as some syntax below is bash-specific
+SHELL:=/usr/bin/env bash
+
+.DEFAULT_GOAL := help
+
+# Code
+VERSION := operator-v3.1.2-10000392021123010
+
+# Define Docker related variables. Releases should modify and double check these vars.
+REGISTRY := oceanbase
+IMAGE := oceanbase-ce
+
+# Use GOPROXY environment variable if set
+GOPROXY := $(shell go env GOPROXY)
+ifeq ($(GOPROXY),)
+GOPROXY := https://mirrors.aliyun.com/goproxy/
+endif
+
+export GOPROXY
+# Active module mode, as we use go modules to manage dependencies
+export GO111MODULE=on
+
+run:
+	go run ../../cmd/cable/app.go
+
+server:
+	@echo "version: $(VERSION)"
+	cd ../../ && docker build -t $(REGISTRY)/$(IMAGE):$(VERSION) -f scripts/observer/Dockerfile .
+	docker push $(REGISTRY)/$(IMAGE):$(VERSION)

test/e2e/README-CN.md

+# e2e 测试
+
+测试需要真实的 Kubernetes 环境。
+测试套件会在 Kubernetes 集群中部署应用，并测试应用是否达到预期。
+
+## 部署 ob-operator
+
+其中，operator 启动参数 `--cluster-name` 需要与测试所使用的 YAML 中配置的保持一致。
+
+以 YAML 方式部署为例：
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      containers:
+      - args:
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --leader-elect
+        - --cluster-name=test  # 需要与测试 YAML 中配置的保持一致，默认是 cn
+        command:
+        - /manager
+        image: ob-operator:latest
+        imagePullPolicy: Always
+        name: manager
+```
+
+需要测试环境的网络与 Kubernetes 集群内部网络互通，或者在 Kubernetes 集群内跑测试。
+observer 阶段的测试会访问 observer 的 Service ClusterIP。
+
+## 测试 statefulapp
+
+```
+ginkgo --focus=/statefulapp/ --regexScansFilePath=true --slowSpecThreshold=3600 --progress --reportPassed
+```
+
+## 测试 observer
+
+```
+ginkgo --focus=/observer/ --regexScansFilePath=true --slowSpecThreshold=3600 --progress --reportPassed
+```

test/e2e/README.md

+# e2e Test
+
+The test requires a real Kubernetes environment.
+The test suite deploys the application in a Kubernetes cluster and tests whether the application meets expectations.
+
+## Deploy ob-operator
+
+The operator startup parameter `--cluster-name` must be the same as that configured in YAML used for the test.
+
+The following uses YAML to deployment operator as an example:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: ob-operator-controller-manager
+  namespace: oceanbase-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: controller-manager
+  template:
+    metadata:
+      labels:
+        control-plane: controller-manager
+    spec:
+      containers:
+      - args:
+        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=127.0.0.1:8080
+        - --leader-elect
+        - --cluster-name=test  # the value must be the same as that configured in YAML used for the test, The default value is cn
+        command:
+        - /manager
+        image: ob-operator:latest
+        imagePullPolicy: Always
+        name: manager
+```
+
+The network that requires the test environment can communicate with the internal network of the Kubernetes cluster or run tests within the Kubernetes cluster.
+Tests in the observer process access the ClusterIP of the Service created for the observer.
+
+## Test for statefulapp
+
+```
+ginkgo --focus=/statefulapp/ --regexScansFilePath=true --slowSpecThreshold=3600 --progress --reportPassed
+```
+
+## Test for observer
+
+```
+ginkgo --focus=/observer/ --regexScansFilePath=true --slowSpecThreshold=3600 --progress --reportPassed
+```

test/e2e/converter/convert.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"encoding/json"
+	"log"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func ConvertObjToUnstructured(k8sObj interface{}, u *unstructured.Unstructured) {
+	tmp, err := json.Marshal(k8sObj)
+	if err != nil {
+		log.Println(err)
+	}
+	_ = u.UnmarshalJSON(tmp)
+}
+
+func ConvertUnstructuredToObj(u *unstructured.Unstructured, k8sObj interface{}) {
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), k8sObj)
+}

test/e2e/converter/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+)
+
+func IsOBClusterInstanceReady(obcluster cloudv1.OBCluster) bool {
+	if obcluster.Status.Status != observerconst.TopologyReady {
+		return false
+	}
+	for _, cluster := range obcluster.Status.Topology {
+		if cluster.ClusterStatus != observerconst.ClusterReady {
+			return false
+		}
+		for _, zone := range cluster.Zone {
+			if zone.ZoneStatus != observerconst.OBZoneReady {
+				return false
+			}
+			if zone.ExpectedReplicas != zone.AvailableReplicas {
+				return false
+			}
+		}
+	}
+	return true
+}

test/e2e/converter/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/core/converter"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func JudgeAllOBServerStatusByObj(obServerList []model.AllServer, obj unstructured.Unstructured) bool {
+	var instance cloudv1.OBCluster
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &instance)
+	return converter.IsAllOBServerActive(obServerList, instance.Spec.Topology)
+}

test/e2e/converter/obzone.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func JudgeOBzoneStatusByObj(obServerList []model.AllServer, obzone cloudv1.OBZone) bool {
+	for _, cluster := range obzone.Status.Topology {
+		for _, zone := range cluster.Zone {
+			for _, node := range zone.Nodes {
+				var nodeIsExists bool
+				nodeIsExists = false
+				for _, obServer := range obServerList {
+					if obServer.SvrIP == node.ServerIP && obServer.Status == node.Status {
+						nodeIsExists = true
+						break
+					}
+				}
+				if !nodeIsExists {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}

test/e2e/converter/rootservice.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"strconv"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+)
+
+func JudgeRootserviceStatusByObj(rsList []model.AllVirtualCoreMeta, rs cloudv1.RootService) bool {
+	for _, cluster := range rs.Status.Topology {
+		for _, zone := range cluster.Zone {
+			var nodeIsExists bool
+			nodeIsExists = false
+			for _, node := range rsList {
+				if node.SvrIP == zone.ServerIP && strconv.FormatInt(node.Role, 10) == zone.Status {
+					nodeIsExists = true
+					break
+				}
+			}
+			if !nodeIsExists {
+				return false
+			}
+		}
+	}
+	return true
+}

test/e2e/converter/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	statefulappCore "github.com/oceanbase/ob-operator/pkg/controllers/statefulapp/const"
+)
+
+func IsStatefulappInstanceReady(statefulappInstance cloudv1.StatefulApp) bool {
+	if statefulappInstance.Status.ClusterStatus != statefulappCore.Ready {
+		return false
+	}
+	for _, subset := range statefulappInstance.Status.Subsets {
+		if subset.AvailableReplicas != subset.ExpectedReplicas {
+			return false
+		}
+		for _, pod := range subset.Pods {
+			if pod.PodPhase != statefulappCore.PodStatusRunning {
+				return false
+			}
+			for _, pvc := range pod.PVCs {
+				if pvc.Phase != statefulappCore.Bound {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}

test/e2e/converter/yaml.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package converter
+
+import (
+	"io/ioutil"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+)
+
+func MakeObjectFromFile(filePath string) interface{} {
+	yamlFile, _ := ioutil.ReadFile(filePath)
+	obj := &unstructured.Unstructured{}
+	dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
+	_, _, _ = dec.Decode(yamlFile, nil, obj)
+	return *obj
+}
+
+func GetObjFromYaml(filePath string) unstructured.Unstructured {
+	obj := MakeObjectFromFile(filePath)
+	return obj.(unstructured.Unstructured)
+}

test/e2e/data/obcluster-01.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: ob
+  name: test-ob
+spec:
+  version: 3.1.1-4
+  clusterID: 1
+  topology:
+    - cluster: test
+      zone:
+      - name: zone1
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 1
+      - name: zone2
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 1
+      - name: zone3
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 1
+  resources:
+    cpu: 2
+    memory: 10Gi
+    storage:
+      - name: data-file
+        storageClassName: "local-path"
+        size: 40Gi
+      - name: data-log
+        storageClassName: "local-path"
+        size: 40Gi
+      - name: log
+        storageClassName: "local-path"
+        size: 30Gi

test/e2e/data/obcluster-02.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: OBCluster
+metadata:
+  namespace: ob
+  name: test-ob
+spec:
+  version: 3.1.1-4
+  clusterID: 1
+  topology:
+    - cluster: test
+      zone:
+      - name: zone1
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone1
+        replicas: 2
+      - name: zone2
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone2
+        replicas: 2
+      - name: zone3
+        region: region-test
+        nodeSelector:
+          topology.kubernetes.io/zone: zone3
+        replicas: 2
+  resources:
+    cpu: 2
+    memory: 10Gi
+    storage:
+      - name: data-file
+        storageClassName: "local-path"
+        size: 40Gi
+      - name: data-log
+        storageClassName: "local-path"
+        size: 40Gi
+      - name: log
+        storageClassName: "local-path"
+        size: 30Gi

test/e2e/data/statefulapp-01.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: StatefulApp
+metadata:
+  namespace: ob
+  name: test-sapp-01
+spec:
+  cluster: test
+  subsets:
+  - name: zone1
+    nodeSelector:
+      topology.kubernetes.io/zone: zone1
+    replicas: 1
+  - name: zone2
+    nodeSelector:
+      topology.kubernetes.io/zone: zone2
+    replicas: 1
+  - name: zone3
+    nodeSelector:
+      topology.kubernetes.io/zone: zone3
+    replicas: 1
+  podTemplate:
+    containers:
+      - name: nginx
+        image: nginx
+        resources:
+          requests:
+            cpu: "1"
+            memory: "2Gi"
+        volumeMounts:
+          - name: log
+            mountPath: /home/admin/oceanbase/log
+          - name: data
+            mountPath: /home/admin/oceanbase/store
+    volumes:
+      - name: log
+        persistentVolumeClaim:
+          claimName: log
+      - name: data
+        persistentVolumeClaim:
+          claimName: data
+  storageTemplates:
+  - name: log
+    pvc:
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi
+  - name: data
+    pvc:
+      accessModes:
+          - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi

test/e2e/data/statefulapp-02.yaml

+apiVersion: cloud.oceanbase.com/v1
+kind: StatefulApp
+metadata:
+  namespace: ob
+  name: test-sapp-01
+spec:
+  cluster: test
+  subsets:
+  - name: zone1
+    nodeSelector:
+      topology.kubernetes.io/zone: zone1
+    replicas: 2
+  - name: zone2
+    nodeSelector:
+      topology.kubernetes.io/zone: zone2
+    replicas: 1
+  - name: zone3
+    nodeSelector:
+      topology.kubernetes.io/zone: zone3
+    replicas: 1
+  podTemplate:
+    containers:
+      - name: nginx
+        image: nginx
+        resources:
+          requests:
+            cpu: "1"
+            memory: "2Gi"
+        volumeMounts:
+          - name: log
+            mountPath: /home/admin/oceanbase/log
+          - name: data
+            mountPath: /home/admin/oceanbase/store
+    volumes:
+      - name: log
+        persistentVolumeClaim:
+          claimName: log
+      - name: data
+        persistentVolumeClaim:
+          claimName: data
+  storageTemplates:
+  - name: log
+    pvc:
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi
+  - name: data
+    pvc:
+      accessModes:
+          - ReadWriteOnce
+      storageClassName: "local-path"
+      resources:
+        requests:
+          storage: 1Gi

test/e2e/e2e.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package e2e
+
+import (
+	_ "github.com/oceanbase/ob-operator/test/e2e/observer"
+	_ "github.com/oceanbase/ob-operator/test/e2e/statefulapp"
+)
+
+func runE2E() {}

test/e2e/e2e_test.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package e2e
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestRunE2E(t *testing.T) {
+	runE2E()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ob-operator e2e test suites")
+}

test/e2e/go.mod

+module github.com/oceanbase/ob-operator/test/e2e
+
+go 1.16
+
+require (
+	github.com/oceanbase/ob-operator v0.0.0-00010101000000-000000000000
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.17.0
+	k8s.io/api v0.22.1
+	k8s.io/apimachinery v0.22.1
+	k8s.io/client-go v0.22.1
+	sigs.k8s.io/controller-runtime v0.10.0
+)
+
+replace github.com/oceanbase/ob-operator => ../../

test/e2e/go.sum

+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
+github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
+github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
+github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
+github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
+github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
+github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
+github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shirou/gopsutil/v3 v3.20.10/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
+go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
+go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
+go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
+go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
+go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
+go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
+go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
+go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
+go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
+go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
+go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
+golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
+gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY=
+k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
+k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE=
+k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c=
+k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM=
+k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400=
+k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw=
+k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
+k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
+k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo=
+k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
+k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2MoKj5U=
+sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

test/e2e/observer/const.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"time"
+)
+
+const (
+	TryInterval                   = 1 * time.Second
+	OBClusterCreateTimeout        = 5 * time.Second
+	OBClusterBootstrapTimeout     = 600 * time.Second
+	OBClusterReadyTimeout         = 30 * time.Second
+	StatefulappUpdateReadyTimeout = 60 * time.Second
+	OBClusterUpdateTReadyimeout   = 120 * time.Second
+)

test/e2e/observer/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package observer
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+
+	observerconst "github.com/oceanbase/ob-operator/pkg/controllers/observer/const"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/model"
+	"github.com/oceanbase/ob-operator/pkg/controllers/observer/sql"
+	testconverter "github.com/oceanbase/ob-operator/test/e2e/converter"
+	testresource "github.com/oceanbase/ob-operator/test/e2e/resource"
+)
+
+var _ = ginkgo.Describe("observer test pipeline", func() {
+
+	var _ = ginkgo.Describe("create default obcluster pipeline", func() {
+
+		var obServerList []model.AllServer
+		var service corev1.Service
+		defaultOBcluster := testconverter.GetObjFromYaml("./data/obcluster-01.yaml")
+		obclusterNamespace := defaultOBcluster.GetNamespace()
+		obclusterName := defaultOBcluster.GetName()
+		statefulappName := fmt.Sprintf("sapp-%s", defaultOBcluster.GetName())
+		serviceName := fmt.Sprintf("svc-%s", defaultOBcluster.GetName())
+		rootserviceName := fmt.Sprintf("rs-%s", defaultOBcluster.GetName())
+		obzoneName := fmt.Sprintf("obzone-%s", defaultOBcluster.GetName())
+
+		ginkgo.It("step: create default obcluster", func() {
+			err := testresource.TestClient.CreateObj(defaultOBcluster)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create default obcluster succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for obcluster created", func() {
+			time.Sleep(OBClusterCreateTimeout)
+		})
+
+		ginkgo.It("step: check default obcluster bootstrap status", func() {
+			gomega.Eventually(func() string {
+				return testresource.TestClient.GetOBClusterStatus(obclusterNamespace, obclusterName)
+			}, OBClusterBootstrapTimeout, TryInterval).Should(
+				gomega.Equal(observerconst.TopologyReady),
+				"obcluster %s bootstrap ready", obclusterName,
+			)
+		})
+
+		ginkgo.It("step: check statefulapp status for obcluster", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(obclusterNamespace, statefulappName)
+			}, OBClusterReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+		ginkgo.It("step: check default obcluster status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeOBClusterInstanceIsReadyByObj(obclusterNamespace, obclusterName)
+			}, OBClusterReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"obcluster %s is ready", obclusterName,
+			)
+		})
+
+		ginkgo.It("step: check service status for obcluster", func() {
+			var err error
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeServicefForOBClusterIsReadyByObj(obclusterNamespace, serviceName)
+			}, OBClusterReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"service %s is ready", serviceName,
+			)
+			service, err = testresource.TestClient.GetService(obclusterNamespace, serviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get service %s succeed", serviceName,
+			)
+		})
+
+		ginkgo.It("step: check observer status", func() {
+			obServerList = sql.GetOBServer(service.Spec.ClusterIP)
+			status := testconverter.JudgeAllOBServerStatusByObj(obServerList, defaultOBcluster)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"all observer is active",
+			)
+		})
+
+		ginkgo.It("step: check rootservice status", func() {
+			rootservice, err := testresource.TestClient.GetRootService(obclusterNamespace, rootserviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get rootservice %s succeed", rootserviceName,
+			)
+			rsList := sql.GetRootService(service.Spec.ClusterIP)
+			status := testconverter.JudgeRootserviceStatusByObj(rsList, rootservice)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"rootservice %s status is ok", rootserviceName,
+			)
+		})
+
+		ginkgo.It("step: check obzone status", func() {
+			obzone, err := testresource.TestClient.GetOBZone(obclusterNamespace, obzoneName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get obzone %s succeed", obzoneName,
+			)
+			status := testconverter.JudgeOBzoneStatusByObj(obServerList, obzone)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"obzone %s status is ok", obzoneName,
+			)
+		})
+
+	})
+
+	var _ = ginkgo.Describe("add observer pipeline", func() {
+
+		var obServerList []model.AllServer
+		var service corev1.Service
+		addOBcluster := testconverter.GetObjFromYaml("./data/obcluster-02.yaml")
+		obclusterNamespace := addOBcluster.GetNamespace()
+		obclusterName := addOBcluster.GetName()
+		statefulappName := fmt.Sprintf("sapp-%s", addOBcluster.GetName())
+		serviceName := fmt.Sprintf("svc-%s", addOBcluster.GetName())
+		rootserviceName := fmt.Sprintf("rs-%s", addOBcluster.GetName())
+		obzoneName := fmt.Sprintf("obzone-%s", addOBcluster.GetName())
+
+		ginkgo.It("step: update obcluster for add observer by zone", func() {
+			err := testresource.TestClient.UpdateOBClusterInstance(addOBcluster)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"update obcluster succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for obcluster updated", func() {
+			time.Sleep(OBClusterCreateTimeout)
+		})
+
+		ginkgo.It("step: check statefulapp status for obcluster", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(obclusterNamespace, statefulappName)
+			}, StatefulappUpdateReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+		ginkgo.It("step: check obcluster status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeOBClusterInstanceIsReadyByObj(obclusterNamespace, obclusterName)
+			}, OBClusterUpdateTReadyimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"obcluster %s is ready", obclusterName,
+			)
+		})
+
+		ginkgo.It("step: get service for obcluster", func() {
+			var err error
+			service, err = testresource.TestClient.GetService(obclusterNamespace, serviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get service %s succeed", serviceName,
+			)
+		})
+
+		ginkgo.It("step: check observer status", func() {
+			obServerList = sql.GetOBServer(service.Spec.ClusterIP)
+			status := testconverter.JudgeAllOBServerStatusByObj(obServerList, addOBcluster)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"all observer is active",
+			)
+		})
+
+		ginkgo.It("step: check rootservice status", func() {
+			rootservice, err := testresource.TestClient.GetRootService(obclusterNamespace, rootserviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get rootservice %s succeed", rootserviceName,
+			)
+			rsList := sql.GetRootService(service.Spec.ClusterIP)
+			status := testconverter.JudgeRootserviceStatusByObj(rsList, rootservice)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"rootservice %s status is ok", rootserviceName,
+			)
+		})
+
+		ginkgo.It("step: check obzone status", func() {
+			obzone, err := testresource.TestClient.GetOBZone(obclusterNamespace, obzoneName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get obzone %s succeed", obzoneName,
+			)
+			status := testconverter.JudgeOBzoneStatusByObj(obServerList, obzone)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"obzone %s status is ok", obzoneName,
+			)
+		})
+
+	})
+
+	var _ = ginkgo.Describe("delete observer pipeline", func() {
+
+		var obServerList []model.AllServer
+		var service corev1.Service
+		delOBcluster := testconverter.GetObjFromYaml("./data/obcluster-01.yaml")
+		obclusterNamespace := delOBcluster.GetNamespace()
+		obclusterName := delOBcluster.GetName()
+		statefulappName := fmt.Sprintf("sapp-%s", delOBcluster.GetName())
+		serviceName := fmt.Sprintf("svc-%s", delOBcluster.GetName())
+		rootserviceName := fmt.Sprintf("rs-%s", delOBcluster.GetName())
+		obzoneName := fmt.Sprintf("obzone-%s", delOBcluster.GetName())
+
+		ginkgo.It("step: update obcluster for del observer by zone", func() {
+			err := testresource.TestClient.UpdateOBClusterInstance(delOBcluster)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"update obcluster succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for obcluster updated", func() {
+			time.Sleep(OBClusterCreateTimeout)
+		})
+
+		ginkgo.It("step: check statefulapp status for obcluster", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(obclusterNamespace, statefulappName)
+			}, StatefulappUpdateReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+		ginkgo.It("step: check obcluster status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeOBClusterInstanceIsReadyByObj(obclusterNamespace, obclusterName)
+			}, OBClusterUpdateTReadyimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"obcluster %s is ready", obclusterName,
+			)
+		})
+
+		ginkgo.It("step: get service for obcluster", func() {
+			var err error
+			service, err = testresource.TestClient.GetService(obclusterNamespace, serviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get service %s succeed", serviceName,
+			)
+		})
+
+		ginkgo.It("step: check observer status", func() {
+			obServerList = sql.GetOBServer(service.Spec.ClusterIP)
+			status := testconverter.JudgeAllOBServerStatusByObj(obServerList, delOBcluster)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"all observer is active",
+			)
+		})
+
+		ginkgo.It("step: check rootservice status", func() {
+			rootservice, err := testresource.TestClient.GetRootService(obclusterNamespace, rootserviceName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get rootservice %s succeed", rootserviceName,
+			)
+			rsList := sql.GetRootService(service.Spec.ClusterIP)
+			status := testconverter.JudgeRootserviceStatusByObj(rsList, rootservice)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"rootservice %s status is ok", rootserviceName,
+			)
+		})
+
+		ginkgo.It("step: check obzone status", func() {
+			obzone, err := testresource.TestClient.GetOBZone(obclusterNamespace, obzoneName)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"get obzone %s succeed", obzoneName,
+			)
+			status := testconverter.JudgeOBzoneStatusByObj(obServerList, obzone)
+			gomega.Expect(status).To(
+				gomega.Equal(true),
+				"obzone %s status is ok", obzoneName,
+			)
+		})
+
+	})
+
+})

test/e2e/resource/client.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+type Client struct {
+	ClientSet       *kubernetes.Clientset
+	DynamicClient   dynamic.Interface
+	DiscoveryClient *discovery.DiscoveryClient
+}
+
+func NewClient() (*rest.Config, *Client) {
+	client := new(Client)
+	if _, err := os.Stat(filepath.Join(os.Getenv("HOME"), ".kube", "config")); err != nil {
+		config, _ := rest.InClusterConfig()
+		client.ClientSet, _ = kubernetes.NewForConfig(config)
+		client.DynamicClient, _ = dynamic.NewForConfig(config)
+		client.DiscoveryClient, _ = discovery.NewDiscoveryClientForConfig(config)
+		return config, client
+	} else {
+		filePath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+		config, _ := clientcmd.BuildConfigFromFlags("", filePath)
+		client.ClientSet, _ = kubernetes.NewForConfig(config)
+		client.DynamicClient, _ = dynamic.NewForConfig(config)
+		client.DiscoveryClient, _ = discovery.NewDiscoveryClientForConfig(config)
+		return config, client
+	}
+}
+
+func (client *Client) GetResource() ([]*metav1.APIGroup, []*metav1.APIResourceList) {
+	group, source, _ := client.DiscoveryClient.ServerGroupsAndResources()
+	return group, source
+}
+
+func (client *Client) GetKind(kind string) string {
+	_, resourceList := client.GetResource()
+	for _, list := range resourceList {
+		for _, resource := range list.APIResources {
+			if resource.Kind == kind {
+				return resource.Name
+			}
+		}
+	}
+	return ""
+}
+
+func (client *Client) GetGVR(unStruct *unstructured.Unstructured) *schema.GroupVersionResource {
+	gvk := unStruct.GroupVersionKind()
+	kind := client.GetKind(gvk.Kind)
+	return &schema.GroupVersionResource{
+		Group:    gvk.Group,
+		Version:  gvk.Version,
+		Resource: kind,
+	}
+}
+
+func (client *Client) CreateObj(obj unstructured.Unstructured) error {
+	gvr := client.GetGVR(&obj)
+	_, err := client.DynamicClient.Resource(*gvr).Namespace(obj.GetNamespace()).Create(context.TODO(), &obj, metav1.CreateOptions{})
+	if err != nil {
+		log.Println(err)
+		return err
+	}
+	return nil
+}
+
+func (client *Client) GetObj(obj unstructured.Unstructured) (interface{}, error) {
+	gvr := client.GetGVR(&obj)
+	res, err := client.DynamicClient.Resource(*gvr).Namespace(obj.GetNamespace()).Get(context.TODO(), obj.GetName(), metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return res, err
+	}
+	return res, nil
+}
+
+func (client *Client) UpdateObj(obj unstructured.Unstructured) error {
+	gvr := client.GetGVR(&obj)
+	_, err := client.DynamicClient.Resource(*gvr).Namespace(obj.GetNamespace()).Update(context.TODO(), &obj, metav1.UpdateOptions{})
+	if err != nil {
+		log.Println(err)
+		return err
+	}
+	return nil
+}
+
+func (client *Client) DeleteObj(obj unstructured.Unstructured) {
+	gvr := client.GetGVR(&obj)
+	_ = client.DynamicClient.Resource(*gvr).Namespace(obj.GetNamespace()).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{})
+}

test/e2e/resource/obcluster.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	testconverter "github.com/oceanbase/ob-operator/test/e2e/converter"
+)
+
+const (
+	OBClusterGroup    = "cloud.oceanbase.com"
+	OBClusterVersion  = "v1"
+	OBClusterKind     = "OBCluster"
+	OBClusterResource = "obclusters"
+)
+
+var (
+	OBClusterRes = schema.GroupVersionResource{
+		Group:    OBClusterGroup,
+		Version:  OBClusterVersion,
+		Resource: OBClusterResource,
+	}
+)
+
+func (client *Client) GetOBClusterInstance(namespace, name string) (cloudv1.OBCluster, error) {
+	var instance cloudv1.OBCluster
+	obj, err := client.DynamicClient.Resource(OBClusterRes).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return instance, err
+	}
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &instance)
+	return instance, nil
+}
+
+func (client *Client) GetOBClusterStatus(namespace, name string) string {
+	instance, err := client.GetOBClusterInstance(namespace, name)
+	if err == nil {
+		return instance.Status.Status
+	}
+	return ""
+}
+
+func (client *Client) UpdateOBClusterInstance(obj unstructured.Unstructured) error {
+	oldObj, _ := client.GetObj(obj)
+	obj.SetResourceVersion(oldObj.(*unstructured.Unstructured).GetResourceVersion())
+	err := client.UpdateObj(obj)
+	if err != nil {
+		log.Println(err)
+		return err
+	}
+	return nil
+}
+
+func (client *Client) JudgeOBClusterInstanceIsReadyByObj(namespace, name string) bool {
+	instance, err := client.GetOBClusterInstance(namespace, name)
+	if err == nil {
+		return testconverter.IsOBClusterInstanceReady(instance)
+	}
+	return false
+}

test/e2e/resource/observer.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource

test/e2e/resource/obzone.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+const (
+	OBZoneGroup         = "cloud.oceanbase.com"
+	OBZoneGroupVersion  = "v1"
+	OBZoneGroupKind     = "OBZone"
+	OBZoneGroupResource = "obzones"
+)
+
+var (
+	OBZoneRes = schema.GroupVersionResource{
+		Group:    OBZoneGroup,
+		Version:  OBZoneGroupVersion,
+		Resource: OBZoneGroupResource,
+	}
+)
+
+func (client *Client) GetOBZone(namespace, name string) (cloudv1.OBZone, error) {
+	var instance cloudv1.OBZone
+	obj, err := client.DynamicClient.Resource(RootServiceRes).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return instance, err
+	}
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &instance)
+	return instance, nil
+}

test/e2e/resource/pod.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (client *Client) GetPodStatus(namespace, name string) corev1.PodPhase {
+	instance, err := client.IsPodExists(namespace, name)
+	if err == nil {
+		return instance.Status.Phase
+	}
+	return ""
+}
+
+func (client *Client) IsPodExists(namespace, name string) (corev1.Pod, error) {
+	instance, err := client.ClientSet.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return corev1.Pod{}, err
+	}
+	return *instance, nil
+}
+
+func (client *Client) DeletePod(namespace, name string) error {
+	err := client.ClientSet.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+	if err != nil {
+		log.Println(err)
+		return err
+	}
+	return nil
+}

test/e2e/resource/rootservice.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+)
+
+const (
+	RootServiceGroup    = "cloud.oceanbase.com"
+	RootServiceVersion  = "v1"
+	RootServiceKind     = "RootService"
+	RootServiceResource = "rootservices"
+)
+
+var (
+	RootServiceRes = schema.GroupVersionResource{
+		Group:    RootServiceGroup,
+		Version:  RootServiceVersion,
+		Resource: RootServiceResource,
+	}
+)
+
+func (client *Client) GetRootService(namespace, name string) (cloudv1.RootService, error) {
+	var instance cloudv1.RootService
+	obj, err := client.DynamicClient.Resource(RootServiceRes).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return instance, err
+	}
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &instance)
+	return instance, nil
+}

test/e2e/resource/service.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (client *Client) GetService(namespace, name string) (corev1.Service, error) {
+	instance, err := client.ClientSet.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return corev1.Service{}, err
+	}
+	return *instance, nil
+}
+
+func (client *Client) JudgeServicefForOBClusterIsReadyByObj(namespace, name string) bool {
+	instance, err := client.ClientSet.CoreV1().Endpoints(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if len(instance.Subsets) > 0 {
+		addresses := instance.Subsets[0]
+		if len(addresses.Addresses) > 0 {
+			return true
+		}
+	}
+	return false
+}

test/e2e/resource/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"context"
+	"log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	testconverter "github.com/oceanbase/ob-operator/test/e2e/converter"
+)
+
+const (
+	StatefulAppGroup    = "cloud.oceanbase.com"
+	StatefulAppVersion  = "v1"
+	StatefulAppKind     = "StatefulApp"
+	StatefulAppResource = "statefulapps"
+)
+
+var (
+	StatefulappRes = schema.GroupVersionResource{
+		Group:    StatefulAppGroup,
+		Version:  StatefulAppVersion,
+		Resource: StatefulAppResource,
+	}
+)
+
+func (client *Client) GetStatefulappInstance(namespace, name string) (cloudv1.StatefulApp, error) {
+	var instance cloudv1.StatefulApp
+	obj, err := client.DynamicClient.Resource(StatefulappRes).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		log.Println(err)
+		return instance, err
+	}
+	_ = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &instance)
+	return instance, nil
+}
+
+func (client *Client) UpdateStatefulappInstance(obj unstructured.Unstructured) error {
+	oldObj, _ := client.GetObj(obj)
+	obj.SetResourceVersion(oldObj.(*unstructured.Unstructured).GetResourceVersion())
+	err := client.UpdateObj(obj)
+	if err != nil {
+		log.Println(err)
+		return err
+	}
+	return nil
+}
+
+func (client *Client) JudgeStatefulappInstanceIsReadyByObj(namespace, name string) bool {
+	instance, err := client.GetStatefulappInstance(namespace, name)
+	if err == nil {
+		return testconverter.IsStatefulappInstanceReady(instance)
+	}
+	return false
+}

test/e2e/resource/suite.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package resource
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	cloudv1 "github.com/oceanbase/ob-operator/apis/cloud/v1"
+	testconverter "github.com/oceanbase/ob-operator/test/e2e/converter"
+)
+
+var cfg *rest.Config
+var TestClient *Client
+var testEnv *envtest.Environment
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t,
+		"ob-operator test suite",
+		[]Reporter{printer.NewlineReporter{}})
+}
+
+var _ = BeforeSuite(func() {
+	config.DefaultReporterConfig.SlowSpecThreshold = 3600
+
+	format.MaxDepth = 1
+	format.UseStringerRepresentation = true
+	format.PrintContextObjects = false
+	format.TruncatedDiff = false
+
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
+	cfg, TestClient = NewClient()
+	Expect(TestClient).NotTo(BeNil())
+
+	By("bootstrapping test environment")
+	var useExistingCluster bool
+	useExistingCluster = true
+	testEnv = &envtest.Environment{
+		UseExistingCluster:    &useExistingCluster,
+		Config:                cfg,
+		CRDDirectoryPaths:     []string{filepath.Join("../../", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
+	}
+
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(cfg).NotTo(BeNil())
+
+	err = cloudv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+}, 60)
+
+var _ = AfterSuite(func() {
+	TestClient.DeleteObj(testconverter.MakeObjectFromFile("./data/statefulapp-01.yaml").(unstructured.Unstructured))
+	TestClient.DeleteObj(testconverter.MakeObjectFromFile("./data/statefulapp-02.yaml").(unstructured.Unstructured))
+
+	// TestClient.DeleteObj(testresource.MakeObjectFromFile("./data/obcluster-01.yaml").(unstructured.Unstructured))
+	// TestClient.DeleteObj(testresource.MakeObjectFromFile("./data/obcluster-02.yaml").(unstructured.Unstructured))
+
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).NotTo(HaveOccurred())
+})

test/e2e/statefulapp/const.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulapp
+
+import (
+	"time"
+)
+
+const (
+	TryInterval              = 1 * time.Second
+	StatefulappCreateTimeout = 5 * time.Second
+	StatefulappReadyTimeout  = 30 * time.Second
+	PodCreateTimeout         = 30 * time.Second
+	PodDeleteTimeout         = 20 * time.Second
+	PodFailoverTimeout       = 30 * time.Second
+)

test/e2e/statefulapp/statefulapp.go

+/*
+Copyright (c) 2021 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package statefulapp
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
+
+	testconverter "github.com/oceanbase/ob-operator/test/e2e/converter"
+	testresource "github.com/oceanbase/ob-operator/test/e2e/resource"
+)
+
+var _ = ginkgo.Describe("statefulapp test pipeline", func() {
+
+	var _ = ginkgo.Describe("create default statefulapp pipeline", func() {
+
+		defaultStatefulapp := testconverter.GetObjFromYaml("./data/statefulapp-01.yaml")
+		statefulappNamespace := defaultStatefulapp.GetNamespace()
+		statefulappName := defaultStatefulapp.GetName()
+
+		ginkgo.It("step: create default statefulapp", func() {
+			err := testresource.TestClient.CreateObj(defaultStatefulapp)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create default statefulapp succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for statefulapp created", func() {
+			time.Sleep(StatefulappCreateTimeout)
+		})
+
+		ginkgo.It("step: check default pod status", func() {
+			// is zone1Pod0 created
+			zone1Pod0 := fmt.Sprintf("%s-test-zone1-%d", statefulappName, 0)
+			_, err := testresource.TestClient.IsPodExists(statefulappNamespace, zone1Pod0)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create pod %s succeed", zone1Pod0,
+			)
+			// is zone1Pod0 running
+			gomega.Eventually(func() corev1.PodPhase {
+				return testresource.TestClient.GetPodStatus(statefulappNamespace, zone1Pod0)
+			}, PodCreateTimeout, TryInterval).Should(
+				gomega.Equal(corev1.PodRunning),
+				"pod %s is running", zone1Pod0,
+			)
+			// is zone2Pod0 created
+			zone2Pod0 := fmt.Sprintf("%s-test-zone2-%d", statefulappName, 0)
+			_, err = testresource.TestClient.IsPodExists(statefulappNamespace, zone2Pod0)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create pod %s succeed", zone2Pod0,
+			)
+			// is zone2Pod0 running
+			gomega.Eventually(func() corev1.PodPhase {
+				return testresource.TestClient.GetPodStatus(statefulappNamespace, zone2Pod0)
+			}, PodCreateTimeout, TryInterval).Should(
+				gomega.Equal(corev1.PodRunning),
+				"pod %s is running", zone2Pod0,
+			)
+			// is zone3Pod0 created
+			zone3Pod0 := fmt.Sprintf("%s-test-zone3-%d", statefulappName, 0)
+			_, err = testresource.TestClient.IsPodExists(statefulappNamespace, zone3Pod0)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create pod %s succeed", zone3Pod0,
+			)
+			// is zone3Pod0 running
+			gomega.Eventually(func() corev1.PodPhase {
+				return testresource.TestClient.GetPodStatus(statefulappNamespace, zone3Pod0)
+			}, PodCreateTimeout, TryInterval).Should(
+				gomega.Equal(corev1.PodRunning),
+				"pod %s is running", zone3Pod0,
+			)
+		})
+
+		ginkgo.It("step: check default statefulapp status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(statefulappNamespace, statefulappName)
+			}, StatefulappReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+	})
+
+	var _ = ginkgo.Describe("add pod by zone pipeline", func() {
+
+		addPodStatefulapp := testconverter.GetObjFromYaml("./data/statefulapp-02.yaml")
+		statefulappNamespace := addPodStatefulapp.GetNamespace()
+		statefulappName := addPodStatefulapp.GetName()
+
+		ginkgo.It("step: update statefulapp for add pod by zone", func() {
+			err := testresource.TestClient.UpdateStatefulappInstance(addPodStatefulapp)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"update statefulapp succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for statefulapp updated", func() {
+			time.Sleep(StatefulappCreateTimeout)
+		})
+
+		ginkgo.It("step: check pod status which is added", func() {
+			// is zone1Pod1 created
+			zone1Pod1 := fmt.Sprintf("%s-test-zone1-%d", statefulappName, 1)
+			_, err := testresource.TestClient.IsPodExists(statefulappNamespace, zone1Pod1)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create pod %s succeed", zone1Pod1,
+			)
+			// is zone1Pod1 running
+			gomega.Eventually(func() corev1.PodPhase {
+				return testresource.TestClient.GetPodStatus(statefulappNamespace, zone1Pod1)
+			}, PodCreateTimeout, TryInterval).Should(
+				gomega.Equal(corev1.PodRunning),
+				"pod %s is running", zone1Pod1,
+			)
+		})
+
+		ginkgo.It("step: check statefulapp status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(statefulappNamespace, statefulappName)
+			}, StatefulappReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+	})
+
+	var _ = ginkgo.Describe("delete pod by zone pipeline", func() {
+
+		delPodStatefulapp := testconverter.GetObjFromYaml("./data/statefulapp-01.yaml")
+		statefulappNamespace := delPodStatefulapp.GetNamespace()
+		statefulappName := delPodStatefulapp.GetName()
+
+		ginkgo.It("step: update statefulapp for delete pod by zone", func() {
+			err := testresource.TestClient.UpdateStatefulappInstance(delPodStatefulapp)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"update statefulapp succeed",
+			)
+		})
+
+		ginkgo.It("step: wait for statefulapp updated", func() {
+			time.Sleep(StatefulappCreateTimeout)
+		})
+
+		ginkgo.It("step: check pod status which is deleted", func() {
+			// is zone1Pod1 deleted
+			zone1Pod1 := fmt.Sprintf("%s-test-zone1-%d", statefulappName, 1)
+			gomega.Eventually(func() bool {
+				_, err := testresource.TestClient.IsPodExists(statefulappNamespace, zone1Pod1)
+				if kubeerrors.IsNotFound(err) {
+					return true
+				}
+				return false
+			}, PodDeleteTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"pod %s is deleted", zone1Pod1,
+			)
+		})
+
+		ginkgo.It("step: check statefulapp status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(statefulappNamespace, statefulappName)
+			}, StatefulappReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+	})
+
+	var _ = ginkgo.Describe("pod failover pipeline", func() {
+
+		failOverStatefulapp := testconverter.GetObjFromYaml("./data/statefulapp-01.yaml")
+		statefulappNamespace := failOverStatefulapp.GetNamespace()
+		statefulappName := failOverStatefulapp.GetName()
+
+		ginkgo.It("step: delete pod for failover", func() {
+			// is zone1Pod0 deleted
+			zone1Pod0 := fmt.Sprintf("%s-test-zone1-%d", statefulappName, 0)
+			err := testresource.TestClient.DeletePod(statefulappNamespace, zone1Pod0)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"delete pod %s succeed", zone1Pod0,
+			)
+		})
+
+		ginkgo.It("step: wait for pod failover", func() {
+			time.Sleep(PodDeleteTimeout)
+		})
+
+		ginkgo.It("step: check pod failover status", func() {
+			// is zone1Pod0 created
+			zone1Pod0 := fmt.Sprintf("%s-test-zone1-%d", statefulappName, 0)
+			_, err := testresource.TestClient.IsPodExists(statefulappNamespace, zone1Pod0)
+			gomega.Expect(err).To(
+				gomega.BeNil(),
+				"create pod %s succeed", zone1Pod0,
+			)
+			// is zone1Pod0 running
+			gomega.Eventually(func() corev1.PodPhase {
+				return testresource.TestClient.GetPodStatus(statefulappNamespace, zone1Pod0)
+			}, PodFailoverTimeout, TryInterval).Should(
+				gomega.Equal(corev1.PodRunning),
+				"pod %s is running", zone1Pod0,
+			)
+		})
+
+		ginkgo.It("step: check statefulapp status", func() {
+			gomega.Eventually(func() bool {
+				return testresource.TestClient.JudgeStatefulappInstanceIsReadyByObj(statefulappNamespace, statefulappName)
+			}, StatefulappReadyTimeout, TryInterval).Should(
+				gomega.Equal(true),
+				"statefulapp %s is ready", statefulappName,
+			)
+		})
+
+	})
+
+})