From 07592637a73233c9c894c6be76e83e5c95048561 Mon Sep 17 00:00:00 2001 From: frf12 Date: Tue, 16 Aug 2022 20:32:03 +0800 Subject: [PATCH] sync obd.sh --- deps/3rd/oceanbase.el7.aarch64.deps | 8 + deps/3rd/oceanbase.el7.x86_64.deps | 7 + deps/3rd/oceanbase.el8.aarch64.deps | 8 + deps/3rd/oceanbase.el8.x86_64.deps | 8 + tools/deploy/activate_obd.sh | 103 ++++ tools/deploy/copy.sh | 33 ++ tools/deploy/init.sql | 42 ++ tools/deploy/init_mini.sql | 40 ++ tools/deploy/init_user.sql | 5 + tools/deploy/obd.sh | 671 +++++++++++++++++++++++++ tools/deploy/obd/config.yaml.template | 13 + tools/deploy/obd/obproxy.include.yaml | 7 + tools/deploy/obd/obproxy.yaml.template | 10 + tools/deploy/obd/observer.include.yaml | 35 ++ 14 files changed, 990 insertions(+) create mode 100644 tools/deploy/activate_obd.sh create mode 100755 tools/deploy/copy.sh create mode 100644 tools/deploy/init.sql create mode 100644 tools/deploy/init_mini.sql create mode 100644 tools/deploy/init_user.sql create mode 100755 tools/deploy/obd.sh create mode 100644 tools/deploy/obd/config.yaml.template create mode 100644 tools/deploy/obd/obproxy.include.yaml create mode 100644 tools/deploy/obd/obproxy.yaml.template create mode 100644 tools/deploy/obd/observer.include.yaml diff --git a/deps/3rd/oceanbase.el7.aarch64.deps b/deps/3rd/oceanbase.el7.aarch64.deps index 1b10e4f69..da678c553 100644 --- a/deps/3rd/oceanbase.el7.aarch64.deps +++ b/deps/3rd/oceanbase.el7.aarch64.deps @@ -3,6 +3,11 @@ os=7 arch=aarch64 repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/7/aarch64/ +[target-community] +os=7 +arch=aarch64 +repo=http://mirrors.aliyun.com/oceanbase/community/stable/el/7/aarch64/ + [deps] devdeps-gtest-1.8.0-16.el7.aarch64.rpm devdeps-isa-l-static-2.22.0-17.el7.aarch64.rpm @@ -24,3 +29,6 @@ obdevtools-llvm-11.0.1-40.el7.aarch64.rpm [tools-deps] devdeps-rocksdb-6.22.1-26.el7.aarch64.rpm + +[test-utils] +# ob-deploy-1.4.0-13.el7.x86_64.rpm target=community diff --git a/deps/3rd/oceanbase.el7.x86_64.deps b/deps/3rd/oceanbase.el7.x86_64.deps index d7ff8b3f2..b7936327d 100644 --- a/deps/3rd/oceanbase.el7.x86_64.deps +++ b/deps/3rd/oceanbase.el7.x86_64.deps @@ -3,6 +3,11 @@ os=7 arch=x86_64 repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/7/x86_64/ +[target-community] +os=7 +arch=x86_64 +repo=http://mirrors.aliyun.com/oceanbase/community/stable/el/7/x86_64/ + [deps] devdeps-gtest-1.8.0-3.el7.x86_64.rpm devdeps-isa-l-static-2.22.0-3.el7.x86_64.rpm @@ -25,3 +30,5 @@ obdevtools-llvm-11.0.1-40.el7.x86_64.rpm [tools-deps] devdeps-rocksdb-6.22.1-26.el7.x86_64.rpm +[test-utils] + diff --git a/deps/3rd/oceanbase.el8.aarch64.deps b/deps/3rd/oceanbase.el8.aarch64.deps index 0d36edc9f..83ed3758d 100644 --- a/deps/3rd/oceanbase.el8.aarch64.deps +++ b/deps/3rd/oceanbase.el8.aarch64.deps @@ -3,6 +3,11 @@ os=8 arch=aarch64 repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/8/aarch64/ +[target-community] +os=8 +arch=aarch64 +repo=http://mirrors.aliyun.com/oceanbase/community/stable/el/8/aarch64/ + [deps] devdeps-gtest-1.8.0-16.el8.aarch64.rpm devdeps-isa-l-static-2.22.0-17.el8.aarch64.rpm @@ -24,3 +29,6 @@ obdevtools-llvm-11.0.1-40.el8.aarch64.rpm [tools-deps] devdeps-rocksdb-6.22.1-30.el8.aarch64.rpm + +[test-utils] +# ob-deploy-1.4.0-13.el7.x86_64.rpm target=community diff --git a/deps/3rd/oceanbase.el8.x86_64.deps b/deps/3rd/oceanbase.el8.x86_64.deps index 3bea4c98a..193112a49 100644 --- a/deps/3rd/oceanbase.el8.x86_64.deps +++ b/deps/3rd/oceanbase.el8.x86_64.deps @@ -3,6 +3,11 @@ os=8 arch=x86_64 repo=http://mirrors.aliyun.com/oceanbase/development-kit/el/8/x86_64/ +[target-community] +os=8 +arch=x86_64 +repo=http://mirrors.aliyun.com/oceanbase/community/stable/el/8/x86_64/ + [deps] devdeps-gtest-1.8.0-3.el8.x86_64.rpm devdeps-isa-l-static-2.22.0-3.el8.x86_64.rpm @@ -24,3 +29,6 @@ obdevtools-llvm-11.0.1-40.el8.x86_64.rpm [tools-deps] devdeps-rocksdb-6.22.1-30.el8.x86_64.rpm + +[test-utils] +# ob-deploy-1.4.0-13.el7.x86_64.rpm target=community diff --git a/tools/deploy/activate_obd.sh b/tools/deploy/activate_obd.sh new file mode 100644 index 000000000..97991f907 --- /dev/null +++ b/tools/deploy/activate_obd.sh @@ -0,0 +1,103 @@ +#!/bin/bash +DIR=$(readlink -f "$(dirname ${BASH_SOURCE[0]})/../..") +export DEPLOY_PATH=$DIR/tools/deploy +DEP_PATH=$DIR/deps/3rd +export OBD_BIN=${_OBD_BIN:-$DEP_PATH/usr/bin/obd} +alias obd="${OBD_BIN}" +export OBD_HOME=$DIR/tools/deploy +export OBD_INSTALL_PRE=$DEP_PATH + +if [ -f $OBD_INSTALL_PRE/etc/profile.d/obd.sh ] +then + source $OBD_INSTALL_PRE/etc/profile.d/obd.sh +fi +obd_dev=${OBD_HOME_GLOBAL:-~/.obd_dev} +mkdir -p $obd_dev/cluster +mkdir -p $OBD_HOME/.obd +[ ! -d $OBD_HOME/.obd/cluster ] && ln -fs $obd_dev/cluster $OBD_HOME/.obd + +if [ -n "$BASH_VERSION" ]; then + complete -F _obd_sh_complete_func ./obd.sh +fi + +function _obd_sh_reply_deploy_names() { + if [[ -d $OBD_HOME/.obd/cluster ]] + then + res=$(ls -p $OBD_HOME/.obd/cluster 2>/dev/null | sed "s#/##") + COMPREPLY=( $(compgen -o filenames -W "${res}" -- ${cur}) ) + fi +} + +function _obd_sh_reply_current_files() { + filename=${cur##*/} + dirname=${cur%*$filename} + res=$(ls -a -p $dirname 2>/dev/null | sed "s#^#$dirname#") + compopt -o nospace + COMPREPLY=( $(compgen -o filenames -W "${res}" -- ${cur}) ) +} + +function _obd_sh_reply_yaml_files() { + filename=${cur##*/} + dirname=${cur%*$filename} + res=$(ls -a -p $dirname 2>/dev/null | grep -E '(*.yaml|*.yml)' | sed "s#^#$dirname#") + compopt -o nospace + COMPREPLY=( $(compgen -o filenames -W "${res}" -- ${cur}) ) +} + +function _obd_sh_complete_func +{ + local all_cmds + declare -A all_cmds + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + all_cmds["$*"]="prepare deploy redeploy reinstall start stop restart destroy upgrade mysqltest pid ssh less gdb sql mysql oracle edit list sys display tenant_create tenant_drop sysbench tpch tpcc tpcds" + case $prev in + list) + return 0 + ;; + -p|--path) + _obd_sh_reply_current_files + ;; + -c|--config) + _obd_sh_reply_yaml_files + ;; + -n|--deploy-name) + _obd_sh_reply_deploy_names + ;; + *) + valid_len=$COMP_CWORD + words=( ${COMP_WORDS[@]::valid_len} ) + index=valid_len + while (( index >= 1 )); do + target="${words[*]}" + cmd=${all_cmds[$target]} + if [[ "$cmd" != "" ]] + then + if [[ $cmd =~ ^_obd_sh_reply.* ]] + then + $cmd + break + else + if [[ $cmd =~ .*'=' ]] + then + compopt -o nospace + fi + COMPREPLY=( $(compgen -W "${cmd}" -- ${cur}) ) + break + fi + fi + index=$(( index - 1)) + tmp=${words[*]::index} + [[ "$tmp" != "" ]] && parent_cmd=${all_cmds[$tmp]} + if [[ "$parent_cmd" =~ ^_obd_sh_reply.* || " $parent_cmd " =~ " ${words[index]} " ]]; then + words[index]='*' + else + break + fi + done + ;; + esac + + +} diff --git a/tools/deploy/copy.sh b/tools/deploy/copy.sh new file mode 100755 index 000000000..7c34420e9 --- /dev/null +++ b/tools/deploy/copy.sh @@ -0,0 +1,33 @@ +#!/bin/bash -x +if [ $# -lt 1 ] +then + echo "Usage ./copy.sh [oceanbase_dev_dir]" + BUILD_DIR=$(find $PWD/../../ -maxdepth 1 -name 'build_*' -type d | head -1) + SOURCE_DIR=$PWD/../../ +else + BUILD_DIR=$1 + SOURCE_DIR=$PWD/../../ +fi + +BIN_DIR=`pwd`/bin${VER} +LIB_DIR=`pwd`/lib +TOOL_DIR=`pwd`/tools +ETC_DIR=`pwd`/etc +DEBUG_DIR=`pwd`/debug +ADMIN_DIR=`pwd`/admin +if [ $# -lt 2 ] +then + mkdir -p $BIN_DIR + mkdir -p $LIB_DIR + mkdir -p $TOOL_DIR + mkdir -p $ETC_DIR + mkdir -p $DEBUG_DIR + mkdir -p $ADMIN_DIR + cp -f $SOURCE_DIR/deps/3rd/usr/local/oceanbase/deps/devel/lib/libaio.so $LIB_DIR + cp -f $SOURCE_DIR/deps/3rd/usr/local/oceanbase/deps/devel/lib/libaio.so.1 $LIB_DIR + cp -f $SOURCE_DIR/deps/3rd/usr/local/oceanbase/deps/devel/lib/mariadb/libmariadb.so $LIB_DIR + cp -f $SOURCE_DIR/deps/3rd/usr/local/oceanbase/deps/devel/lib/mariadb/libmariadb.so.3 $LIB_DIR + cp -f $BUILD_DIR/src/observer/observer $BIN_DIR/observer +fi + +exit 0 diff --git a/tools/deploy/init.sql b/tools/deploy/init.sql new file mode 100644 index 000000000..4188b8ca4 --- /dev/null +++ b/tools/deploy/init.sql @@ -0,0 +1,42 @@ +system sleep 5; +alter system set balancer_idle_time = '10s'; +alter system set enable_auto_refresh_location_cache = "False"; +alter system set merger_warm_up_duration_time = '0s'; +alter system set zone_merge_concurrency = 2; +alter system set merger_check_interval = '10s'; +alter system set enable_syslog_wf=False; +create user if not exists 'admin' IDENTIFIED BY 'admin'; +use oceanbase; +create database if not exists test; + +use test; +grant all on *.* to 'admin' WITH GRANT OPTION; + + + +set @@session.ob_query_timeout = 40000000; +create resource unit box1 max_cpu 2, max_memory 4073741824, max_iops 128, max_disk_size '5G', max_session_num 64, MIN_CPU=1, MIN_MEMORY=4073741824, MIN_IOPS=128; +create resource pool pool2 unit = 'box1', unit_num = 1; +create tenant mysql replica_num = 1, resource_pool_list=('pool2') set ob_tcp_invited_nodes='%', ob_compatibility_mode='mysql', parallel_max_servers=10, parallel_servers_target=10, secure_file_priv = ""; +set @@session.ob_query_timeout = 10000000; +system sleep 5; +alter tenant sys set variables recyclebin = 'on'; +alter tenant sys set variables ob_enable_truncate_flashback = 'on'; +alter tenant mysql set variables ob_tcp_invited_nodes='%'; +alter tenant mysql set variables recyclebin = 'on'; +alter tenant mysql set variables ob_enable_truncate_flashback = 'on'; + +select count(*) from oceanbase.__all_server group by zone limit 1 into @num; +set @sql_text = concat('alter resource pool pool1', ' unit_num = ', @num); +prepare stmt from @sql_text; +execute stmt; +deallocate prepare stmt; +set @sql_text = concat('alter resource pool pool2', ' unit_num = ', @num); +prepare stmt from @sql_text; +execute stmt; +deallocate prepare stmt; + +select primary_zone from oceanbase.__all_tenant where tenant_id = 1 into @zone_name; +alter tenant oracle primary_zone = @zone_name; +alter tenant mysql primary_zone = @zone_name; + diff --git a/tools/deploy/init_mini.sql b/tools/deploy/init_mini.sql new file mode 100644 index 000000000..ac7ff071f --- /dev/null +++ b/tools/deploy/init_mini.sql @@ -0,0 +1,40 @@ +system sleep 5; +alter system set balancer_idle_time = '10s'; +alter system set enable_auto_refresh_location_cache = "False"; +alter system set merger_warm_up_duration_time = '0s'; +alter system set zone_merge_concurrency = 2; +alter system set merger_check_interval = '10s'; +alter system set enable_syslog_wf=False; +alter system set freeze_trigger_percentage = '70'; +create user if not exists 'admin' IDENTIFIED BY 'admin'; +use oceanbase; +create database if not exists test; + +use test; +grant all on *.* to 'admin' WITH GRANT OPTION; + + + +#FIXME: schema拆分模式建租户耗时增加，这里先加大语句超时时间先绕过 +set @@session.ob_query_timeout = 40000000; +create resource unit box1 max_cpu 2, max_memory 805306368, max_iops 128, max_disk_size '5G', max_session_num 64, MIN_CPU=1, MIN_MEMORY=805306368, MIN_IOPS=128; +create resource pool pool2 unit = 'box1', unit_num = 1; +create tenant mysql replica_num = 1, resource_pool_list=('pool2') set ob_tcp_invited_nodes='%', ob_compatibility_mode='mysql', parallel_max_servers=10, parallel_servers_target=10, ob_sql_work_area_percentage=20, secure_file_priv = ""; +alter resource unit sys_unit_config min_memory=1073741824,max_memory=1073741824; +set @@session.ob_query_timeout = 10000000; +system sleep 5; +alter tenant sys set variables recyclebin = 'on'; +alter tenant sys set variables ob_enable_truncate_flashback = 'on'; +alter tenant mysql set variables ob_tcp_invited_nodes='%'; +alter tenant mysql set variables recyclebin = 'on'; +alter tenant mysql set variables ob_enable_truncate_flashback = 'on'; + +select count(*) from oceanbase.__all_server group by zone limit 1 into @num; +set @sql_text = concat('alter resource pool pool2', ' unit_num = ', @num); +prepare stmt from @sql_text; +execute stmt; +deallocate prepare stmt; + +select primary_zone from oceanbase.__all_tenant where tenant_id = 1 into @zone_name; +alter tenant mysql primary_zone = @zone_name; + diff --git a/tools/deploy/init_user.sql b/tools/deploy/init_user.sql new file mode 100644 index 000000000..8039fdc32 --- /dev/null +++ b/tools/deploy/init_user.sql @@ -0,0 +1,5 @@ +use oceanbase; +create user if not exists 'admin' IDENTIFIED BY 'admin'; +grant all on *.* to 'admin' WITH GRANT OPTION; +create database obproxy; + diff --git a/tools/deploy/obd.sh b/tools/deploy/obd.sh new file mode 100755 index 000000000..548b89811 --- /dev/null +++ b/tools/deploy/obd.sh @@ -0,0 +1,671 @@ +#!/bin/bash + +BASE_DIR=$(git rev-parse --show-toplevel 2>/dev/null) +DEPLOY_PATH="$BASE_DIR/tools/deploy" +OBSERVER_BIN="$BASE_DIR/tools/deploy/bin/observer" +OBD_CLUSTER_PATH="$DEPLOY_PATH"/.obd/cluster +shopt -s expand_aliases +source $DEPLOY_PATH/activate_obd.sh +tag="latest" + + +function obd_exec { + echo -e "\033[32m>>> obd $* \033[0m" + $OBD_BIN $* $VERBOSE_FLAG +} +alias obd="obd_exec" + +function variables_parpare { + if [[ "$(readlink -f "$BASE_DIR"/..)" == "$OB_FLOW_WORK_DIR" ]] + then + path=$(readlink -f "$BASE_DIR") + export task="${path##*/}" + else + export task="default" + fi + port_gen=$((100*($(id -u)%500)+10000)) + HOST=$(hostname -i) + DATA_PATH="/data/$(whoami)" + IPADDRESS="127.0.0.1" + COMPONENT="oceanbase-ce" + DEP_PATH=$BASE_DIR/deps/3rd + OBCLIENT_BIN=$DEP_PATH/u01/obclient/bin/obclient + MYSQLTEST_BIN=$DEP_PATH/u01/obclient/bin/mysqltest + + export OBD_HOME=$DEPLOY_PATH + export OBD_INSTALL_PRE=$DEP_PATH + DEFAULT_DEPLOY_NAME_FILE=$OBD_HOME/.obd/.default_deploy +} + +function copy_sh { + if [[ -f copy.sh ]] + then + sh copy.sh + else + echo 'can not find copy.sh' + fi +} + +function mirror_create { + # obproxy mirror create + if [[ "$WITH_LOCAL_PROXY" != "" ]] && [[ -x $DEPLOY_PATH/bin/obproxy ]] + then + odp_version=$("$DEPLOY_PATH"/bin/obproxy --version 2>&1 | grep -E 'obproxy \(OceanBase [\.0-9]+ \S+\)' | grep -Eo '\s[.0-9]+\s') + obd mirror create -n obproxy -p "$DEPLOY_PATH" -t $tag -V $odp_version -f + fi + + # observer mirror create + obs_version=$($OBSERVER_BIN -V 2>&1 | grep -E "observer \(OceanBase( CE)? ([.0-9]+)\)" | grep -Eo '([.0-9]+)') + if [[ "$obs_version" == "" ]] + then + echo "$OBSERVER_BIN not found" + return 1 + fi + [[ -f "$BASE_DIR/tools/deploy/obd/.observer_obd_plugin_version" ]] && obs_version=$(cat $BASE_DIR/tools/deploy/obd/.observer_obd_plugin_version) + obs_mirror_info=$(obd_exec mirror create -n $COMPONENT -p "$DEPLOY_PATH" -V "$obs_version" -t $tag -f) && success=1 + if [[ "$success" != "1" ]] + then + echo "$obs_mirror_info" + echo "create observer mirror failed" + return 2 + else + echo "$obs_mirror_info" + fi + obs_package_hash=$(echo "$obs_mirror_info"| grep -E '^md5:' | awk '{print $2}') +} + + + +## generate config +function generate_config { + app_name=$task.$USER.$HOST + ocp_config_server='http://ocp-cfg.alibaba.net:8080/services?User_ID=alibaba&UID=test' + proxy_cfg_url=${ocp_config_server}\&Action=GetObProxyConfig\&ObRegionGroup=$app_name + + port_num=$port_gen + mysql_port=$port_num && port_num=$((port_num+1)) + rpc_port=$port_num && port_num=$((port_num+1)) + mysql_port2=$port_num && port_num=$((port_num+1)) + rpc_port2=$port_num && port_num=$((port_num+1)) + listen_port=$port_num && port_num=$((port_num+1)) + prometheus_listen_port=$port_num && port_num=$((port_num+1)) + + base_template=$(cat obd/config.yaml.template) + proxy_conf=$(cat obd/obproxy.yaml.template) + if [[ "$WITH_LOCAL_PROXY" != "" ]] + then + proxy_conf=$(echo "$proxy_conf" | sed 's/package_hash: [0-9a-z]*/tag: {{%% TAG %%}}/g') + fi + proxy_conf=${proxy_conf//'{{%% TAG %%}}'/$tag} + proxy_conf=${proxy_conf//'{{%% COMPONENT %%}}'/$COMPONENT} + proxy_conf=${proxy_conf//'{{%% LISTEN_PORT %%}}'/$listen_port} + proxy_conf=${proxy_conf//'{{%% PROMETHEUS_LISTEN_PORT %%}}'/$prometheus_listen_port} + proxy_conf=${proxy_conf//'{{%% OBPORXY_HOME_PATH %%}}'/"$DATA_PATH"/obproxy} + proxy_conf=${proxy_conf//'{{%% OBPROXY_CONFIG_SERVER_URL %%}}'/$proxy_cfg_url} + + + base_template=${base_template//"{{%% COMPONENT %%}}"/$COMPONENT} + base_template=${base_template//"{{%% TAG %%}}"/$tag} + base_template=${base_template//"{{%% DEPLOY_PATH %%}}"/$DEPLOY_PATH} + base_template=${base_template//"{{%% TOOLS_PATH %%}}"/$BASE_DIR/tools} + base_template=${base_template//"{{%% MINI_SIZE %%}}"/$mem} + base_template=${base_template//"{{%% OBCONFIG_URL %%}}"/$ocp_config_server} + base_template=${base_template//"{{%% APPNAME %%}}"/$app_name} + base_template=${base_template//"{{%% EXTRA_PARAM %%}}"/} + + # simple.yaml + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: $IPADDRESS + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port + home_path: $DATA_PATH/observer1 + zone: zone1 + # The directory for data storage. The default value is home_path/store. + # data_dir: /data + # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. + # redo_dir: /redo +EOF +) + single_conf=${base_template} + single_conf=${single_conf//"{{%% SERVERS %%}}"/$SERVERS} + single_without_proxy_conf=${single_conf//"{{%% PROXY_CONF %%}}"/} + single_with_proxy_conf=${single_conf//"{{%% PROXY_CONF %%}}"/$proxy_conf} + + [ ! -f ./single.yaml ] && echo "$single_without_proxy_conf" > ./single.yaml && echo "generate yaml config file: $(readlink -f ./single.yaml)" + [ ! -f ./single-with-proxy.yaml ] && echo "$single_with_proxy_conf" > ./single-with-proxy.yaml && echo "generate yaml config file: $(readlink -f ./single-with-proxy.yaml)" + + # 2x1 + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: $IPADDRESS + - name: server2 + ip: $IPADDRESS + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port + home_path: $DATA_PATH/observer1 + # The directory for data storage. The default value is home_path/store. + # data_dir: /data + # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. + # redo_dir: /redo + zone: zone1 + server2: + mysql_port: $mysql_port2 + rpc_port: $rpc_port2 + home_path: $DATA_PATH/observer2 + # The directory for data storage. The default value is home_path/store. + # data_dir: /data + # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. + # redo_dir: /redo + zone: zone2 +EOF +) + + distributed_conf=${base_template} + distributed_conf=${distributed_conf//"{{%% SERVERS %%}}"/$SERVERS} + distributed_without_proxy_conf=${distributed_conf//"{{%% PROXY_CONF %%}}"/} + distributed_with_proxy_conf=${distributed_conf//"{{%% PROXY_CONF %%}}"/$proxy_conf} + [ ! -f ./distributed.yaml ] && echo "$distributed_without_proxy_conf" > ./distributed.yaml && echo "generate yaml config file: $(readlink -f ./distributed.yaml)" + [ ! -f ./distributed-with-proxy.yaml ] && echo "$distributed_with_proxy_conf" > ./distributed-with-proxy.yaml && echo "generate yaml config file: $(readlink -f ./distributed-with-proxy.yaml)" +} + +function show_deploy_name { + echo -e "\e[1m\033[32mDeploy name: $deploy_name \033[0m" +} + +function get_deploy_name { + [ "$HELP" == "1" ] && return + if [[ "$deploy_name" != "" ]] + then + return + fi + if [[ "$DEPLOY_NAME" == "" ]] + then + [[ "$YAML_CONF" != "" ]] && yaml_name=${YAML_CONF##*/} && deploy_name=${yaml_name%.yaml} && show_deploy_name && return + mkdir -p $OBD_CLUSTER_PATH + cluster_dirs=$(ls -1 $OBD_CLUSTER_PATH) + cluster_num=$(echo "$cluster_dirs" | wc -l ) + if (( ${cluster_num} == 1 )) && [[ -f $DEFAULT_DEPLOY_NAME_FILE ]] + then + deploy_name=$(cat "$DEFAULT_DEPLOY_NAME_FILE") && + show_deploy_name && + return + fi + echo """ +Deploy name is required. Use -n to set the deploy name. +Available deployments: +$cluster_dirs +""" && exit 1 + else + deploy_name="$DEPLOY_NAME" && show_deploy_name && return + fi +} + +function get_baseurl { + repo_file=$OBD_HOME/.obd/mirror/remote/taobao.repo + start=0 + end=$(cat $repo_file | wc -l ) + for line in $(grep -nE "^\[.*\]$" $repo_file) + do + num=$(echo "$line" | awk -F ":" '{print $1}') + [ "$find_section" == "1" ] && end=$num + is_target_section=$(echo $(echo "$line" | grep -E "\[\s*$repository\s*\]")) + [ "$is_target_section" != "" ] && start=$num && find_section='1' + done + repo_section=$(awk "NR>=$start && NR<=$end" $repo_file) + baseurl=$(echo "$repo_section" | grep baseurl | awk -F '=' '{print $2}') +} + +function get_version_and_release { + start=0 + end=$(cat $config_yaml | wc -l) + for line in $(grep -nE '^\S+:' $config_yaml) + do + num=$(echo "$line" | awk -F ":" '{print $1}') + [ "$find_obproxy" == "1" ] && end=$num + [ "$(echo "$line" | grep obproxy)" != "" ] && start=$num && find_obproxy='1' + done + odp_conf=$(awk "NR>=$start && NR<=$end" $config_yaml) + version=$(echo "$odp_conf" | grep 'version:' | awk '{print $2}') + release=$(echo "$odp_conf" | grep 'release:' | awk '{print $2}') + include_file=$(echo "$odp_conf"| grep 'include:'| awk '{print $2}') + if [[ -f $include_file ]] + then + _repo=$(echo $(grep '__mirror_repository_section_name:' $include_file | awk '{print $2}')) + [ "$_repo" != "" ] && repository=$_repo + version=$(echo $(grep 'version:' $include_file | awk '{print $2}')) + release=$(echo $(grep 'release:' $include_file | awk '{print $2}')) + fi +} + +function get_obproxy { + repository="taobao" + obproxy_mirror_repository=$(echo $(grep '__mirror_repository_section_name' $config_yaml | awk -F':' '{print $2}')) + [ "$obproxy_mirror_repository" != "" ] && repository=$obproxy_mirror_repository + + get_version_and_release + get_baseurl + obd mirror disable remote + if [[ "$baseurl" != "" && "$version" != "" && "$release" != "" ]] + then + pkg_name="obproxy-$version-$release.x86_64.rpm" + if [ "$(find $OBD_HOME/.obd/mirror/local -name $pkg_name)" == "" ] + then + download_dir=$OBD_HOME/.obd_download + mkdir -p $download_dir + wget $baseurl/obproxy/$pkg_name -O "$download_dir/$pkg_name" -o $download_dir/obproxy.down + obd mirror clone "$download_dir/$pkg_name" && rm -rf $download_dir && return 0 + else + return 0 + fi + fi + obd mirror enable $repository +} + +function deploy_cluster { + [[ "$YAML_CONF" == "" ]] || yaml_config_args="-c $YAML_CONF" + get_deploy_name + if [[ "$YAML_CONF" != "" ]] + then + config_yaml=$YAML_CONF + else + config_yaml=$OBD_CLUSTER_PATH/$deploy_name/config.yaml + fi + [[ "$(grep -E "^obproxy:" $config_yaml)" != "" ]] && ( get_obproxy || exit 1 ) + + echo "$deploy_name" > "$DEFAULT_DEPLOY_NAME_FILE" + if [[ "$HIDE_DESTROY" == "1" ]] + then + obd cluster destroy "$deploy_name" -f > /dev/null 2>&1 + else + obd cluster destroy "$deploy_name" -f + fi + obd cluster deploy "$deploy_name" $yaml_config_args -C || exit 1 + if ! obd cluster start "$deploy_name" -f; + then + while [[ "$NO_CONFIRM" != "1" && "$(grep 'config_status: NEED_REDEPLOY' $OBD_CLUSTER_PATH/$deploy_name/.data)" != "" ]] + do + read -r -p "Start $deploy_name failed, do you want to edit config and continue?[Y/n]" input + case $input in + [Yy]);& + "") + obd cluster edit-config "$deploy_name" + if obd cluster start "$deploy_name" -f; + then + break + else + continue + fi + ;; + [Nn]) + exit 1 + esac + done + fi + init_sql +} + +function init_sql { + obd test mysqltest "$deploy_name" "$INIT_FLIES" --init-only $CLIENT_BIN_ARGS +} + +function start_cluster { + get_deploy_name + obd cluster start "$deploy_name" -f +} + +function stop_cluster { + get_deploy_name + obd cluster stop "$deploy_name" +} + +function restart_cluster { + get_deploy_name + obd cluster restart "$deploy_name" +} + +function upgreade_cluster { + get_deploy_name + obd cluster upgrade "$deploy_name" +} + +function destroy_cluster { + get_deploy_name + obd cluster destroy "$deploy_name" -f +} + +function reinstall_cluster { + get_deploy_name + obd cluster reinstall "$deploy_name" -c $COMPONENT --hash=$obs_package_hash +} + +function tool_cmd { + cmd=$1 + get_deploy_name + if [[ $HELP == "1" ]] + then + obd tool command --help + return + fi + obd tool command "$deploy_name" $cmd $extra_args +} + +function connect { + get_deploy_name + if [[ $HELP == "1" ]] + then + obd tool db_connect --help + return + fi + obd tool db_connect "$deploy_name" $OBCLIENT_BIN_ARGS $extra_args +} + +function mysqltest { + if [[ $HELP == "1" ]] + then + obd test mysqltest --help + return + fi + get_deploy_name + + if [[ "$NEED_REBOOT" == "1" || "$YAML_CONF" != "" ]] + then + mirror_create || return 1 + deploy_cluster || return 2 + fi + obd test mysqltest "$deploy_name" $CLIENT_BIN_ARGS $extra_args $INIT_FLIES +} + +function deploy { + if [[ "$YAML_CONF" != *.yaml && "$DEPLOY_NAME" == "" ]] + then + echo "Use -n to set deploy name, or -c to set the deploy config" + exit 1 + fi + get_deploy_name + if [[ "$YAML_CONF" != "" && -d "$OBD_CLUSTER_PATH"/"$deploy_name" ]] + then + while true + do + read -r -p "Deploy name: $deploy_name already exists. Do you want to OVERWRITE it?[y/n]" input + case $input in + [Yy]) + break + ;; + [Nn]) + exit 1 + esac + done + fi + HIDE_DESTROY=1 + [[ "$EXEC_CP" == "1" ]] && copy_sh + mirror_create || exit 1 + deploy_cluster +} + +function edit { + get_deploy_name + obd cluster edit-config $deploy_name + if [[ "$(grep 'config_status: NEED_REDEPLOY' $OBD_CLUSTER_PATH/$deploy_name/.data)" != "" ]] + then + echo -e "\e[33mif you need redeploy, please use '$entrance redeploy -n $deploy_name'\e[0m" + fi +} + +function display { + get_deploy_name + obd cluster display $deploy_name +} + +function tenant_create { + get_deploy_name + obd cluster tenant create $deploy_name $extra_args +} + +function tenant_drop { + get_deploy_name + obd cluster tenant drop $deploy_name $extra_args +} + +function sysbench { + get_deploy_name + obd test sysbench $deploy_name $OBCLIENT_BIN_ARGS $extra_args +} + +function tpch { + get_deploy_name + obd test tpch $deploy_name $OBCLIENT_BIN_ARGS $extra_args +} + +function tpcc { + get_deploy_name + obd test tpcc $deploy_name $OBCLIENT_BIN_ARGS $extra_args +} + +function tpcds { + get_deploy_name + obd test tpcds $deploy_name $OBCLIENT_BIN_ARGS $extra_args +} + + +function help_info { + echo """ +Usage: $entrance [options] + +Available commonds: + +prepare [-p DATA_PATH -h HOST] Prepare for deployment. + +deploy -c YAML_CONF [-n DEPLOY_NAME] Deploy a cluster by a deploy yaml file. Default deploy name will be the name of yaml file. + +redeploy [-c YAML_CONF -n DEPLOY_NAME] Redeploy cluster. + +reinstall [-n DEPLOY_NAME] Reinstall cluster. (Change bin file, sync libs and restart) + +start [-n DEPLOY_NAME] Start cluster. + +stop [-n DEPLOY_NAME] Stop a started cluster. + +restart [-n DEPLOY_NAME] Restart cluster. + +destroy [-n DEPLOY_NAME] Destroy cluster. + +upgrade [-n DEPLOY_NAME] Upgrade cluster. + +list [-n DEPLOY_NAME] List cluster. + +display [-n DEPLOY_NAME] Display cluster info. + +tenant_create [-n DEPLOY_NAME] Create tenant. + +tenant_drop [-n DEPLOY_NAME] Drop tenant. + +sysbench [-n DEPLOY_NAME] Run sysbench, use '--help' for more details. + +tpch [-n DEPLOY_NAME] Run tpch test, use '--help' for more details. + +tpcc [-n DEPLOY_NAME] Run tpcc test, use '--help' for more details. + +tpcds [-n DEPLOY_NAME] Run tpcds test, use '--help' for more details. + +mysqltest [-n DEPLOY_NAME] Run mysqltest, use '--help' for more details. + +pid [-n DEPLOY_NAME] Get pid list for servers, use '--help' for more details. + +ssh [-n DEPLOY_NAME] Ssh to target server and change directory to log path, use '--help' for more details. + +less [-n DEPLOY_NAME] Use command less to the observer.log, use '--help' for more details. + +gdb [-n DEPLOY_NAME] Use gdb to attch target server, use '--help' for more details. + +sql [-n DEPLOY_NAME] Connect to target server by root@sys, use '--help' for more details. + +sys [-n DEPLOY_NAME] Connect to target server by root@sys, use '--help' for more details. + +mysql [-n DEPLOY_NAME] Connect to target server by root@mysql, use '--help' for more details. + +oracle [-n DEPLOY_NAME] Connect to target server by SYS@oracle, use '--help' for more details. + +Options: +-V, --version Show version of obd. + +-c YAML_CONF, --config=YAML_CONF The deploy yaml file. + +-n DEPLOY_NAME, --deploy-name=DEPLOY_NAME + The name of the deployment. + +-v VERBOSE Activate verbose output. + +-p DATA_PATH, --data-path=DATA_PATH The data path for server deployment, it can be changed in the yaml file. + +--ip IPADDRESS The ipaddress for server deployment, it can be changed in the yaml file. + +--port PORT_BEGIN The port starting point. All the ports can be changed in the yaml file. + +--with-local-obproxy Use local obproxy. + +--skip-copy Skip copy.sh. + +--cp Exec copy.sh. + +--reboot Redeploy cluster before mysqltest + +https://yuque.antfin-inc.com/docs/share/7f5dc9e8-dbe8-4f59-852d-b7ce57b88fdd +""" +} + +function main() { + entrance=${OBD_SH_ENTRANCE:-obd.sh} + variables_parpare + commond="$1" + shift + extra_args="" + while true; do + case "$1" in + -v ) VERBOSE_FLAG='-v'; set -x; shift ;; + --with-local-obproxy) WITH_LOCAL_PROXY="1";SKIP_COPY="1"; shift ;; + -c | --config ) YAML_CONF="$2"; shift 2 ;; + -n | --deploy-name ) DEPLOY_NAME="$2"; shift 2 ;; + -p | --data-path ) DATA_PATH="$2"; shift 2 ;; + -N ) NO_CONFIRM="1"; shift ;; + --ip ) IPADDRESS="$2"; shift 2 ;; + # --disable-reboot ) DISABLE_REBOOT="1"; extra_args="$extra_args $1"; shift ;; + --reboot ) NEED_REBOOT="1"; shift ;; + --cp ) EXEC_CP="1"; shift ;; + --skip-copy ) SKIP_COPY="1"; shift ;; + -- ) shift ;; + "" ) break ;; + * ) extra_args="$extra_args $1"; [[ "$1" == "--help" || "$1" == "-h" ]] && HELP="1" ; shift ;; + esac + done + if [[ ! -f $DEPLOY_PATH/.obd/.obd_environ || "$(grep '"OBD_DEV_MODE": "1"' $DEPLOY_PATH/.obd/.obd_environ)" == "" ]] + then + obd devmode enable || (echo "Exec obd cmd failed. If your branch is based on 3.1_opensource_release, please go to the deps/3rd directory and execute 'bash dep_create.sh all' to install obd." && exit 1) + obd mirror disable remote + fi + if [[ "$(grep '"OBD_DEPLOY_BASE_DIR":' $DEPLOY_PATH/.obd/.obd_environ)" == "" ]] + then + obd env set OBD_DEPLOY_BASE_DIR "$DEPLOY_PATH" + fi + case $commond in + -V | --version) + obd --version + ;; + prepare) + [[ "$SKIP_COPY" == "" ]] && copy_sh + generate_config + ;; + deploy) + deploy + ;; + redeploy) + deploy_cluster + ;; + start) + start_cluster + ;; + stop) + stop_cluster + ;; + restart) + restart_cluster + ;; + destroy) + destroy_cluster + ;; + upgrade) + [[ "$EXEC_CP" == "1" ]] && copy_sh + mirror_create || exit 1 + upgreade_cluster + ;; + mysqltest) + [[ "$EXEC_CP" == "1" ]] && copy_sh + mysqltest + ;; + sql);& + sys) + connect + ;; + mysql) + extra_args="--tenant mysql $extra_args" + connect + ;; + oracle) + extra_args="--user SYS --tenant oracle $extra_args" + connect + ;; + pid) + tool_cmd pid + ;; + ssh) + tool_cmd ssh + ;; + less) + tool_cmd less + ;; + gdb) + tool_cmd gdb + ;; + reinstall) + get_deploy_name + [[ "$EXEC_CP" == "1" ]] && copy_sh + mirror_create && reinstall_cluster + ;; + list) + obd cluster list + ;; + edit) + edit + ;; + display) + display + ;; + tenant_create) + tenant_create + ;; + tenant_drop) + tenant_drop + ;; + sysbench) + sysbench + ;; + tpch) + tpch + ;; + tpcc) + tpcc + ;; + tpcds) + tpcds + ;; + *) + help_info + ;; + esac +} + +main "$@" diff --git a/tools/deploy/obd/config.yaml.template b/tools/deploy/obd/config.yaml.template new file mode 100644 index 000000000..751599634 --- /dev/null +++ b/tools/deploy/obd/config.yaml.template @@ -0,0 +1,13 @@ +oceanbase-ce: +{{%% SERVERS %%}} + tag: latest + include: obd/observer.include.yaml + global: + # for default system config used by farm, please see tools/deploy/obd/observer.include.yaml + # You can also specify the configuration directly below (stored locally, switching the working directory and redeploying will still take effect) + devname: lo + # memory_limit: '10G' + # system_memory: '5G' + # datafile_size: '20G' + # cpu_count: '24' +{{%% PROXY_CONF %%}} diff --git a/tools/deploy/obd/obproxy.include.yaml b/tools/deploy/obd/obproxy.include.yaml new file mode 100644 index 000000000..412eb5edd --- /dev/null +++ b/tools/deploy/obd/obproxy.include.yaml @@ -0,0 +1,7 @@ +config: + enable_cluster_checkout: false + skip_proxy_sys_private_check: true + syslog_level: 'DEBUG' + enable_metadb_used: 'false' + enable_qa_mode: 'TRUE' + location_expire_period: '1' diff --git a/tools/deploy/obd/obproxy.yaml.template b/tools/deploy/obd/obproxy.yaml.template new file mode 100644 index 000000000..91cbb773f --- /dev/null +++ b/tools/deploy/obd/obproxy.yaml.template @@ -0,0 +1,10 @@ +obproxy-ce: + servers: + - 127.0.0.1 + global: + listen_port: {{%% LISTEN_PORT %%}} + prometheus_listen_port: {{%% PROMETHEUS_LISTEN_PORT %%}} + home_path: {{%% OBPORXY_HOME_PATH %%}} + include: obd/obproxy.include.yaml + depends: + - oceanbase-ce diff --git a/tools/deploy/obd/observer.include.yaml b/tools/deploy/obd/observer.include.yaml new file mode 100644 index 000000000..3da86b2b9 --- /dev/null +++ b/tools/deploy/obd/observer.include.yaml @@ -0,0 +1,35 @@ +runtime_dependencies: + - src_path: plugin_dir + target_path: plugin_dir + - src_path: admin + target_path: admin + - src_path: lib + target_path: lib + - src_path: tools + target_path: tools + - src_path: etc + target_path: etc + - src_path: wallet + target_path: wallet +env: + ASAN_OPTIONS: 'abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:log_path=./log/asan.log' + ASAN_SYMBOLIZER_PATH: ./tools/llvm-symbolizer +config: + cluster_id: 1 + major_freeze_duty_time: 'Disable' + # datafile_disk_percentage: '50' + datafile_size: '20G' + memory_limit: '10G' + system_memory: '5G' + cpu_count: '24' + stack_size: '512K' + cache_wash_threshold: '1G' + __min_full_resource_pool_memory: "268435456" + workers_per_cpu_quota: '10' + schema_history_expire_time: '1d' + net_thread_count: '4' + minor_freeze_times: '10' + enable_separate_sys_clog: 'FALSE' + enable_merge_by_turn: 'FALSE' + syslog_io_bandwidth_limit: '10G' + enable_async_syslog: 'FALSE' -- GitLab