From 5bdf28791052727df06761a8540d90e81fe5e0fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Dec 2020 00:55:54 +0300 Subject: [PATCH] Check for conflict markers in docs --- docker/server/alpine-build.sh | 20 +++++----- docker/server/entrypoint.alpine.sh | 26 ++++++------- docs/en/operations/settings/settings.md | 39 +++++++++---------- docs/ru/interfaces/formats.md | 5 +-- .../0_stateless/01238_http_memory_tracking.sh | 2 +- .../0_stateless/01383_log_broken_table.sh | 8 ++-- .../01505_pipeline_executor_UAF.sh | 2 +- ...01514_distributed_cancel_query_on_error.sh | 2 +- utils/check-style/check-style | 6 ++- 9 files changed, 56 insertions(+), 54 deletions(-) diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh index 30101225b3..c9ba03f7f3 100755 --- a/docker/server/alpine-build.sh +++ b/docker/server/alpine-build.sh @@ -47,13 +47,13 @@ cp "${DOCKER_BUILD_FOLDER}/entrypoint.alpine.sh" "${CONTAINER_ROOT_FOLDER}/ ## get glibc components from ubuntu 20.04 and put them to expected place docker pull ubuntu:20.04 ubuntu20image=$(docker create --rm ubuntu:20.04) -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" -docker cp -L ${ubuntu20image}:/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" - -docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull \ No newline at end of file +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" + +docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull diff --git a/docker/server/entrypoint.alpine.sh b/docker/server/entrypoint.alpine.sh index e2edda9ca2..f0cc62d276 100755 --- a/docker/server/entrypoint.alpine.sh +++ b/docker/server/entrypoint.alpine.sh @@ -26,17 +26,17 @@ fi CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" # port is needed to check if clickhouse-server is ready for connections -HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" +HTTP_PORT="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=http_port)" # get CH directories locations -DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" -TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" -USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" -LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" -LOG_DIR="$(dirname $LOG_PATH || true)" -ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" -ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" -FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" +DATA_DIR="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=logger.log || true)" +LOG_DIR="$(dirname "${LOG_PATH}" || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname "${ERROR_LOG_PATH}" || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=format_schema_path || true)" CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" @@ -92,7 +92,7 @@ fi if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then # Listen only on localhost until the initialization is done - $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & + $gosu /usr/bin/clickhouse-server --config-file="${CLICKHOUSE_CONFIG}" -- --listen_host=127.0.0.1 & pid="$!" # check if clickhouse is ready to accept connections @@ -107,7 +107,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then sleep 1 done - if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then + if [ -n "$CLICKHOUSE_PASSWORD" ]; then printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" fi @@ -130,7 +130,7 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then . "$f" fi ;; - *.sql) echo "$0: running $f"; cat "$f" | "$clickhouseclient" ; echo ;; + *.sql) echo "$0: running $f"; "$clickhouseclient" < "$f" ; echo ;; *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "$clickhouseclient"; echo ;; *) echo "$0: ignoring $f" ;; esac @@ -145,7 +145,7 @@ fi # if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" + exec $gosu /usr/bin/clickhouse-server --config-file="${CLICKHOUSE_CONFIG}" "$@" fi # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 093e4e50df..86e16fe481 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -324,7 +324,7 @@ Consider the table: CREATE TABLE table_with_enum_column_for_tsv_insert (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory(); ``` -When the `input_format_tsv_enum_as_number` setting is enabled: +When the `input_format_tsv_enum_as_number` setting is enabled: ```sql SET input_format_tsv_enum_as_number = 1; @@ -1248,7 +1248,7 @@ Consider the table: CREATE TABLE table_with_enum_column_for_csv_insert (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory(); ``` -When the `input_format_csv_enum_as_number` setting is enabled: +When the `input_format_csv_enum_as_number` setting is enabled: ```sql SET input_format_csv_enum_as_number = 1; @@ -1841,7 +1841,7 @@ Default value: 0. Enables or disables synchronous data insertion into a [Distributed](../../engines/table-engines/special/distributed.md#distributed) table. -By default, when inserting data into a `Distributed` table, the ClickHouse server sends data to cluster nodes in asynchronous mode. When `insert_distributed_sync=1`, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true). +By default, when inserting data into a `Distributed` table, the ClickHouse server sends data to cluster nodes in asynchronous mode. When `insert_distributed_sync=1`, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true). Possible values: @@ -2109,8 +2109,8 @@ Enables [ORDER BY](../../sql-reference/statements/select/order-by.md#optimize_re Possible values: -- 0 — `ORDER BY` optimization is disabled. -- 1 — `ORDER BY` optimization is enabled. +- 0 — `ORDER BY` optimization is disabled. +- 1 — `ORDER BY` optimization is enabled. Default value: `1`. @@ -2124,8 +2124,8 @@ Allows to execute `ALTER TABLE ... UPDATE|DELETE` queries ([mutations](../../sql Possible values: -- 0 - Mutations execute asynchronously. -- 1 - The query waits for all mutations to complete on the current server. +- 0 - Mutations execute asynchronously. +- 1 - The query waits for all mutations to complete on the current server. - 2 - The query waits for all mutations to complete on all replicas (if they exist). Default value: `0`. @@ -2137,11 +2137,11 @@ Default value: `0`. ## ttl_only_drop_parts {#ttl_only_drop_parts} -Enables or disables complete dropping of data parts where all rows are expired in [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. +Enables or disables complete dropping of data parts where all rows are expired in [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. -When `ttl_only_drop_parts` is disabled (by default), the ClickHouse server only deletes expired rows according to their TTL. +When `ttl_only_drop_parts` is disabled (by default), the ClickHouse server only deletes expired rows according to their TTL. -When `ttl_only_drop_parts` is enabled, the ClickHouse server drops a whole part when all rows in it are expired. +When `ttl_only_drop_parts` is enabled, the ClickHouse server drops a whole part when all rows in it are expired. Dropping whole parts instead of partial cleaning TTL-d rows allows having shorter `merge_with_ttl_timeout` times and lower impact on system performance. @@ -2152,14 +2152,14 @@ Possible values: Default value: `0`. -**See Also** +**See Also** - [CREATE TABLE query clauses and settings](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) (`merge_with_ttl_timeout` setting) - [Table TTL](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) ## lock_acquire_timeout {#lock_acquire_timeout} -Defines how many seconds a locking request waits before failing. +Defines how many seconds a locking request waits before failing. Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When the timeout expires and the locking request fails, the ClickHouse server throws an exception "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. @@ -2179,11 +2179,11 @@ When the setting is enabled and the argument of `CAST` function is `Nullable`, t Possible values: - 0 — The `CAST` result has exactly the destination type specified. -- 1 — If the argument type is `Nullable`, the `CAST` result is transformed to `Nullable(DestinationDataType)`. +- 1 — If the argument type is `Nullable`, the `CAST` result is transformed to `Nullable(DestinationDataType)`. Default value: `0`. -**Examples** +**Examples** The following query results in the destination data type exactly: @@ -2215,17 +2215,17 @@ Result: └───┴───────────────────────────────────────────────────┘ ``` -**See Also** +**See Also** - [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) function ## output_format_pretty_max_value_width {#output_format_pretty_max_value_width} -Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pretty) formats. If the value width exceeds the limit, the value is cut. +Limits the width of value displayed in [Pretty](../../interfaces/formats.md#pretty) formats. If the value width exceeds the limit, the value is cut. Possible values: -- Positive integer. +- Positive integer. - 0 — The value is cut completely. Default value: `10000` symbols. @@ -2352,7 +2352,7 @@ Default value: `0`. ## persistent {#persistent} -Disables persistency for the [Set](../../engines/table-engines/special/set.md#set) and [Join](../../engines/table-engines/special/join.md#join) table engines. +Disables persistency for the [Set](../../engines/table-engines/special/set.md#set) and [Join](../../engines/table-engines/special/join.md#join) table engines. Reduces the I/O overhead. Suitable for scenarios that pursue performance and do not require persistence. @@ -2426,7 +2426,7 @@ Result: [ {"number":"0"}, {"number":"1"}, -{"number":"2"} +{"number":"2"} ] ``` @@ -2447,7 +2447,6 @@ Result: {"number":"2"} ``` -======= ## allow_nullable_key {#allow-nullable-key} Allows using of the [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable)-typed values in a sorting and a primary key for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree) tables. diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 482e4999ce..ce29272b45 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -9,7 +9,6 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT Поддерживаемые форматы и возможность использовать их в запросах `INSERT` и `SELECT` перечислены в таблице ниже. -======= | Формат | INSERT | SELECT | |-----------------------------------------------------------------------------------------|--------|--------| | [TabSeparated](#tabseparated) | ✔ | ✔ | @@ -1252,7 +1251,7 @@ SELECT * FROM line_as_string; ## RawBLOB {#rawblob} -В этом формате все входные данные считываются в одно значение. Парсить можно только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. +В этом формате все входные данные считываются в одно значение. Парсить можно только таблицу с одним полем типа [String](../sql-reference/data-types/string.md) или подобным ему. Результат выводится в бинарном виде без разделителей и экранирования. При выводе более одного значения формат неоднозначен и будет невозможно прочитать данные снова. Ниже приведено сравнение форматов `RawBLOB` и [TabSeparatedRaw](#tabseparatedraw). @@ -1272,7 +1271,7 @@ SELECT * FROM line_as_string; - строки представлены как длина в формате varint (unsigned [LEB128](https://en.wikipedia.org/wiki/LEB128)), а затем байты строки. При передаче на вход `RawBLOB` пустых данных, ClickHouse бросает исключение: - + ``` text Code: 108. DB::Exception: No data to insert ``` diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.sh b/tests/queries/0_stateless/01238_http_memory_tracking.sh index edace0c353..b317a6c109 100755 --- a/tests/queries/0_stateless/01238_http_memory_tracking.sh +++ b/tests/queries/0_stateless/01238_http_memory_tracking.sh @@ -13,7 +13,7 @@ ${CLICKHOUSE_CLIENT} --format Null -n <<<'SELECT sleepEachRow(1) FROM numbers(5) yes 'SELECT 1' 2>/dev/null | { head -n1000 } | { - xargs -i ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&max_memory_usage_for_user=$((1<<30))" -d '{}' + xargs -I{} ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&max_memory_usage_for_user=$((1<<30))" -d '{}' } | grep -x -c 1 wait diff --git a/tests/queries/0_stateless/01383_log_broken_table.sh b/tests/queries/0_stateless/01383_log_broken_table.sh index 6cecd050a6..80efa7e390 100755 --- a/tests/queries/0_stateless/01383_log_broken_table.sh +++ b/tests/queries/0_stateless/01383_log_broken_table.sh @@ -5,7 +5,7 @@ CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none . "$CURDIR"/../shell_config.sh -function test() +function test_func() { ENGINE=$1 MAX_MEM=4096 @@ -32,9 +32,9 @@ function test() $CLICKHOUSE_CLIENT --query "DROP TABLE log"; } -test TinyLog | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' -test StripeLog | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' -test Log | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' +test_func TinyLog | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' +test_func StripeLog | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' +test_func Log | grep -v -P '^(Memory limit|0\t0|File not found|[1-9]000000\t)' rm "${CLICKHOUSE_TMP}/insert_result" rm "${CLICKHOUSE_TMP}/select_result" diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index 283e6662a4..f259badea8 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Regression for UAF in ThreadPool. # (Triggered under TSAN) -for i in {1..10}; do +for _ in {1..10}; do ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' # Binding to specific CPU is not required, but this makes the test more reliable. taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' diff --git a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh index f251ff6138..b0abd99d38 100755 --- a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh +++ b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh @@ -16,6 +16,6 @@ opts=( ) ${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArray(repeat('a', if(_shard_num == 2, 100000, 1))), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" |& { # the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. - fgrep -q 'DB::Exception: Received from 127.3:9000. DB::Exception: Memory limit (for query) exceeded:' + grep -F -q 'DB::Exception: Received from 127.3:9000. DB::Exception: Memory limit (for query) exceeded:' # while if this will not correctly then it will got the exception from the 127.2:9000 and fail } diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 6727d5bd24..d3653deb98 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -131,4 +131,8 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | # Forbid stringstream because it's easy to use them incorrectly and hard to debug possible issues find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | grep -vP $EXCLUDE_DIRS | - xargs grep 'std::[io]\?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream" + xargs grep -P 'std::[io]?stringstream' | grep -v "STYLE_CHECK_ALLOW_STD_STRING_STREAM" && echo "Use WriteBufferFromOwnString or ReadBufferFromString instead of std::stringstream" + +# Conflict markers +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | + xargs grep -P '^(<<<<<<<|=======|>>>>>>>)$' | grep -P '.' && echo "Conflict markers are found in files" -- GitLab