From adaae8a12ce09008f078be97ca2708dbbdbf65b3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 13 Oct 2020 13:59:43 +0300 Subject: [PATCH] Added OutputFormat setting date_time_output_format --- docs/en/operations/settings/settings.md | 27 +++++++++- docs/en/sql-reference/data-types/datetime.md | 3 +- .../en/sql-reference/data-types/datetime64.md | 1 + src/Core/Settings.h | 1 + src/Core/SettingsEnums.cpp | 5 ++ src/Core/SettingsEnums.h | 1 + src/DataTypes/DataTypeDateTime.cpp | 16 +++++- src/DataTypes/DataTypeDateTime64.cpp | 16 +++++- src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 9 ++++ src/IO/WriteHelpers.h | 49 +++++++++++++++---- .../01516_date_time_output_format.reference | 12 +++++ .../01516_date_time_output_format.sql | 36 ++++++++++++++ 13 files changed, 162 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/01516_date_time_output_format.reference create mode 100644 tests/queries/0_stateless/01516_date_time_output_format.sql diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f3d096f1e1..baeabd0250 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -389,6 +389,31 @@ See also: - [DateTime data type.](../../sql-reference/data-types/datetime.md) - [Functions for working with dates and times.](../../sql-reference/functions/date-time-functions.md) +## date\_time\_output_\_format {#settings-date_time_output_format} + +Allows choosing different output formats of the text representation of date and time. + +Possible values: + +- `'simple'` - Simple output format. + + Clickhouse output date and time `YYYY-MM-DD hh:mm:ss` format. For example, `'2019-08-20 10:18:56'`. Calculation is performed according to the data type's time zone (if present) or server time zone. + +- `'iso'` - ISO output format. + + Clickhouse output date and time in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) `YYYY-MM-DDThh:mm:ssZ` format. For example, `'2019-08-20T10:18:56Z'`. Note that output is in UTC (`Z` means UTC). + +- `'unix_timestamp'` - Unix timestamp output format. + + Clickhouse output date and time in [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) format. For example `'1566285536'`. + +Default value: `'simple'`. + +See also: + +- [DateTime data type.](../../sql-reference/data-types/datetime.md) +- [Functions for working with dates and times.](../../sql-reference/functions/date-time-functions.md) + ## join\_default\_strictness {#settings-join_default_strictness} Sets default strictness for [JOIN clauses](../../sql-reference/statements/select/join.md#select-join). @@ -2066,4 +2091,4 @@ Possible values: - 1 — The bigint data type is enabled. - 0 — The bigint data type is disabled. -Default value: `0`. \ No newline at end of file +Default value: `0`. diff --git a/docs/en/sql-reference/data-types/datetime.md b/docs/en/sql-reference/data-types/datetime.md index a2ae68ebf1..db9e84205a 100644 --- a/docs/en/sql-reference/data-types/datetime.md +++ b/docs/en/sql-reference/data-types/datetime.md @@ -27,7 +27,7 @@ You can explicitly set a time zone for `DateTime`-type columns when creating a t The [clickhouse-client](../../interfaces/cli.md) applies the server time zone by default if a time zone isn’t explicitly set when initializing the data type. To use the client time zone, run `clickhouse-client` with the `--use_client_time_zone` parameter. -ClickHouse outputs values in `YYYY-MM-DD hh:mm:ss` text format by default. You can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. +ClickHouse outputs values depending on the value of the [date\_time\_output\_format](../../operations/settings/settings.md#settings-date_time_output_format) setting. `YYYY-MM-DD hh:mm:ss` text format by default. Additionaly you can change the output with the [formatDateTime](../../sql-reference/functions/date-time-functions.md#formatdatetime) function. When inserting data into ClickHouse, you can use different formats of date and time strings, depending on the value of the [date\_time\_input\_format](../../operations/settings/settings.md#settings-date_time_input_format) setting. @@ -120,6 +120,7 @@ FROM dt - [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) - [The `date_time_input_format` setting](../../operations/settings/settings.md#settings-date_time_input_format) +- [The `date_time_output_format` setting](../../operations/settings/settings.md#settings-date_time_output_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) - [The `Date` data type](../../sql-reference/data-types/date.md) diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index 9a3b198b5e..5cba831509 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -96,6 +96,7 @@ FROM dt - [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md) - [Functions for working with arrays](../../sql-reference/functions/array-functions.md) - [The `date_time_input_format` setting](../../operations/settings/settings.md#settings-date_time_input_format) +- [The `date_time_output_format` setting](../../operations/settings/settings.md#settings-date_time_output_format) - [The `timezone` server configuration parameter](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) - [Operators for working with dates and times](../../sql-reference/operators/index.md#operators-datetime) - [`Date` data type](../../sql-reference/data-types/date.md) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a1a7a690e4..8f303e3fb4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -415,6 +415,7 @@ class IColumn; M(Bool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ + M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ \ M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ M(Bool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index b4db51a506..c337cd5e3c 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -66,6 +66,11 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, ErrorCodes::BAD_ARGUMENT {"best_effort", FormatSettings::DateTimeInputFormat::BestEffort}}) +IMPLEMENT_SETTING_ENUM_WITH_RENAME(DateTimeOutputFormat, ErrorCodes::BAD_ARGUMENTS, + {{"simple", FormatSettings::DateTimeOutputFormat::Simple}, + {"iso", FormatSettings::DateTimeOutputFormat::ISO}, + {"unix_timestamp", FormatSettings::DateTimeOutputFormat::UnixTimestamp}}) + IMPLEMENT_SETTING_ENUM(LogsLevel, ErrorCodes::BAD_ARGUMENTS, {{"none", LogsLevel::none}, {"fatal", LogsLevel::fatal}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 426497fff7..80b9bf9add 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -83,6 +83,7 @@ DECLARE_SETTING_ENUM(DistributedProductMode) DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeInputFormat, FormatSettings::DateTimeInputFormat) +DECLARE_SETTING_ENUM_WITH_RENAME(DateTimeOutputFormat, FormatSettings::DateTimeOutputFormat) enum class LogsLevel { diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index 9ea698d4fb..bfb4473e42 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -59,9 +59,21 @@ String DataTypeDateTime::doGetName() const return out.str(); } -void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const +void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - writeDateTimeText(assert_cast(column).getData()[row_num], ostr, time_zone); + auto value = assert_cast(column).getData()[row_num]; + switch (settings.date_time_output_format) + { + case FormatSettings::DateTimeOutputFormat::Simple: + writeDateTimeText(value, ostr, time_zone); + return; + case FormatSettings::DateTimeOutputFormat::UnixTimestamp: + writeIntText(value, ostr); + return; + case FormatSettings::DateTimeOutputFormat::ISO: + writeDateTimeTextISO(value, ostr, utc_time_zone); + return; + } } void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/DataTypeDateTime64.cpp b/src/DataTypes/DataTypeDateTime64.cpp index ee4139c2b7..ef1a971510 100644 --- a/src/DataTypes/DataTypeDateTime64.cpp +++ b/src/DataTypes/DataTypeDateTime64.cpp @@ -57,9 +57,21 @@ std::string DataTypeDateTime64::doGetName() const return out.str(); } -void DataTypeDateTime64::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & /*settings*/) const +void DataTypeDateTime64::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - writeDateTimeText(assert_cast(column).getData()[row_num], scale, ostr, time_zone); + auto value = assert_cast(column).getData()[row_num]; + switch (settings.date_time_output_format) + { + case FormatSettings::DateTimeOutputFormat::Simple: + writeDateTimeText(value, scale, ostr, time_zone); + return; + case FormatSettings::DateTimeOutputFormat::UnixTimestamp: + writeDateTimeUnixTimestamp(value, scale, ostr); + return; + case FormatSettings::DateTimeOutputFormat::ISO: + writeDateTimeTextISO(value, scale, ostr, utc_time_zone); + return; + } } void DataTypeDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 065b14f86b..4ee5309eac 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -127,6 +127,7 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; format_settings.avro.output_codec = settings.output_format_avro_codec; format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; + format_settings.date_time_output_format = settings.date_time_output_format; return format_settings; } diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index a97bd9bf6c..20c8a03223 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -99,6 +99,15 @@ struct FormatSettings DateTimeInputFormat date_time_input_format = DateTimeInputFormat::Basic; + enum class DateTimeOutputFormat + { + Simple, + ISO, + UnixTimestamp + }; + + DateTimeOutputFormat date_time_output_format = DateTimeOutputFormat::Simple; + UInt64 input_allow_errors_num = 0; Float32 input_allow_errors_ratio = 0; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 1f0fe09505..d55fc4c5ef 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -636,6 +636,19 @@ inline void writeUUIDText(const UUID & uuid, WriteBuffer & buf) buf.write(s, sizeof(s)); } +template +inline void writeDecimalTypeFractionalText(typename DecimalType::NativeType fractional, UInt32 scale, WriteBuffer & buf) +{ + static constexpr UInt32 MaxScale = DecimalUtils::maxPrecision(); + + char data[20] = {'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'}; + static_assert(sizeof(data) >= MaxScale); + + for (Int32 pos = scale - 1; pos >= 0 && fractional; --pos, fractional /= DateTime64(10)) + data[pos] += fractional % DateTime64(10); + + writeString(&data[0], static_cast(scale), buf); +} static const char digits100[201] = "00010203040506070809" @@ -760,15 +773,7 @@ inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & if (scale > 0) { buf.write(fractional_time_delimiter); - - char data[20] = {'0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'}; - static_assert(sizeof(data) >= MaxScale); - - auto fractional = c.fractional; - for (Int32 pos = scale - 1; pos >= 0 && fractional; --pos, fractional /= DateTime64(10)) - data[pos] += fractional % DateTime64(10); - - writeString(&data[0], static_cast(scale), buf); + writeDecimalTypeFractionalText(c.fractional, scale, buf); } } @@ -798,6 +803,32 @@ inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const D buf.write(" GMT", 4); } +inline void writeDateTimeTextISO(time_t datetime, WriteBuffer & buf, const DateLUTImpl & utc_time_zone) +{ + writeDateTimeText<'-', ':', 'T'>(datetime, buf, utc_time_zone); + buf.write('Z'); +} + +inline void writeDateTimeTextISO(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & utc_time_zone) +{ + writeDateTimeText<'-', ':', 'T'>(datetime64, scale, buf, utc_time_zone); + buf.write('Z'); +} + +inline void writeDateTimeUnixTimestamp(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf) +{ + static constexpr UInt32 MaxScale = DecimalUtils::maxPrecision(); + scale = scale > MaxScale ? MaxScale : scale; + + auto c = DecimalUtils::split(datetime64, scale); + writeIntText(c.whole, buf); + + if (scale > 0) + { + buf.write('.'); + writeDecimalTypeFractionalText(c.fractional, scale, buf); + } +} /// Methods for output in binary format. template diff --git a/tests/queries/0_stateless/01516_date_time_output_format.reference b/tests/queries/0_stateless/01516_date_time_output_format.reference new file mode 100644 index 0000000000..8cb5ca739d --- /dev/null +++ b/tests/queries/0_stateless/01516_date_time_output_format.reference @@ -0,0 +1,12 @@ +2020-10-15 00:00:00 +2020-10-15 00:00:00 +2020-10-14T21:00:00Z +2020-10-14T21:00:00Z +1602709200 +1602709200 +2020-10-15 00:00:00.000 +2020-10-15 00:00:00.123 +2020-10-14T21:00:00.000Z +2020-10-14T21:00:00.123Z +1602709200.000 +1602709200.123 diff --git a/tests/queries/0_stateless/01516_date_time_output_format.sql b/tests/queries/0_stateless/01516_date_time_output_format.sql new file mode 100644 index 0000000000..224d8ef103 --- /dev/null +++ b/tests/queries/0_stateless/01516_date_time_output_format.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS test_datetime; + +CREATE TABLE test_datetime(timestamp DateTime('Europe/Moscow')) ENGINE=Log; + +INSERT INTO test_datetime VALUES ('2020-10-15 00:00:00'); + +SET date_time_output_format = 'simple'; +SELECT timestamp FROM test_datetime; +SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Europe/Moscow'), '%Y-%m-%d %R:%S') as formatted_simple FROM test_datetime; + +SET date_time_output_format = 'iso'; +SELECT timestamp FROM test_datetime; +SELECT formatDateTime(toDateTime('2020-10-15 00:00:00', 'Europe/Moscow'), '%Y-%m-%dT%R:%SZ', 'UTC') as formatted_iso FROM test_datetime;; + +SET date_time_output_format = 'unix_timestamp'; +SELECT timestamp FROM test_datetime; +SELECT toUnixTimestamp(timestamp) FROM test_datetime; + +SET date_time_output_format = 'simple'; +DROP TABLE test_datetime; + +CREATE TABLE test_datetime(timestamp DateTime64(3, 'Europe/Moscow')) Engine=Log; + +INSERT INTO test_datetime VALUES ('2020-10-15 00:00:00'), (1602709200123); + +SET date_time_output_format = 'simple'; +SELECT timestamp FROM test_datetime; + +SET date_time_output_format = 'iso'; +SELECT timestamp FROM test_datetime; + +SET date_time_output_format = 'unix_timestamp'; +SELECT timestamp FROM test_datetime; + +SET date_time_output_format = 'simple'; +DROP TABLE test_datetime; -- GitLab