Merge pull request #22011 from ClickHouse/min_max_time_system_parts_datetime64

Expose DateTime64 minmax part index in system.parts and system.parts_columns

src/Storages/MergeTree/IMergeTreeDataPart.cpp

@@ -333,40 +333,49 @@ IMergeTreeDataPart::State IMergeTreeDataPart::getState() const
 }
 
 
-DayNum IMergeTreeDataPart::getMinDate() const
+std::pair<DayNum, DayNum> IMergeTreeDataPart::getMinMaxDate() const
 {
     if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized)
-        return DayNum(minmax_idx.hyperrectangle[storage.minmax_idx_date_column_pos].left.get<UInt64>());
-    else
-        return DayNum();
-}
-
-
-DayNum IMergeTreeDataPart::getMaxDate() const
-{
-    if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized)
-        return DayNum(minmax_idx.hyperrectangle[storage.minmax_idx_date_column_pos].right.get<UInt64>());
+    {
+        const auto & hyperrectangle = minmax_idx.hyperrectangle[storage.minmax_idx_date_column_pos];
+        return {DayNum(hyperrectangle.left.get<UInt64>()), DayNum(hyperrectangle.right.get<UInt64>())};
+    }
     else
-        return DayNum();
+        return {};
 }
 
-time_t IMergeTreeDataPart::getMinTime() const
+std::pair<time_t, time_t> IMergeTreeDataPart::getMinMaxTime() const
 {
     if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized)
-        return minmax_idx.hyperrectangle[storage.minmax_idx_time_column_pos].left.get<UInt64>();
-    else
-        return 0;
-}
+    {
+        const auto & hyperrectangle = minmax_idx.hyperrectangle[storage.minmax_idx_time_column_pos];
 
+        /// The case of DateTime
+        if (hyperrectangle.left.getType() == Field::Types::UInt64)
+        {
+            assert(hyperrectangle.right.getType() == Field::Types::UInt64);
+            return {hyperrectangle.left.get<UInt64>(), hyperrectangle.right.get<UInt64>()};
+        }
+        /// The case of DateTime64
+        else if (hyperrectangle.left.getType() == Field::Types::Decimal64)
+        {
+            assert(hyperrectangle.right.getType() == Field::Types::Decimal64);
 
-time_t IMergeTreeDataPart::getMaxTime() const
-{
-    if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized)
-        return minmax_idx.hyperrectangle[storage.minmax_idx_time_column_pos].right.get<UInt64>();
+            auto left = hyperrectangle.left.get<DecimalField<Decimal64>>();
+            auto right = hyperrectangle.right.get<DecimalField<Decimal64>>();
+
+            assert(left.getScale() == right.getScale());
+
+            return { left.getValue() / left.getScaleMultiplier(), right.getValue() / right.getScaleMultiplier() };
+        }
+        else
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Part minmax index by time is neither DateTime or DateTime64");
+    }
     else
-        return 0;
+        return {};
 }
 
+
 void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns)
 {
     columns = new_columns;

src/Storages/MergeTree/IMergeTreeDataPart.h

@@ -155,13 +155,11 @@ public:
 
     bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); }
 
-    /// If the partition key includes date column (a common case), these functions will return min and max values for this column.
-    DayNum getMinDate() const;
-    DayNum getMaxDate() const;
+    /// If the partition key includes date column (a common case), this function will return min and max values for that column.
+    std::pair<DayNum, DayNum> getMinMaxDate() const;
 
-    /// otherwise, if the partition key includes dateTime column (also a common case), these functions will return min and max values for this column.
-    time_t getMinTime() const;
-    time_t getMaxTime() const;
+    /// otherwise, if the partition key includes dateTime column (also a common case), this function will return min and max values for that column.
+    std::pair<time_t, time_t> getMinMaxTime() const;
 
     bool isEmpty() const { return rows_count == 0; }
 

src/Storages/MergeTree/MergeTreeData.cpp

@@ -469,15 +469,19 @@ void MergeTreeData::checkPartitionKeyAndInitMinMax(const KeyDescription & new_pa
     DataTypes minmax_idx_columns_types = getMinMaxColumnsTypes(new_partition_key);
 
     /// Try to find the date column in columns used by the partition key (a common case).
-    bool encountered_date_column = false;
+    /// If there are no - DateTime or DateTime64 would also suffice.
+
+    bool has_date_column = false;
+    bool has_datetime_column = false;
+
     for (size_t i = 0; i < minmax_idx_columns_types.size(); ++i)
     {
-        if (typeid_cast<const DataTypeDate *>(minmax_idx_columns_types[i].get()))
+        if (isDate(minmax_idx_columns_types[i]))
         {
-            if (!encountered_date_column)
+            if (!has_date_column)
             {
                 minmax_idx_date_column_pos = i;
-                encountered_date_column = true;
+                has_date_column = true;
             }
             else
             {
@@ -486,16 +490,18 @@ void MergeTreeData::checkPartitionKeyAndInitMinMax(const KeyDescription & new_pa
             }
         }
     }
-    if (!encountered_date_column)
+    if (!has_date_column)
     {
         for (size_t i = 0; i < minmax_idx_columns_types.size(); ++i)
         {
-            if (typeid_cast<const DataTypeDateTime *>(minmax_idx_columns_types[i].get()))
+            if (isDateTime(minmax_idx_columns_types[i])
+                || isDateTime64(minmax_idx_columns_types[i])
+            )
             {
-                if (!encountered_date_column)
+                if (!has_datetime_column)
                 {
                     minmax_idx_time_column_pos = i;
-                    encountered_date_column = true;
+                    has_datetime_column = true;
                 }
                 else
                 {

src/Storages/System/StorageSystemParts.cpp

@@ -137,14 +137,17 @@ void StorageSystemParts::processNextStorage(
         if (columns_mask[src_index++])
             columns[res_index++]->insert(static_cast<UInt64>(part.use_count() - 1));
 
+        auto min_max_date = part->getMinMaxDate();
+        auto min_max_time = part->getMinMaxTime();
+
         if (columns_mask[src_index++])
-            columns[res_index++]->insert(part->getMinDate());
+            columns[res_index++]->insert(min_max_date.first);
         if (columns_mask[src_index++])
-            columns[res_index++]->insert(part->getMaxDate());
+            columns[res_index++]->insert(min_max_date.second);
         if (columns_mask[src_index++])
-            columns[res_index++]->insert(static_cast<UInt32>(part->getMinTime()));
+            columns[res_index++]->insert(static_cast<UInt32>(min_max_time.first));
         if (columns_mask[src_index++])
-            columns[res_index++]->insert(static_cast<UInt32>(part->getMaxTime()));
+            columns[res_index++]->insert(static_cast<UInt32>(min_max_time.second));
         if (columns_mask[src_index++])
             columns[res_index++]->insert(part->info.partition_id);
         if (columns_mask[src_index++])

src/Storages/System/StorageSystemPartsColumns.cpp

@@ -32,6 +32,8 @@ StorageSystemPartsColumns::StorageSystemPartsColumns(const StorageID & table_id_
         {"refcount",                                   std::make_shared<DataTypeUInt32>()},
         {"min_date",                                   std::make_shared<DataTypeDate>()},
         {"max_date",                                   std::make_shared<DataTypeDate>()},
+        {"min_time",                                   std::make_shared<DataTypeDateTime>()},
+        {"max_time",                                   std::make_shared<DataTypeDateTime>()},
         {"partition_id",                               std::make_shared<DataTypeString>()},
         {"min_block_number",                           std::make_shared<DataTypeInt64>()},
         {"max_block_number",                           std::make_shared<DataTypeInt64>()},
@@ -95,8 +97,10 @@ void StorageSystemPartsColumns::processNextStorage(
 
         /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
         auto use_count = part.use_count() - 1;
-        auto min_date = part->getMinDate();
-        auto max_date = part->getMaxDate();
+
+        auto min_max_date = part->getMinMaxDate();
+        auto min_max_time = part->getMinMaxTime();
+
         auto index_size_in_bytes = part->getIndexSizeInBytes();
         auto index_size_in_allocated_bytes = part->getIndexSizeInAllocatedBytes();
 
@@ -141,9 +145,14 @@ void StorageSystemPartsColumns::processNextStorage(
                 columns[res_index++]->insert(UInt64(use_count));
 
             if (columns_mask[src_index++])
-                columns[res_index++]->insert(min_date);
+                columns[res_index++]->insert(min_max_date.first);
+            if (columns_mask[src_index++])
+                columns[res_index++]->insert(min_max_date.second);
             if (columns_mask[src_index++])
-                columns[res_index++]->insert(max_date);
+                columns[res_index++]->insert(static_cast<UInt32>(min_max_time.first));
+            if (columns_mask[src_index++])
+                columns[res_index++]->insert(static_cast<UInt32>(min_max_time.second));
+
             if (columns_mask[src_index++])
                 columns[res_index++]->insert(part->info.partition_id);
             if (columns_mask[src_index++])

tests/queries/0_stateless/01773_min_max_time_system_parts_datetime64.reference

+2000-01-02 03:04:05	2001-02-03 04:05:06
+2000-01-02 03:04:05	2001-02-03 04:05:06

tests/queries/0_stateless/01773_min_max_time_system_parts_datetime64.sql

+DROP TABLE IF EXISTS test;
+CREATE TABLE test (time DateTime64(3)) ENGINE = MergeTree ORDER BY tuple() PARTITION BY toStartOfInterval(time, INTERVAL 2 YEAR);
+
+INSERT INTO test VALUES ('2000-01-02 03:04:05.123'), ('2001-02-03 04:05:06.789');
+
+SELECT min_time, max_time FROM system.parts WHERE table = 'test' AND database = currentDatabase();
+SELECT min_time, max_time FROM system.parts_columns WHERE table = 'test' AND database = currentDatabase();
+
+DROP TABLE test;