add arbitrary const expressions in limit processing

dbms/src/Common/ErrorCodes.cpp

@@ -414,6 +414,7 @@ namespace ErrorCodes
     extern const int PROTOBUF_FIELD_NOT_REPEATED = 437;
     extern const int DATA_TYPE_CANNOT_BE_PROMOTED = 438;
     extern const int CANNOT_SCHEDULE_TASK = 439;
+    extern const int INVALID_LIMIT_EXPRESSION = 440;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Interpreters/InterpreterSelectQuery.cpp

@@ -33,6 +33,7 @@
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 #include <Interpreters/InterpreterSetQuery.h>
+#include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/DatabaseAndTableWithAlias.h>
 #include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
@@ -45,6 +46,7 @@
 #include <TableFunctions/TableFunctionFactory.h>
 
 #include <Core/Field.h>
+#include <Core/Types.h>
 #include <Columns/Collator.h>
 #include <Common/typeid_cast.h>
 #include <Parsers/queryToString.h>
@@ -68,6 +70,7 @@ namespace ErrorCodes
     extern const int NOT_IMPLEMENTED;
     extern const int PARAMETER_OUT_OF_BOUND;
     extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int INVALID_LIMIT_EXPRESSION;
 }
 
 InterpreterSelectQuery::InterpreterSelectQuery(
@@ -717,19 +720,29 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt
 }
 
 
-static void getLimitLengthAndOffset(ASTSelectQuery & query, size_t & length, size_t & offset)
+void InterpreterSelectQuery::getLimitLengthAndOffset(ASTSelectQuery & query, size_t & length, size_t & offset)
 {
     length = 0;
     offset = 0;
     if (query.limit_length)
     {
-        length = safeGet<UInt64>(typeid_cast<ASTLiteral &>(*query.limit_length).value);
+        getLimitUIntValue(query.limit_length, length);
         if (query.limit_offset)
-            offset = safeGet<UInt64>(typeid_cast<ASTLiteral &>(*query.limit_offset).value);
+            getLimitUIntValue(query.limit_offset, offset);
     }
 }
 
 
+void InterpreterSelectQuery::getLimitUIntValue(const ASTPtr& ptr, size_t& result)
+{
+    const auto& eval_result = evaluateConstantExpression(ptr, context);
+    if (!isNumber(eval_result.second)) {
+        throw Exception("Illegal limit expression", ErrorCodes::INVALID_LIMIT_EXPRESSION);
+    }
+    result = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), eval_result.first);
+}
+
+
 void InterpreterSelectQuery::executeFetchColumns(
         QueryProcessingStage::Enum processing_stage, Pipeline & pipeline,
         const PrewhereInfoPtr & prewhere_info, const Names & columns_to_remove_after_prewhere)
@@ -1230,7 +1243,7 @@ static SortDescription getSortDescription(ASTSelectQuery & query)
     return order_descr;
 }
 
-static size_t getLimitForSorting(ASTSelectQuery & query)
+size_t InterpreterSelectQuery::getLimitForSorting(ASTSelectQuery & query)
 {
     /// Partial sort can be done if there is LIMIT but no DISTINCT or LIMIT BY.
     size_t limit = 0;

dbms/src/Interpreters/InterpreterSelectQuery.h

@@ -171,6 +171,11 @@ private:
      */
     void getDatabaseAndTableNames(String & database_name, String & table_name);
 
+    size_t getLimitForSorting(ASTSelectQuery & query);
+
+    void getLimitLengthAndOffset(ASTSelectQuery & query, size_t & length, size_t & offset);
+
+    void getLimitUIntValue(const ASTPtr& ptr, size_t& result);
     /// Different stages of query execution.
 
     /// dry_run - don't read from table, use empty header block instead.

dbms/src/Interpreters/evaluateConstantExpression.cpp

@@ -40,18 +40,18 @@ std::pair<Field, std::shared_ptr<const IDataType>> evaluateConstantExpression(co
     expr_for_constant_folding->execute(block_with_constants);
 
     if (!block_with_constants || block_with_constants.rows() == 0)
-        throw Exception("Logical error: empty block after evaluation of constant expression for IN or VALUES", ErrorCodes::LOGICAL_ERROR);
+        throw Exception("Logical error: empty block after evaluation of constant expression for IN, VALUES or LIMIT", ErrorCodes::LOGICAL_ERROR);
 
     String name = node->getColumnName();
 
     if (!block_with_constants.has(name))
-        throw Exception("Element of set in IN or VALUES is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS);
+        throw Exception("Element of set in IN, VALUES or LIMIT is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS);
 
     const ColumnWithTypeAndName & result = block_with_constants.getByName(name);
     const IColumn & result_column = *result.column;
 
     if (!result_column.isColumnConst())
-        throw Exception("Element of set in IN or VALUES is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS);
+        throw Exception("Element of set in IN, VALUES or LIMIT is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS);
 
     return std::make_pair(result_column[0], result.type);
 }

dbms/src/Parsers/ParserSelectQuery.cpp

@@ -173,15 +173,15 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             throw Exception("Can not use TOP and LIMIT together", ErrorCodes::TOP_AND_LIMIT_TOGETHER);
 
         ParserToken s_comma(TokenType::Comma);
-        ParserNumber num;
+        ParserExpressionElement exp_element;
 
-        if (!num.parse(pos, select_query->limit_length, expected))
+        if (!exp_element.parse(pos, select_query->limit_length, expected))
             return false;
 
         if (s_comma.ignore(pos, expected))
         {
             select_query->limit_offset = select_query->limit_length;
-            if (!num.parse(pos, select_query->limit_length, expected))
+            if (!exp_element.parse(pos, select_query->limit_length, expected))
                 return false;
         }
         else if (s_by.ignore(pos, expected))
@@ -194,7 +194,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         }
         else if (s_offset.ignore(pos, expected))
         {
-            if (!num.parse(pos, select_query->limit_offset, expected))
+            if (!exp_element.parse(pos, select_query->limit_offset, expected))
                 return false;
         }
     }
@@ -206,15 +206,15 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             return false;
 
         ParserToken s_comma(TokenType::Comma);
-        ParserNumber num;
+        ParserExpressionElement exp_element;
 
-        if (!num.parse(pos, select_query->limit_length, expected))
+        if (!exp_element.parse(pos, select_query->limit_length, expected))
             return false;
 
         if (s_comma.ignore(pos, expected))
         {
             select_query->limit_offset = select_query->limit_length;
-            if (!num.parse(pos, select_query->limit_length, expected))
+            if (!exp_element.parse(pos, select_query->limit_length, expected))
                 return false;
         }
     }

dbms/src/Parsers/tests/select_parser.cpp

@@ -18,7 +18,7 @@ try
         " GROUP BY UniqID"
         " HAVING SUM(Refresh) > 100"
         " ORDER BY Visits, PageViews"
-        " LIMIT 1000, 10"
+        " LIMIT LENGTH('STRING OF 20 SYMBOLS') - 20 + 1000, 10.05 / 5.025 * 5"
         " INTO OUTFILE 'test.out'"
         " FORMAT TabSeparated";
 

dbms/tests/queries/0_stateless/00834_limit_with_constant_expressions.reference

+0
+3
+4

dbms/tests/queries/0_stateless/00834_limit_with_constant_expressions.sql

+SELECT * FROM numbers(10) LIMIT 0.33 / 0.165 - 0.33 + 0.67;
+SELECT * FROM numbers(10) LIMIT LENGTH('NNN') + COS(0), TO_DATE('0000-00-02');
+SELECT * FROM numbers(10) LIMIT LENGTH('NNN') + COS(0), TO_DATE('0000-00-02');
+SELECT * FROM numbers(10) LIMIT a + 5 - a;
+SELECT * FROM numbers(10) LIMIT a + b;
+SELECT * FROM numbers(10) LIMIT "Hello";
+SELECT * FROM numbers(10) LIMIT ;

debian/changelog

@@ -2,4 +2,4 @@ clickhouse (19.1.6) unstable; urgency=low
 
   * Modified source code
 
- --  <root@yandex-team.ru>  Thu, 24 Jan 2019 00:17:07 +0300
+ -- Vitaliy Karnienko <karnienko@yandex-team.ru>  Sun, 03 Feb 2019 20:36:04 +0300