fix(sql): fix designated timestamp filter with now() function applied...

fix(sql): fix designated timestamp filter with now() function applied incorrectly in some cases (#2014)

Fix the bug resulting in queries of with designated timestamp filter with now() function

```
WHERE ts >= '2022-03-23T08:00:00.000000Z' AND ts < '2022-03-25T10:00:00.000000Z' AND ts > dateadd('d', -10, now())
```

could return different results depending of order of comparisons in WHERE

core/src/main/java/io/questdb/griffin/model/IntervalOperation.java

@@ -25,6 +25,8 @@
 package io.questdb.griffin.model;
 
 public final class IntervalOperation {
+    public static final short NONE = 0;
+
     public static final short INTERSECT = 1;
     public static final short INTERSECT_BETWEEN = 3;
     public static final short INTERSECT_INTERVALS = 4;

core/src/main/java/io/questdb/griffin/model/IntervalUtils.java

@@ -47,7 +47,7 @@ public final class IntervalUtils {
             int periodCount,
             short operation,
             LongList out) {
-        addHiLoInterval(lo, hi, period, periodType, periodCount, (short) 0, (short) 0, operation, out);
+        addHiLoInterval(lo, hi, period, periodType, periodCount, IntervalDynamicIndicator.NONE, IntervalOperation.NONE, operation, out);
     }
 
     public static void addHiLoInterval(
@@ -78,33 +78,33 @@ public final class IntervalUtils {
                                        short dynamicIndicator,
                                        short operation,
                                        LongList out) {
-        addHiLoInterval(lo, hi, 0, (char) 0, 1, adjustment, dynamicIndicator, operation, out);
+        addHiLoInterval(lo, hi, 0, PeriodType.NONE, 1, adjustment, dynamicIndicator, operation, out);
     }
 
     public static void addHiLoInterval(long lo, long hi, short operation, LongList out) {
-        addHiLoInterval(lo, hi, 0, (char) 0, 1, operation, out);
+        addHiLoInterval(lo, hi, 0, PeriodType.NONE, 1, operation, out);
     }
 
     public static void apply(LongList temp, long lo, long hi, int period, char periodType, int count) {
         temp.add(lo, hi);
         if (count > 1) {
             switch (periodType) {
-                case 'y':
+                case PeriodType.YEAR:
                     addYearIntervals(period, count, temp);
                     break;
-                case 'M':
+                case PeriodType.MONTH:
                     addMonthInterval(period, count, temp);
                     break;
-                case 'h':
+                case PeriodType.HOUR:
                     addMillisInterval(period * Timestamps.HOUR_MICROS, count, temp);
                     break;
-                case 'm':
+                case PeriodType.MINUTE:
                     addMillisInterval(period * Timestamps.MINUTE_MICROS, count, temp);
                     break;
-                case 's':
+                case PeriodType.SECOND:
                     addMillisInterval(period * Timestamps.SECOND_MICROS, count, temp);
                     break;
-                case 'd':
+                case PeriodType.DAY:
                     addMillisInterval(period * Timestamps.DAY_MICROS, count, temp);
                     break;
             }
@@ -120,7 +120,7 @@ public final class IntervalUtils {
         int count = getEncodedPeriodCount(intervals, index);
 
         intervals.setPos(index);
-        if (periodType == 0) {
+        if (periodType == PeriodType.NONE) {
             intervals.extendAndSet(index + 1, hi);
             intervals.setQuick(index, lo);
             return;
@@ -578,12 +578,12 @@ public final class IntervalUtils {
 
                 replaceHiLoInterval(low, hi, period, type, count, operation, out);
                 switch (type) {
-                    case 'y':
-                    case 'M':
-                    case 'h':
-                    case 'm':
-                    case 's':
-                    case 'd':
+                    case PeriodType.YEAR:
+                    case PeriodType.MONTH:
+                    case PeriodType.HOUR:
+                    case PeriodType.MINUTE:
+                    case PeriodType.SECOND:
+                    case PeriodType.DAY:
                         break;
                     default:
                         throw SqlException.$(position, "Unknown period: " + type + " at " + (p - 1));

core/src/main/java/io/questdb/griffin/model/IntervalModel.java → core/src/main/java/io/questdb/griffin/model/PeriodType.java

@@ -24,8 +24,13 @@
 
 package io.questdb.griffin.model;
 
-public interface IntervalModel {
-    void intersectIntervals(long lo, long hi);
+public final class PeriodType {
+    public static final char NONE = (char) 0;
 
-    void intersectIntervals(CharSequence seq, int lo, int lim, int position);
+    public static final char YEAR = 'y';
+    public static final char MONTH = 'M';
+    public static final char HOUR = 'h';
+    public static final char MINUTE = 'm';
+    public static final char SECOND = 's';
+    public static final char DAY = 'd';
 }

core/src/main/java/io/questdb/griffin/model/RuntimeIntervalModel.java

@@ -103,6 +103,7 @@ public class RuntimeIntervalModel implements RuntimeIntrinsicIntervalModel {
         int size = intervals.size();
         int dynamicStart = size - dynamicRangeList.size() * STATIC_LONGS_PER_DYNAMIC_INTERVAL;
         int dynamicIndex = 0;
+        boolean firstFuncApplied = false;
 
         for (int i = dynamicStart; i < size; i += STATIC_LONGS_PER_DYNAMIC_INTERVAL) {
             Function dynamicFunction = dynamicRangeList.getQuick(dynamicIndex++);
@@ -191,7 +192,7 @@ public class RuntimeIntervalModel implements RuntimeIntrinsicIntervalModel {
 
             // Do not apply operation (intersect, subtract)
             // if this is first element and no pre-calculated static intervals exist
-            if (divider > 0) {
+            if (firstFuncApplied || divider > 0) {
                 switch (operation) {
                     case IntervalOperation.INTERSECT:
                     case IntervalOperation.INTERSECT_BETWEEN:
@@ -208,6 +209,7 @@ public class RuntimeIntervalModel implements RuntimeIntrinsicIntervalModel {
                         throw new UnsupportedOperationException("Interval operation " + operation + " is not supported");
                 }
             }
+            firstFuncApplied = true;
         }
     }
 

core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java

@@ -1544,6 +1544,59 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest {
                         "57.78947915182423\tABC\n");
     }
 
+    @Test
+    public void testFilterTimestamps() throws Exception {
+        // ts
+        // 2022-03-22 10:00:00.0
+        // 2022-03-23 10:00:00.0
+        // 2022-03-24 10:00:00.0
+        // 2022-03-25 10:00:00.0
+        // 2022-03-26 10:00:00.0
+        // 2022-03-27 10:00:00.0
+        // 2022-03-28 10:00:00.0
+        // 2022-03-29 10:00:00.0
+        // 2022-03-30 10:00:00.0
+        // 2022-03-31 10:00:00.0
+        currentMicros = 1649186452792000L; // '2022-04-05T19:20:52.792Z'
+        assertQuery(
+                "min\tmax\n" +
+                        "\t\n",
+                "SELECT min(ts), max(ts)\n" +
+                        "FROM tab\n" +
+                        "WHERE ts >= '2022-03-23T08:00:00.000000Z' AND ts < '2022-03-25T10:00:00.000000Z' AND ts > dateadd('d', -10, systimestamp())",
+                "CREATE TABLE tab AS (\n" +
+                        "    SELECT dateadd('d', CAST(-(10-x) AS INT) , '2022-03-31T10:00:00.000000Z') AS ts \n" +
+                        "    FROM long_sequence(10)\n" +
+                        ") TIMESTAMP(ts) PARTITION BY DAY",
+                null,
+                null,
+                null,
+                false,
+                false,
+                true
+        );
+
+        compiler.compile("drop table tab", sqlExecutionContext);
+
+        assertQuery(
+                "min\tmax\n" +
+                        "\t\n",
+                "SELECT min(ts), max(ts)\n" +
+                        " FROM tab\n" +
+                        " WHERE ts >= '2022-03-23T08:00:00.000000Z' AND ts < '2022-03-25T10:00:00.000000Z' AND ts > dateadd('d', -10, now())",
+                "CREATE TABLE tab AS (\n" +
+                        "    SELECT dateadd('d', CAST(-(10-x) AS INT) , '2022-03-31T10:00:00.000000Z') AS ts \n" +
+                        "    FROM long_sequence(10)\n" +
+                        ") TIMESTAMP(ts) PARTITION BY DAY",
+                null,
+                null,
+                null,
+                false,
+                false,
+                true
+        );
+    }
+
     @Test
     public void testFilterWithIndexedBindVariableSingleIndexedSymbol() throws Exception {
         bindVariableService.clear();

core/src/test/java/io/questdb/griffin/WhereClauseParserTest.java

@@ -31,6 +31,7 @@ import io.questdb.cairo.sql.Function;
 import io.questdb.cairo.sql.RecordMetadata;
 import io.questdb.griffin.engine.functions.bind.BindVariableServiceImpl;
 import io.questdb.griffin.model.*;
+import io.questdb.std.LongList;
 import io.questdb.std.Numbers;
 import io.questdb.std.NumericException;
 import io.questdb.std.ObjList;
@@ -247,12 +248,12 @@ public class WhereClauseParserTest extends AbstractCairoTest {
 
     @Test
     public void testBadOperators() {
-        testBadOperator(">","too few arguments for '>' [found=1,expected=2]");
-        testBadOperator(">=","too few arguments for '>=' [found=1,expected=2]");
-        testBadOperator("<","too few arguments for '<' [found=1,expected=2]");
-        testBadOperator("<=","too few arguments for '<=' [found=1,expected=2]");
-        testBadOperator("=","too few arguments for '=' [found=1,expected=2]");
-        testBadOperator("!=","too few arguments for '!=' [found=1,expected=2]");
+        testBadOperator(">", "too few arguments for '>' [found=1,expected=2]");
+        testBadOperator(">=", "too few arguments for '>=' [found=1,expected=2]");
+        testBadOperator("<", "too few arguments for '<' [found=1,expected=2]");
+        testBadOperator("<=", "too few arguments for '<=' [found=1,expected=2]");
+        testBadOperator("=", "too few arguments for '=' [found=1,expected=2]");
+        testBadOperator("!=", "too few arguments for '!=' [found=1,expected=2]");
     }
 
     @Test
@@ -828,6 +829,92 @@ public class WhereClauseParserTest extends AbstractCairoTest {
         }
     }
 
+    @Test
+    public void testInterval() throws Exception {
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp >= '2022-03-23T08:00:00.000000Z'",
+                        "timestamp < '2022-03-25T10:00:00.000000Z'",
+                        "timestamp > '2022-03-26T19:20:52.792Z'"
+                },
+                "[]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp >= '2022-03-23T08:00:00.000000Z'",
+                        "timestamp < '2022-03-25T10:00:00.000000Z'",
+                        "timestamp > dateadd('d', -10, now())"
+                },
+                "[]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp >= '2022-03-23T08:00:00.000000Z'",
+                        "timestamp < '2022-03-25T10:00:00.000000Z'",
+                        "timestamp > dateadd('d', -10, '2022-04-05T19:20:52.792Z')"
+                },
+                "[]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp BETWEEN '2022-03-23T08:00:00.000000Z' AND now()",
+                        "timestamp BETWEEN now() AND '2022-03-23T08:00:00.000000Z'",
+                        "timestamp IN ('2022-03-23')",
+                        "timestamp > dateadd('d', 1,'2022-03-23T08:00:00.000000Z')"
+                },
+                "[]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp BETWEEN '2022-03-23T08:00:00.000000Z' AND '2022-03-25T10:00:00.000000Z'",
+                        "timestamp BETWEEN '2022-03-23T08:00:00.000000Z' AND now()",
+                        "timestamp NOT IN ('2022-03-25')",
+                        "timestamp != now() - 15",
+                        "timestamp > '2021-01'",
+                        "timestamp < '2022-04'",
+                        "timestamp > '2022-05'"
+                },
+                "[]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp BETWEEN '2022-03-23T08:00:00.000000Z' AND '2022-03-25T10:00:00.000000Z'",
+                        "timestamp NOT IN ('2022-03-25')",
+                        "timestamp != now() - 15",
+                        "timestamp > '2021-01'",
+                        "timestamp < '2022-04'"
+                },
+                "[1648022400000000,1648202400000000]"
+        );
+
+        andShuffleExpressionsTest(
+                new String[]{
+                        "timestamp BETWEEN '2022-03-23T08:00:00.000000Z' AND '2022-03-25T10:00:00.000000Z'",
+                        "timestamp NOT IN ('2022-03-25')",
+                        "timestamp != now() - 15",
+                        "timestamp > '2021-01'",
+                        "timestamp < '2022-04'",
+                        "timestamp NOT BETWEEN '2022-03-23T08:00:00.000000Z' AND '2022-03-25T10:00:00.000000Z'"
+                },
+                "[1648022400000000,1648202400000000]"
+        );
+    }
+
+    @Test
+    public void testSeeminglyLookingDynamicInterval() throws Exception {
+        // not equivalent to: timestamp >= '2022-03-23T08:00:00.000000Z' AND timestamp < '2022-03-25T10:00:00.000000Z' AND timestamp > '2022-03-26T19:20:52.792Z'
+        // because 'systimestamp' is neither constant/runtime-constant, so the latter AND is not intrinsic and thus is out of the intervals model
+        String whereExpression = "timestamp >= '2022-03-23T08:00:00.000000Z' AND timestamp < '2022-03-25T10:00:00.000000Z' AND timestamp > dateadd('d', -10, systimestamp())";
+        currentMicros = 1649186452792000L; // '2022-04-05T19:20:52.792Z'
+        LongList intervals = modelOf(whereExpression).buildIntervalModel().calculateIntervals(sqlExecutionContext);
+        Assert.assertEquals("[1648022400000000,1648202399999999]", intervals.toString());
+    }
+
     @Test
     public void testIntervalDontIntersect() throws Exception {
         // because of intervals being processed from right to left
@@ -1797,6 +1884,39 @@ public class WhereClauseParserTest extends AbstractCairoTest {
         }
     }
 
+    private void andShuffleExpressionsTest(String[] expressions, String expected) throws SqlException {
+        shuffleExpressionsTest(expressions, " AND ", expected, 0);
+    }
+
+    private void shuffleExpressionsTest(String[] expressions, String separator, String expected, int k) throws SqlException {
+        for (int i = k; i < expressions.length; i++) {
+            swap(expressions, i, k);
+            shuffleExpressionsTest(expressions, separator, expected, k + 1);
+            swap(expressions, k, i);
+        }
+        if (k == expressions.length - 1) {
+            sink.clear();
+            for (int j = 0; j < expressions.length; j++) {
+                sink.put(expressions[j]).put(separator);
+            }
+            sink.clear(sink.length() - separator.length());
+            String expression = sink.toString();
+            Assert.assertEquals(
+                    expected,
+                    modelOf(expression)
+                            .buildIntervalModel()
+                            .calculateIntervals(sqlExecutionContext)
+                            .toString()
+            );
+        }
+    }
+
+    private static final void swap(String[] arr, int i, int j) {
+        String tmp = arr[i];
+        arr[i] = arr[j];
+        arr[j] = tmp;
+    }
+
     private void assertFilter(IntrinsicModel m, CharSequence expected) throws SqlException {
         Assert.assertNotNull(m.filter);
         TestUtils.assertEquals(expected, toRpn(m.filter));