[IOTDB-1321] Filter RowRecord automatically if any column in it is null or all...

[IOTDB-1321] Filter RowRecord automatically if any column in it is null or all columns are null (#3153)
Co-authored-by: NAlima777 <wxw19981014@gmail.com>

antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4

@@ -251,17 +251,22 @@ specialClause
     | orderByTimeClause specialLimit? #orderByTimeStatement
     | groupByTimeClause orderByTimeClause? specialLimit? #groupByTimeStatement
     | groupByFillClause orderByTimeClause? specialLimit? #groupByFillStatement
-    | fillClause slimitClause? alignByDeviceClauseOrDisableAlign? #fillStatement
-    | alignByDeviceClauseOrDisableAlign #alignByDeviceStatementOrDisableAlignInSpecialClause
     | groupByLevelClause orderByTimeClause? specialLimit? #groupByLevelStatement
+    | fillClause slimitClause? alignByDeviceClauseOrDisableAlign? #fillStatement
     ;
 
 specialLimit
     : limitClause slimitClause? alignByDeviceClauseOrDisableAlign? #limitStatement
     | slimitClause limitClause? alignByDeviceClauseOrDisableAlign? #slimitStatement
-    | alignByDeviceClauseOrDisableAlign #alignByDeviceClauseOrDisableAlignInSpecialLimit
+    | withoutNullClause limitClause? slimitClause? alignByDeviceClauseOrDisableAlign? #withoutNullStatement
+    | alignByDeviceClauseOrDisableAlign #alignByDeviceClauseOrDisableAlignStatement
     ;
 
+withoutNullClause
+    : WITHOUT NULL (ALL | ANY)
+    ;
+
+
 orderByTimeClause
     : ORDER BY TIME (DESC | ASC)?
     ;
@@ -1312,6 +1317,14 @@ NULL
     : N U L L
     ;
 
+WITHOUT
+    : W I T H O U T
+    ;
+
+ANY
+    : A N Y
+    ;
+
 //============================
 // End of the keywords list
 //============================

docs/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md

@@ -1308,6 +1308,20 @@ Total line number = 10
 It costs 0.009s
 ```
 
+#### Null Value Control over Query Results
+
+* IoTDB will join all the sensor value by its time, and if some sensors don't have values in that timestamp, we will fill it with null. In some analysis scenarios, we only need the row if all the columns of it have value.
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ANY
+```
+
+* In group by query, we will fill null for any group by interval if the columns don't have values in that group by interval. However, if all columns in that group by interval are null, maybe users don't need that RowRecord, so we can use `WITHOUT NULL ALL` to filter that row.
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ALL
+```
+
 ### Use Alias
 
 Since the unique data model of IoTDB, lots of additional information like device will be carried before each sensor. Sometimes, we want to query just one specific device, then these prefix information show frequently will be redundant in this situation, influencing the analysis of result set. At this time, we can use `AS` function provided by IoTDB, assign an alias to time series selected in query.  

docs/zh/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md

@@ -297,6 +297,20 @@ It costs 0.018s
 
 更多语法请参照[SQL REFERENCE](../Appendix/SQL-Reference.md)。
 
+#### 结果空值过滤
+
+* 如果结果集中，任意一列为null，则过滤掉该行；即获得的结果集不包含任何空值
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ANY
+```
+
+* 在降采样查询中，如果结果集的某一行所有列都为null，则过滤掉该行；即获得的结果集不包含所有值都为null的行
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ALL
+```
+
 
 #### 聚合函数
 

server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java

@@ -542,6 +542,8 @@ public class PlanExecutor implements IPlanExecutor {
     }
     queryDataSet.setRowLimit(queryPlan.getRowLimit());
     queryDataSet.setRowOffset(queryPlan.getRowOffset());
+    queryDataSet.setWithoutAllNull(queryPlan.isWithoutAllNull());
+    queryDataSet.setWithoutAnyNull(queryPlan.isWithoutAnyNull());
     return queryDataSet;
   }
 

server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java

@@ -62,6 +62,12 @@ public class QueryOperator extends SFWOperator {
 
   private IndexType indexType;
 
+  // if true, we don't need the row whose any column is null
+  private boolean withoutAnyNull;
+
+  // if true, we don't need the row whose all columns are null
+  private boolean withoutAllNull;
+
   public QueryOperator(int tokenIntType) {
     super(tokenIntType);
     operatorType = Operator.OperatorType.QUERY;
@@ -250,4 +256,20 @@ public class QueryOperator extends SFWOperator {
   public void setAscending(boolean ascending) {
     this.ascending = ascending;
   }
+
+  public boolean isWithoutAnyNull() {
+    return withoutAnyNull;
+  }
+
+  public void setWithoutAnyNull(boolean withoutAnyNull) {
+    this.withoutAnyNull = withoutAnyNull;
+  }
+
+  public boolean isWithoutAllNull() {
+    return withoutAllNull;
+  }
+
+  public void setWithoutAllNull(boolean withoutAllNull) {
+    this.withoutAllNull = withoutAllNull;
+  }
 }

server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java

@@ -46,6 +46,12 @@ public abstract class QueryPlan extends PhysicalPlan {
 
   private boolean enableRedirect = false;
 
+  // if true, we don't need the row whose any column is null
+  private boolean withoutAnyNull;
+
+  // if true, we don't need the row whose all columns are null
+  private boolean withoutAllNull;
+
   public QueryPlan() {
     super(true);
     setOperatorType(Operator.OperatorType.QUERY);
@@ -150,4 +156,20 @@ public abstract class QueryPlan extends PhysicalPlan {
   public void setVectorPathToIndex(Map<String, Integer> vectorPathToIndex) {
     this.vectorPathToIndex = vectorPathToIndex;
   }
+
+  public boolean isWithoutAnyNull() {
+    return withoutAnyNull;
+  }
+
+  public void setWithoutAnyNull(boolean withoutAnyNull) {
+    this.withoutAnyNull = withoutAnyNull;
+  }
+
+  public boolean isWithoutAllNull() {
+    return withoutAllNull;
+  }
+
+  public void setWithoutAllNull(boolean withoutAllNull) {
+    this.withoutAllNull = withoutAllNull;
+  }
 }

server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java

@@ -81,8 +81,8 @@ import org.apache.iotdb.db.qp.physical.crud.GroupByTimePlan;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AggregationCallContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AggregationElementContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AliasClauseContext;
-import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignInSpecialLimitContext;
-import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceStatementOrDisableAlignInSpecialClauseContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignStatementContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterClauseContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterTimeseriesContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterUserContext;
@@ -216,6 +216,7 @@ import org.apache.iotdb.db.qp.sql.SqlBaseParser.UdfAttributeContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.UdfCallContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.UnsetTTLStatementContext;
 import org.apache.iotdb.db.qp.sql.SqlBaseParser.WhereClauseContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.WithoutNullStatementContext;
 import org.apache.iotdb.db.qp.utils.DatetimeUtils;
 import org.apache.iotdb.db.query.executor.fill.IFill;
 import org.apache.iotdb.db.query.executor.fill.LinearFill;
@@ -1132,11 +1133,7 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
       parseSlimitClause(ctx.slimitClause(), queryOp);
     }
     if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
-      if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
-        parseAlignByDeviceClause(queryOp);
-      } else {
-        parseDisableAlign(queryOp);
-      }
+      parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
     }
     return queryOp;
   }
@@ -1148,23 +1145,42 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
       parseLimitClause(ctx.limitClause(), queryOp);
     }
     if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
-      if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
-        parseAlignByDeviceClause(queryOp);
-      } else {
-        parseDisableAlign(queryOp);
-      }
+      parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
     }
     return queryOp;
   }
 
   @Override
-  public Operator visitAlignByDeviceClauseOrDisableAlignInSpecialLimit(
-      AlignByDeviceClauseOrDisableAlignInSpecialLimitContext ctx) {
-    if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
+  public Operator visitAlignByDeviceClauseOrDisableAlignStatement(
+      AlignByDeviceClauseOrDisableAlignStatementContext ctx) {
+    parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
+    return queryOp;
+  }
+
+  private void parseAlignByDeviceClauseOrDisableAlign(
+      AlignByDeviceClauseOrDisableAlignContext ctx) {
+    if (ctx.alignByDeviceClause() != null) {
       parseAlignByDeviceClause(queryOp);
     } else {
       parseDisableAlign(queryOp);
     }
+  }
+
+  @Override
+  public Operator visitWithoutNullStatement(WithoutNullStatementContext ctx) {
+    if (ctx.withoutNullClause().WITHOUT() != null) {
+      queryOp.setWithoutAllNull(ctx.withoutNullClause().ALL() != null);
+      queryOp.setWithoutAnyNull(ctx.withoutNullClause().ANY() != null);
+    }
+    if (ctx.limitClause() != null) {
+      parseLimitClause(ctx.limitClause(), queryOp);
+    }
+    if (ctx.slimitClause() != null) {
+      parseSlimitClause(ctx.slimitClause(), queryOp);
+    }
+    if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
+      parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
+    }
     return queryOp;
   }
 
@@ -1208,22 +1224,7 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
       parseSlimitClause(ctx.slimitClause(), queryOp);
     }
     if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
-      if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
-        parseAlignByDeviceClause(queryOp);
-      } else {
-        parseDisableAlign(queryOp);
-      }
-    }
-    return queryOp;
-  }
-
-  @Override
-  public Operator visitAlignByDeviceStatementOrDisableAlignInSpecialClause(
-      AlignByDeviceStatementOrDisableAlignInSpecialClauseContext ctx) {
-    if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
-      parseAlignByDeviceClause(queryOp);
-    } else {
-      parseDisableAlign(queryOp);
+      parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
     }
     return queryOp;
   }

server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java

@@ -540,7 +540,7 @@ public class PhysicalGenerator {
 
   @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
   private PhysicalPlan transformQuery(QueryOperator queryOperator) throws QueryProcessException {
-    QueryPlan queryPlan = null;
+    QueryPlan queryPlan;
 
     if (queryOperator.hasAggregation()) {
       queryPlan = new AggPhysicalPlanRule().transform(queryOperator);
@@ -587,6 +587,9 @@ public class PhysicalGenerator {
       }
     }
 
+    queryPlan.setWithoutAllNull(queryOperator.isWithoutAllNull());
+    queryPlan.setWithoutAnyNull(queryOperator.isWithoutAnyNull());
+
     if (queryOperator.getIndexType() != null) {
       if (queryPlan instanceof QueryIndexPlan) {
         ((QueryIndexPlan) queryPlan).setIndexType(queryOperator.getIndexType());

server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithoutValueFilter.java

@@ -57,7 +57,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
 
     private final ManagedSeriesReader reader;
     private final String pathName;
-    private BlockingQueue<BatchData> blockingQueue;
+    private final BlockingQueue<BatchData> blockingQueue;
 
     public ReadTask(
         ManagedSeriesReader reader, BlockingQueue<BatchData> blockingQueue, String pathName) {
@@ -256,6 +256,10 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
 
       long minTime = timeHeap.pollFirst();
 
+      if (withoutAnyNull && filterRowRecord(seriesNum, minTime)) {
+        continue;
+      }
+
       if (rowOffset == 0) {
         timeBAOS.write(BytesUtils.longToBytes(minTime));
       }
@@ -388,22 +392,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
             }
           }
 
-          // move next
-          cachedBatchDataArray[seriesIndex].next();
-
-          // check the interrupted status of query before taking next batch
-          QueryTimeManager.checkQueryAlive(queryId);
-
-          // get next batch if current batch is empty and still have remaining batch data in queue
-          if (!cachedBatchDataArray[seriesIndex].hasCurrent()
-              && !noMoreDataInQueueArray[seriesIndex]) {
-            fillCache(seriesIndex);
-          }
-
-          // try to put the next timestamp into the heap
-          if (cachedBatchDataArray[seriesIndex].hasCurrent()) {
-            timeHeap.add(cachedBatchDataArray[seriesIndex].currentTime());
-          }
+          prepareForNext(seriesIndex);
         }
       }
 
@@ -465,6 +454,60 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
     return tsQueryDataSet;
   }
 
+  /** if any column in the row record is null, we filter it. */
+  private boolean filterRowRecord(int seriesNum, long minTime)
+      throws IOException, InterruptedException {
+    boolean hasNull = false;
+    for (int seriesIndex = 0; seriesIndex < seriesNum; seriesIndex++) {
+      if (cachedBatchDataArray[seriesIndex] == null
+          || !cachedBatchDataArray[seriesIndex].hasCurrent()
+          || cachedBatchDataArray[seriesIndex].currentTime() != minTime) {
+        hasNull = true;
+      } else {
+        if (TSDataType.VECTOR == cachedBatchDataArray[seriesIndex].getDataType()) {
+          for (TsPrimitiveType primitiveVal : cachedBatchDataArray[seriesIndex].getVector()) {
+            if (primitiveVal == null) {
+              hasNull = true;
+              break;
+            }
+          }
+        }
+      }
+      if (hasNull) {
+        break;
+      }
+    }
+    if (hasNull) {
+      for (int seriesIndex = 0; seriesIndex < seriesNum; seriesIndex++) {
+        if (cachedBatchDataArray[seriesIndex] != null
+            && cachedBatchDataArray[seriesIndex].hasCurrent()
+            && cachedBatchDataArray[seriesIndex].currentTime() == minTime) {
+          prepareForNext(seriesIndex);
+        }
+      }
+      return true;
+    }
+    return false;
+  }
+
+  private void prepareForNext(int seriesIndex) throws IOException, InterruptedException {
+    // move next
+    cachedBatchDataArray[seriesIndex].next();
+
+    // check the interrupted status of query before taking next batch
+    QueryTimeManager.checkQueryAlive(queryId);
+
+    // get next batch if current batch is empty and still have remaining batch data in queue
+    if (!cachedBatchDataArray[seriesIndex].hasCurrent() && !noMoreDataInQueueArray[seriesIndex]) {
+      fillCache(seriesIndex);
+    }
+
+    // try to put the next timestamp into the heap
+    if (cachedBatchDataArray[seriesIndex].hasCurrent()) {
+      timeHeap.add(cachedBatchDataArray[seriesIndex].currentTime());
+    }
+  }
+
   protected void fillCache(int seriesIndex) throws IOException, InterruptedException {
     BatchData batchData = blockingQueueArray[seriesIndex].take();
     // no more batch data in this time series queue
@@ -565,7 +608,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
   public Object[] nextRowInObjects() throws IOException {
     int seriesNumber = seriesReaderList.size();
 
-    Long minTime = timeHeap.pollFirst();
+    long minTime = timeHeap.pollFirst();
     Object[] rowInObjects = new Object[seriesNumber + 1];
     rowInObjects[seriesNumber] = minTime;
 

server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java

@@ -43,18 +43,6 @@ public class QueryDataSetUtils {
 
   private QueryDataSetUtils() {}
 
-  /**
-   * convert query data set by fetch size.
-   *
-   * @param queryDataSet -query dataset
-   * @param fetchSize -fetch size
-   * @return -convert query dataset
-   */
-  public static TSQueryDataSet convertQueryDataSetByFetchSize(
-      QueryDataSet queryDataSet, int fetchSize) throws IOException {
-    return convertQueryDataSetByFetchSize(queryDataSet, fetchSize, null);
-  }
-
   @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
   public static TSQueryDataSet convertQueryDataSetByFetchSize(
       QueryDataSet queryDataSet, int fetchSize, WatermarkEncoder watermarkEncoder)
@@ -79,6 +67,12 @@ public class QueryDataSetUtils {
     for (int i = 0; i < fetchSize; i++) {
       if (queryDataSet.hasNext()) {
         RowRecord rowRecord = queryDataSet.next();
+        // filter rows whose columns are null according to the rule
+        if ((queryDataSet.isWithoutAllNull() && rowRecord.isAllNull())
+            || (queryDataSet.isWithoutAnyNull() && rowRecord.hasNullField())) {
+          i--;
+          continue;
+        }
         if (watermarkEncoder != null) {
           rowRecord = watermarkEncoder.encodeRecord(rowRecord);
         }

server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAllNullIT.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IoTDBWithoutAllNullIT {
+
+  private static final String[] dataSet =
+      new String[] {
+        "SET STORAGE GROUP TO root.testWithoutAllNull",
+        "CREATE TIMESERIES root.testWithoutAllNull.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+        "CREATE TIMESERIES root.testWithoutAllNull.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+        "CREATE TIMESERIES root.testWithoutAllNull.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1) " + "values(6, 26)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s2) " + "values(7, false)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(9, 29, true)",
+        "flush",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(10, 20, true)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2,s3) "
+            + "values(11, 21, false, 11.1)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(12, 22, true)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2,s3) "
+            + "values(13, 23, false, 33.3)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s3) " + "values(14, 24, 44.4)",
+        "INSERT INTO root.testWithoutAllNull.d1(timestamp,s2,s3) " + "values(15, true, 55.5)",
+      };
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    EnvironmentUtils.closeStatMonitor();
+    EnvironmentUtils.envSetUp();
+    IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(1000);
+    Class.forName(Config.JDBC_DRIVER_NAME);
+    prepareData();
+  }
+
+  private static void prepareData() {
+    try (Connection connection =
+            DriverManager.getConnection(
+                Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+
+      for (String sql : dataSet) {
+        statement.execute(sql);
+      }
+
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
+    EnvironmentUtils.cleanEnv();
+  }
+
+  @Test
+  public void withoutAllNullTest1() {
+    String[] retArray1 = new String[] {"6,20,true,null", "11,24,true,55.5"};
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute(
+              "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ALL");
+
+      assertTrue(hasResultSet);
+      int cnt;
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+          assertEquals(retArray1[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray1.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void withoutAllNullTest2() {
+    String[] retArray1 = new String[] {"11,24,true,55.5"};
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute(
+              "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ALL limit 1 offset 1");
+
+      assertTrue(hasResultSet);
+      int cnt;
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+          assertEquals(retArray1[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray1.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void withoutAllNullTest3() {
+    String[] retArray1 = new String[] {"11,24,true,55.5"};
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute(
+              "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ANY");
+
+      assertTrue(hasResultSet);
+      int cnt;
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+                  + ","
+                  + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+          assertEquals(retArray1[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray1.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+}

server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAnyNullIT.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IoTDBWithoutAnyNullIT {
+
+  private static final String[] dataSet =
+      new String[] {
+        "SET STORAGE GROUP TO root.testWithoutAnyNull",
+        "CREATE TIMESERIES root.testWithoutAnyNull.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+        "CREATE TIMESERIES root.testWithoutAnyNull.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+        "CREATE TIMESERIES root.testWithoutAnyNull.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+            + "values(1, 21, false, 11.1)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2) " + "values(2, 22, true)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+            + "values(3, 23, false, 33.3)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s3) " + "values(4, 24, 44.4)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2,s3) " + "values(5, true, 55.5)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1) " + "values(6, 26)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2) " + "values(7, false)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s3) " + "values(8, 88.8)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) " + "values(9, 29, true, 99.9)",
+        "flush",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+            + "values(10, 20, true, 10.0)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+            + "values(11, 21, false, 11.1)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2) " + "values(12, 22, true)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+            + "values(13, 23, false, 33.3)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s3) " + "values(14, 24, 44.4)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2,s3) " + "values(15, true, 55.5)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1) " + "values(16, 26)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2) " + "values(17, false)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s3) " + "values(18, 88.8)",
+        "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) " + "values(19, 29, true, 99.9)"
+      };
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    EnvironmentUtils.closeStatMonitor();
+    EnvironmentUtils.envSetUp();
+    IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(1000);
+    Class.forName(Config.JDBC_DRIVER_NAME);
+    prepareData();
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
+    EnvironmentUtils.cleanEnv();
+  }
+
+  @Test
+  public void withoutAnyNullTest1() {
+    String[] retArray1 =
+        new String[] {
+          "1,21,false,11.1",
+          "3,23,false,33.3",
+          "9,29,true,99.9",
+          "10,20,true,10.0",
+          "11,21,false,11.1",
+          "13,23,false,33.3",
+          "19,29,true,99.9"
+        };
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute("select * from root.testWithoutAnyNull.d1 WITHOUT NULL ANY");
+
+      assertTrue(hasResultSet);
+      int cnt;
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+          assertEquals(retArray1[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray1.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void withoutAnyNullTest2() {
+    String[] retArray =
+        new String[] {"10,20,true,10.0", "11,21,false,11.1", "13,23,false,33.3", "19,29,true,99.9"};
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute(
+              "select * from root.testWithoutAnyNull.d1 WHERE time >= 10 WITHOUT NULL ANY");
+
+      int cnt;
+      assertTrue(hasResultSet);
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+          assertEquals(retArray[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void withoutAnyNullTest3() {
+    String[] retArray1 =
+        new String[] {
+          "3,23,false,33.3",
+          "9,29,true,99.9",
+          "10,20,true,10.0",
+          "11,21,false,11.1",
+          "13,23,false,33.3"
+        };
+    try (Connection connection =
+            DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+      boolean hasResultSet =
+          statement.execute(
+              "select * from root.testWithoutAnyNull.d1 WITHOUT NULL ANY limit 5 offset 1");
+
+      assertTrue(hasResultSet);
+      int cnt;
+      try (ResultSet resultSet = statement.getResultSet()) {
+        cnt = 0;
+        while (resultSet.next()) {
+          String ans =
+              resultSet.getString(TIMESTAMP_STR)
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+                  + ","
+                  + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+          assertEquals(retArray1[cnt], ans);
+          cnt++;
+        }
+        assertEquals(retArray1.length, cnt);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
+
+  private static void prepareData() {
+    try (Connection connection =
+            DriverManager.getConnection(
+                Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+        Statement statement = connection.createStatement()) {
+
+      for (String sql : dataSet) {
+        statement.execute(sql);
+      }
+
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}

tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/RowRecord.java

@@ -25,8 +25,13 @@ import java.util.List;
 
 public class RowRecord {
 
-  private long timestamp;
-  private List<Field> fields;
+  private final long timestamp;
+  private final List<Field> fields;
+  /** if any column is null, this field should be set to true; otherwise false */
+  private boolean hasNullField = false;
+
+  /** if any column is not null, this field should be set to false; otherwise true */
+  private boolean allNull = true;
 
   public RowRecord(long timestamp) {
     this.timestamp = timestamp;
@@ -36,14 +41,31 @@ public class RowRecord {
   public RowRecord(long timestamp, List<Field> fields) {
     this.timestamp = timestamp;
     this.fields = fields;
+    for (Field field : fields) {
+      if (field == null || field.getDataType() == null) {
+        hasNullField = true;
+      } else {
+        allNull = false;
+      }
+    }
   }
 
   public void addField(Field f) {
     this.fields.add(f);
+    if (f == null || f.getDataType() == null) {
+      hasNullField = true;
+    } else {
+      allNull = false;
+    }
   }
 
   public void addField(Object value, TSDataType dataType) {
     this.fields.add(Field.getField(value, dataType));
+    if (value == null || dataType == null) {
+      hasNullField = true;
+    } else {
+      allNull = false;
+    }
   }
 
   @Override
@@ -61,19 +83,15 @@ public class RowRecord {
     return timestamp;
   }
 
-  public void setTimestamp(long timestamp) {
-    this.timestamp = timestamp;
-  }
-
   public List<Field> getFields() {
     return fields;
   }
 
-  public void setFields(List<Field> fields) {
-    this.fields = fields;
+  public boolean hasNullField() {
+    return hasNullField;
   }
 
-  public void setField(int index, Field field) {
-    this.fields.set(index, field);
+  public boolean isAllNull() {
+    return allNull;
   }
 }

tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/QueryDataSet.java

@@ -42,6 +42,12 @@ public abstract class QueryDataSet {
    */
   protected EndPoint endPoint = null;
 
+  /** if any column is null, we don't need that row */
+  protected boolean withoutAnyNull;
+
+  /** Only if all columns are null, we don't need that row */
+  protected boolean withoutAllNull;
+
   /** For redirect query. Need keep consistent with EndPoint in rpc.thrift. */
   public static class EndPoint {
     private String ip = null;
@@ -97,7 +103,12 @@ public abstract class QueryDataSet {
     // proceed to the OFFSET row by skipping rows
     while (rowOffset > 0) {
       if (hasNextWithoutConstraint()) {
-        nextWithoutConstraint(); // DO NOT use next()
+        RowRecord rowRecord = nextWithoutConstraint(); // DO NOT use next()
+        // filter rows whose columns are null according to the rule
+        if ((withoutAllNull && rowRecord.isAllNull())
+            || (withoutAnyNull && rowRecord.hasNullField())) {
+          continue;
+        }
         rowOffset--;
       } else {
         return false;
@@ -167,4 +178,20 @@ public abstract class QueryDataSet {
   public void setEndPoint(EndPoint endPoint) {
     this.endPoint = endPoint;
   }
+
+  public boolean isWithoutAnyNull() {
+    return withoutAnyNull;
+  }
+
+  public void setWithoutAnyNull(boolean withoutAnyNull) {
+    this.withoutAnyNull = withoutAnyNull;
+  }
+
+  public boolean isWithoutAllNull() {
+    return withoutAllNull;
+  }
+
+  public void setWithoutAllNull(boolean withoutAllNull) {
+    this.withoutAllNull = withoutAllNull;
+  }
 }