Merge pull request #9387 from taosdata/szhou/feature/diff-ignore-negative

diff support ignore negative

src/client/src/tscSQLParser.c

@@ -2675,6 +2675,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
   const char* msg16 = "elapsed duration should be greater than or equal to database precision";
   const char* msg17 = "elapsed/twa should not be used in nested query if inner query has group by clause";
   const char* msg18 = "the second parameter is not an integer";
+  const char* msg19 = "the second paramter of diff should be 0 or 1";
+
 
   switch (functionId) {
     case TSDB_FUNC_COUNT: {
@@ -2773,9 +2775,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
       // no parameters or more than one parameter for function
       if (pItem->pNode->Expr.paramList == NULL ||
-          (functionId != TSDB_FUNC_LEASTSQR && functionId != TSDB_FUNC_DERIVATIVE && functionId != TSDB_FUNC_ELAPSED && numOfParams != 1) ||
+          (functionId != TSDB_FUNC_LEASTSQR && functionId != TSDB_FUNC_DERIVATIVE && functionId != TSDB_FUNC_ELAPSED && functionId != TSDB_FUNC_DIFF
+             && numOfParams != 1) ||
           ((functionId == TSDB_FUNC_LEASTSQR || functionId == TSDB_FUNC_DERIVATIVE) && numOfParams != 3) ||
-          (functionId == TSDB_FUNC_ELAPSED && numOfParams != 1 && numOfParams != 2)) {
+          (functionId == TSDB_FUNC_ELAPSED && numOfParams != 1 && numOfParams != 2) ||
+          (functionId == TSDB_FUNC_DIFF && numOfParams != 1 && numOfParams != 2)) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
@@ -2908,6 +2912,21 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
           tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, LONG_BYTES);
         }
+      } else if (functionId == TSDB_FUNC_DIFF) {
+        char val[8] = {0};
+        if (numOfParams == 2) {
+          tVariant* variantDiff = &pParamElem[1].pNode->value;
+          if (variantDiff->nType != TSDB_DATA_TYPE_BIGINT) {
+            return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+          }
+          tVariantDump(variantDiff, val, TSDB_DATA_TYPE_BIGINT, true);
+
+          int64_t ignoreNegative = GET_INT64_VAL(val);
+          if (ignoreNegative != 0 && ignoreNegative != 1) {
+            return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg19);
+          }
+        }
+        tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t));
       }
 
       SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex);

src/query/src/qAggMain.c

@@ -204,6 +204,7 @@ typedef struct SElapsedInfo {
 
 typedef struct {
   bool valueAssigned;
+  bool ignoreNegative;
   union {
     int64_t i64Prev;
     double d64Prev;
@@ -3064,6 +3065,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResIn
   SDiffFuncInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
   pDiffInfo->valueAssigned = false;
   pDiffInfo->i64Prev = 0;
+  pDiffInfo->ignoreNegative = (pCtx->param[0].i64 == 1) ? true : false;
   return true;
 }
 
@@ -3296,6 +3298,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {
           *pOutput = (int32_t)(pData[i] - pDiffInfo->i64Prev);  // direct previous may be null
@@ -3318,6 +3323,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {
           *pOutput = pData[i] - pDiffInfo->i64Prev;  // direct previous may be null
@@ -3340,6 +3348,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {  // initial value is not set yet
           SET_DOUBLE_VAL(pOutput, pData[i] - pDiffInfo->d64Prev);  // direct previous may be null
@@ -3362,6 +3373,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {  // initial value is not set yet
           *pOutput = (float)(pData[i] - pDiffInfo->d64Prev);  // direct previous may be null
@@ -3384,6 +3398,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {  // initial value is not set yet
           *pOutput = (int16_t)(pData[i] - pDiffInfo->i64Prev);  // direct previous may be null
@@ -3407,6 +3424,9 @@ static void diff_function(SQLFunctionCtx *pCtx) {
         if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) {
           continue;
         }
+        if ((pDiffInfo->ignoreNegative) && (pData[i] < 0)) {
+          continue;
+        }
 
         if (pDiffInfo->valueAssigned) {  // initial value is not set yet
           *pOutput = (int8_t)(pData[i] - pDiffInfo->i64Prev);  // direct previous may be null

tests/develop-test/2-query/diff_ignore_negative.py

+import sys 
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1<shenglian zhou>: [TD-11204]Difference improvement that can ignore negative 
+        ''' 
+        return
+    
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self._conn = conn
+
+    def run(self):
+        print("running {}".format(__file__))
+        tdSql.execute("drop database if exists diffneg")
+        tdSql.execute("create database if not exists diffneg")
+        tdSql.execute('use diffneg')
+        tdSql.execute('create table st(ts timestamp, c1 int, c2 float) tags(t int);')
+
+        tdSql.execute('create table ct1 using st tags(1);')
+
+        tdSql.execute('create table ct2 using st tags(2);')
+
+        tdSql.execute('insert into ct1 values(1642662622000, 1, 2)(1642662622001, 2, 4)(1642662622003, -4, -6)(1642662622004, 4, 8);')
+
+        tdSql.execute('insert into ct2 values(1642662622001, 11, 22)(1642662622002, 22, 44)(1642662622003, -44, -66) (1642662622004, 44, 88);')
+
+        tdSql.query('select diff(c1) from st group by tbname')
+        tdSql.checkRows(6)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(1, 1, -6)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(2, 1, 8)
+        tdSql.checkData(2, 2, 'ct1')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(3, 1, 11)
+        tdSql.checkData(3, 2, 'ct2')
+        tdSql.checkData(4, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(4, 1, -66)
+        tdSql.checkData(4, 2, 'ct2')
+        tdSql.checkData(5, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(5, 1, 88)
+        tdSql.checkData(5, 2, 'ct2')
+
+        tdSql.query('select diff(c1,0) from st group by tbname')
+        tdSql.checkRows(6)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(1, 1, -6)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(2, 1, 8)
+        tdSql.checkData(2, 2, 'ct1')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(3, 1, 11)
+        tdSql.checkData(3, 2, 'ct2')
+        tdSql.checkData(4, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(4, 1, -66)
+        tdSql.checkData(4, 2, 'ct2')
+        tdSql.checkData(5, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(5, 1, 88)
+        tdSql.checkData(5, 2, 'ct2')
+
+        tdSql.query('select diff(c1,1) from st group by tbname')
+        tdSql.checkRows(4)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(1, 1, 2)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(2, 1, 11)
+        tdSql.checkData(2, 2, 'ct2')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(3, 1, 22)
+        tdSql.checkData(3, 2, 'ct2')
+
+        tdSql.query('select diff(c2) from st group by tbname')
+        tdSql.checkRows(6)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 2.0)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(1, 1, -10.0)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(2, 1, 14.0)
+        tdSql.checkData(2, 2, 'ct1')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(3, 1, 22.0)
+        tdSql.checkData(3, 2, 'ct2')
+        tdSql.checkData(4, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(4, 1, -110.0)
+        tdSql.checkData(4, 2, 'ct2')
+        tdSql.checkData(5, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(5, 1, 154.0)
+        tdSql.checkData(5, 2, 'ct2')
+
+        tdSql.query('select diff(c2,0) from st group by tbname')
+        tdSql.checkRows(6)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 2.0)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(1, 1, -10.0)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(2, 1, 14.0)
+        tdSql.checkData(2, 2, 'ct1')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(3, 1, 22.0)
+        tdSql.checkData(3, 2, 'ct2')
+        tdSql.checkData(4, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 3000))
+        tdSql.checkData(4, 1, -110.0)
+        tdSql.checkData(4, 2, 'ct2')
+        tdSql.checkData(5, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(5, 1, 154.0)
+        tdSql.checkData(5, 2, 'ct2')
+
+        tdSql.query('select diff(c2,1) from st group by tbname')
+        tdSql.checkRows(4)
+        tdSql.checkData(0, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 1000))
+        tdSql.checkData(0, 1, 2.0)
+        tdSql.checkData(0, 2, 'ct1')
+        tdSql.checkData(1, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(1, 1, 4.0)
+        tdSql.checkData(1, 2, 'ct1')
+        tdSql.checkData(2, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 2000))
+        tdSql.checkData(2, 1, 22.0)
+        tdSql.checkData(2, 2, 'ct2')
+        tdSql.checkData(3, 0, datetime.datetime(2022, 1, 20, 15, 10, 22, 4000))
+        tdSql.checkData(3, 1, 44.0)
+        tdSql.checkData(3, 2, 'ct2')
+
+        tdSql.execute('drop database diffneg')
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/develop-test/fulltest-query.sh

@@ -7,3 +7,5 @@ python3 ./test.py -f 2-query/ts_2016.py
 python3 ./test.py -f 2-query/function_mavg.py
 python3 ./test.py -f 2-query/escape.py
 python3 ./test.py -f 2-query/func_compare.py
+python3 ./test.py -f 2-query/diff_ignore_negative.py
+python3 ./test.py -f 2-query/diff_funcs.py