Merge pull request #3631 from taosdata/feature/crash

Feature/crash

src/common/src/tglobal.c

@@ -131,7 +131,7 @@ uint16_t tsHttpPort = 6041;  // only tcp, range tcp[6041]
 int32_t  tsHttpCacheSessions = 1000;
 int32_t  tsHttpSessionExpire = 36000;
 int32_t  tsHttpMaxThreads = 2;
-int32_t  tsHttpEnableCompress = 0;
+int32_t  tsHttpEnableCompress = 1;
 int32_t  tsHttpEnableRecordSql = 0;
 int32_t  tsTelegrafUseFieldNum = 0;
 

src/plugins/http/inc/httpJson.h

@@ -19,7 +19,7 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-#define JSON_BUFFER_SIZE 10240
+#define JSON_BUFFER_SIZE 16384
 struct HttpContext;
 
 enum { JsonNumber, JsonString, JsonBoolean, JsonArray, JsonObject, JsonNull };

src/plugins/http/src/httpJson.c

@@ -52,12 +52,12 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t
     }
 
     if (len < 0) {
-      httpDebug("context:%p, fd:%d, socket write errno:%d, times:%d", pContext, pContext->fd, errno, countWait);
+      httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait);
       if (++countWait > HTTP_WRITE_RETRY_TIMES) break;
       taosMsleep(HTTP_WRITE_WAIT_TIME_MS);
       continue;
     } else if (len == 0) {
-      httpDebug("context:%p, fd:%d, socket write errno:%d, connect already closed", pContext, pContext->fd, errno);
+      httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno));
       break;
     } else {
       countWait = 0;
@@ -131,14 +131,14 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
         httpWriteBufNoTrace(buf->pContext, sLen, len);
         remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen);
       } else {
-        httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
+        httpDebug("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
                   buf->pContext->fd, isTheLast, buf->buf);
-        return 0;  // there is no data to dump.
+        remain = 0;  // there is no data to dump.
       }
     } else {
       httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
                 buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf);
-      return 0;
+      remain = 0;
     }
   }
 

src/plugins/http/src/httpUtil.c

@@ -406,15 +406,21 @@ int32_t httpGzipCompressInit(HttpContext *pContext) {
 
 int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) {
   int32_t err = 0;
+  int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out);
   pContext->gzipStream.next_in = (Bytef *) srcData;
   pContext->gzipStream.avail_in = (uLong) nSrcData;
   pContext->gzipStream.next_out = (Bytef *) destData;
   pContext->gzipStream.avail_out = (uLong) (*nDestData);
 
-  while (pContext->gzipStream.avail_in != 0 && pContext->gzipStream.total_out < (uLong) (*nDestData)) {
+  while (pContext->gzipStream.avail_in != 0) {
     if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) {
       return -1;
     }
+
+    int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen;
+    if (cacheLen >= *nDestData) {
+      return -2;
+    }
   }
 
   if (pContext->gzipStream.avail_in != 0) {
@@ -427,16 +433,16 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData,
         break;
       }
       if (err != Z_OK) {
-        return -2;
+        return -3;
       }
     }
 
     if (deflateEnd(&pContext->gzipStream) != Z_OK) {
-      return -3;
+      return -4;
     }
   }
 
-  *nDestData = (int32_t) (pContext->gzipStream.total_out);
+  *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen;
   return 0;
 }
 

tests/script/general/http/gzip.sim

@@ -3,7 +3,7 @@ sleep 3000
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/cfg.sh -n dnode1 -c http -v 1
-system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032
+system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000
@@ -18,10 +18,22 @@ sql use d1
 sql create table table_rest (ts timestamp, i int) 
 print sql length is 270KB
 restful d1 table_rest 1591072800 10000 gzip
+restful d1 table_rest 1591172800 10000 gzip
+restful d1 table_rest 1591272800 10000 gzip
+restful d1 table_rest 1591372800 10000 gzip
+restful d1 table_rest 1591472800 10000 gzip
+restful d1 table_rest 1591572800 10000 gzip
+restful d1 table_rest 1591672800 10000 gzip
+restful d1 table_rest 1591772800 10000 gzip
+restful d1 table_rest 1591872800 10000 gzip
+restful d1 table_rest 1591972800 10000 gzip
+
 sql select * from table_rest;
 print rows: $rows
-if $rows != 10000 then
+if $rows != 100000 then
   return -1
 endi
 
+system curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql --compressed
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/jenkins/basic.txt

@@ -79,6 +79,7 @@ cd ../../../debug; make
 
 ./test.sh -f general/http/autocreate.sim
 ./test.sh -f general/http/chunked.sim
+./test.sh -f general/http/gzip.sim
 ./test.sh -f general/http/restful.sim
 ./test.sh -f general/http/restful_insert.sim
 ./test.sh -f general/http/restful_limit.sim