diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 94ec6fde0fef0c0caf8e4c167bd738a306bcd346..a75e712f387cd684486147449adcdfe357a54d67 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -131,7 +131,7 @@ uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] int32_t tsHttpCacheSessions = 1000; int32_t tsHttpSessionExpire = 36000; int32_t tsHttpMaxThreads = 2; -int32_t tsHttpEnableCompress = 0; +int32_t tsHttpEnableCompress = 1; int32_t tsHttpEnableRecordSql = 0; int32_t tsTelegrafUseFieldNum = 0; diff --git a/src/plugins/http/inc/httpJson.h b/src/plugins/http/inc/httpJson.h index ac0a632137256af96de8503775e5a34fc81b1f1c..fcb74253b99c0cd8921d981e3ba7a4a027bf2b31 100644 --- a/src/plugins/http/inc/httpJson.h +++ b/src/plugins/http/inc/httpJson.h @@ -19,7 +19,7 @@ #include #include -#define JSON_BUFFER_SIZE 10240 +#define JSON_BUFFER_SIZE 16384 struct HttpContext; enum { JsonNumber, JsonString, JsonBoolean, JsonArray, JsonObject, JsonNull }; diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index e9a8947df2650a590bc7c7cd0f12c5d6c8a4dd20..7600fb3e43af236c59a5afea0a4ad981019b2b32 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -52,12 +52,12 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t } if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d, times:%d", pContext, pContext->fd, errno, countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d, connect already closed", pContext, pContext->fd, errno); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno)); break; } else { countWait = 0; @@ -131,14 +131,14 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { - httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext, + httpDebug("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext, buf->pContext->fd, isTheLast, buf->buf); - return 0; // there is no data to dump. + remain = 0; // there is no data to dump. } } else { httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); - return 0; + remain = 0; } } diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 2c8879f8804828fdd9de268ab1b6348df3a6b77c..39168ee96dc0ef83dc329614f63543d2c6c7fdfa 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -406,15 +406,21 @@ int32_t httpGzipCompressInit(HttpContext *pContext) { int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { int32_t err = 0; + int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out); pContext->gzipStream.next_in = (Bytef *) srcData; pContext->gzipStream.avail_in = (uLong) nSrcData; pContext->gzipStream.next_out = (Bytef *) destData; pContext->gzipStream.avail_out = (uLong) (*nDestData); - while (pContext->gzipStream.avail_in != 0 && pContext->gzipStream.total_out < (uLong) (*nDestData)) { + while (pContext->gzipStream.avail_in != 0) { if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { return -1; } + + int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen; + if (cacheLen >= *nDestData) { + return -2; + } } if (pContext->gzipStream.avail_in != 0) { @@ -427,16 +433,16 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, break; } if (err != Z_OK) { - return -2; + return -3; } } if (deflateEnd(&pContext->gzipStream) != Z_OK) { - return -3; + return -4; } } - *nDestData = (int32_t) (pContext->gzipStream.total_out); + *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen; return 0; } diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim index 0289e337a66f7fbb52752ef51f00432d8f1c3ff6..9c77567abb9874459a2363a93cb62cb57cf02b1a 100644 --- a/tests/script/general/http/gzip.sim +++ b/tests/script/general/http/gzip.sim @@ -3,7 +3,7 @@ sleep 3000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c maxSQLLength -v 7340032 +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 system sh/exec.sh -n dnode1 -s start sleep 3000 @@ -18,10 +18,22 @@ sql use d1 sql create table table_rest (ts timestamp, i int) print sql length is 270KB restful d1 table_rest 1591072800 10000 gzip +restful d1 table_rest 1591172800 10000 gzip +restful d1 table_rest 1591272800 10000 gzip +restful d1 table_rest 1591372800 10000 gzip +restful d1 table_rest 1591472800 10000 gzip +restful d1 table_rest 1591572800 10000 gzip +restful d1 table_rest 1591672800 10000 gzip +restful d1 table_rest 1591772800 10000 gzip +restful d1 table_rest 1591872800 10000 gzip +restful d1 table_rest 1591972800 10000 gzip + sql select * from table_rest; print rows: $rows -if $rows != 10000 then +if $rows != 100000 then return -1 endi +system curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql --compressed + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index adb22aa265a34b95f61f881984dedb007805a268..9e42adfea9db15e91f836a57ea2e18ebb8b2c5ac 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -79,6 +79,7 @@ cd ../../../debug; make ./test.sh -f general/http/autocreate.sim ./test.sh -f general/http/chunked.sim +./test.sh -f general/http/gzip.sim ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim