before fix thread-safe

deps/SZ/sz/CMakeLists.txt

-add_library (SZ
-  src/ArithmeticCoding.c
-  src/ByteToolkit.c
-  src/CacheTable.c
-  src/callZlib.c
-  src/CompressElement.c
-  src/conf.c
-  src/dataCompression.c
-  src/dictionary.c
-  src/DynamicByteArray.c
-  src/DynamicDoubleArray.c
-  src/DynamicFloatArray.c
-  src/DynamicIntArray.c
-  src/Huffman.c
-  src/iniparser.c
-  src/MultiLevelCacheTable.c
-  src/MultiLevelCacheTableWideInterval.c
-  src/pastri.c
-  src/exafelSZ.c
-  src/rw.c
-  src/rwf.c
-  src/sz.c
-  src/szd_double.c
-  src/szd_double_pwr.c
-  src/szd_double_ts.c
-  src/szd_float.c
-  src/szd_float_pwr.c
-  src/szd_float_ts.c
-  src/szd_int16.c
-  src/szd_int32.c
-  src/szd_int64.c
-  src/szd_int8.c
-  src/sz_double.c
-  src/sz_double_pwr.c
-  src/sz_double_ts.c
-  src/szd_uint16.c
-  src/szd_uint32.c
-  src/szd_uint64.c
-  src/szd_uint8.c
-  src/szf.c
-  src/sz_float.c
-  src/sz_float_pwr.c
-  src/sz_float_ts.c
-  src/sz_int16.c
-  src/sz_int32.c
-  src/sz_int64.c
-  src/sz_int8.c
-  src/sz_omp.c
-  src/sz_uint16.c
-  src/sz_uint32.c
-  src/sz_uint64.c
-  src/sz_uint8.c
-  src/TightDataPointStorageD.c
-  src/TightDataPointStorageF.c
-  src/TightDataPointStorageI.c
-  src/TypeManager.c
-  src/utility.c
-  src/VarSet.c
-  src/sz_stats.c
-)
-
-target_include_directories(SZ 
-  PUBLIC 
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/sz>
-  )
-
-
-target_compile_options(SZ
-	PRIVATE $<$<CONFIG:Debug>:-Wall -Wextra -Wpedantic -Wno-unused-parameter>
-	)
-
-if(BUILD_PASTRI)
-  target_compile_definitions(SZ PUBLIC HAVE_PASTRI)
-endif()
-if(BUILD_TIMECMPR)
-  target_compile_definitions(SZ PUBLIC HAVE_TIMECMPR)
-endif()
-if(BUILD_RANDOMACCESS)
-  target_compile_definitions(SZ PUBLIC HAVE_RANDOMACCESS)
-endif()
-if(BUILD_FORTRAN)
-  enable_language(Fortran)
-  target_sources(SZ PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/rw_interface.F90
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/sz_interface.F90
-  )
-endif()
-if(BUILD_STATS)
-  target_compile_definitions(SZ PUBLIC HAVE_WRITESTATS)
-endif()

deps/SZ/sz/include/defines.h

@@ -24,7 +24,7 @@
 //prediction mode of temporal dimension based compression
 #define SZ_PREVIOUS_VALUE_ESTIMATE 0
 
-#define MIN_NUM_OF_ELEMENTS 20 //if the # elements <= 20, skip the compression
+#define MIN_NUM_OF_ELEMENTS 0 //if the # elements <= 20, skip the compression
 
 #define ABS 0
 #define REL 1

deps/SZ/sz/include/rw.h

@@ -60,7 +60,7 @@ float *readFloatData_systemEndian(char *srcFilePath, size_t *nbEle, int *status)
 void writeByteData(unsigned char *bytes, size_t byteLength, char *tgtFilePath, int *status);
 void writeDoubleData(double *data, size_t nbEle, char *tgtFilePath, int *status);
 void writeFloatData(float *data, size_t nbEle, char *tgtFilePath, int *status);
-void writeData(void *data, int dataType, size_t nbEle, char *tgtFilePath, int *status);
+void writeDataSZ(void *data, int dataType, size_t nbEle, char *tgtFilePath, int *status);
 void writeFloatData_inBytes(float *data, size_t nbEle, char* tgtFilePath, int *status);
 void writeDoubleData_inBytes(double *data, size_t nbEle, char* tgtFilePath, int *status);
 void writeShortData_inBytes(short *states, size_t stateLength, char *tgtFilePath, int *status);

deps/SZ/sz/include/td_sz.h

+
+#ifndef _TD_SZ_H
+#define _TD_SZ_H
+#include "defines.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//
+// compress interface to tdengine return value is count of output with bytes
+//
+int tdszCompress(int type, const char * input, const int nelements, const char * output);
+
+//
+// decompress interface to tdengine return value is count of output with bytes
+//
+int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ----- #ifndef _SZ_H  ----- */

deps/SZ/sz/src/conf.c

@@ -94,7 +94,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
     
     confparams_cpr->plus_bits = 3;
     
-    if(sz_cfgFile == NULL)
+    if(sz_cfgFile == NULL || access(sz_cfgFile, F_OK) != 0)
     {
 		dataEndianType = LITTLE_ENDIAN_DATA;
 		confparams_cpr->sol_ID = SZ;
@@ -116,10 +116,10 @@ int SZ_ReadConf(const char* sz_cfgFile) {
 		else
 			confparams_cpr->gzipMode = 1; //high speed mode
 		
-		confparams_cpr->errorBoundMode = PSNR;
+		confparams_cpr->errorBoundMode = ABS;
 		confparams_cpr->psnr = 90;
-		confparams_cpr->absErrBound = 1E-4;
-		confparams_cpr->relBoundRatio = 1E-4;
+		confparams_cpr->absErrBound = 1E-10;
+		confparams_cpr->relBoundRatio = 1E-10;
 		confparams_cpr->accelerate_pw_rel_compression = 1;
 		
 		confparams_cpr->pw_relBoundRatio = 1E-3;

deps/SZ/sz/src/td_sz.c

+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "td_sz.h"
+#include "CompressElement.h"
+#include "DynamicByteArray.h"
+#include "DynamicIntArray.h"
+#include "TightDataPointStorageD.h"
+#include "TightDataPointStorageF.h"
+#include "zlib.h"
+#include "rw.h"
+#include "Huffman.h"
+#include "conf.h"
+#include "utility.h"
+#include "exafelSZ.h"
+#include "sz.h"
+#include "defines.h"
+
+
+//
+// compress interface to tdengine return value is count of output with bytes
+//
+int tdszCompress(int type, const char * input, const int nelements, const char * output)
+{
+	size_t outSize = 0;
+	void* pOut = SZ_compress(type, (void*)input, &outSize, 0, 0, 0, 0, (size_t)nelements);
+	if(pOut == NULL)
+		return 0;
+	
+	// copy to dest
+	memcpy(output, pOut, outSize);
+	free(pOut);
+    return outSize;
+}
+
+//
+// decompress interface to tdengine return value is count of output with bytes
+//
+int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output)
+{
+	int width = 0;
+	if(type == SZ_FLOAT)
+	    width = sizeof(float);
+	else if(type == SZ_DOUBLE)
+	    width = sizeof(double);	
+	else
+	    return 0;	
+
+	void* pOut = SZ_decompress(type, (void*)input, compressedSize, 0, 0, 0, 0, (size_t)nelements);
+	if(pOut == NULL)
+		return 0;
+
+
+	size_t outSize = nelements * width;
+
+	// copy to dest
+	memcpy(output, pOut, outSize);
+	free(pOut);
+
+    return outSize;
+}

deps/SZ/zlib/CMakeLists.txt

-add_library(ZLIB 
-  ./gzclose.c
-  ./uncompr.c
-  ./trees.c
-  ./gzread.c
-  ./adler32.c
-  ./gzwrite.c
-  ./compress.c
-  ./inftrees.c
-  ./crc32.c
-  ./inffast.c
-  ./zutil.c
-  ./gzlib.c
-  ./infback.c
-  ./inflate.c
-  ./deflate.c
-  )
-
-target_include_directories(ZLIB
-  PUBLIC 
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  )
-
-

deps/SZ/zstd/CMakeLists.txt

-add_library(zstd
-  ./common/entropy_common.c
-  ./common/pool.c
-  ./common/threading.c
-  ./common/debug.c
-  ./common/xxhash.c
-  ./common/fse_decompress.c
-  ./common/zstd_common.c
-  ./common/error_private.c
-  ./compress/zstd_ldm.c
-  ./compress/zstd_lazy.c
-  ./compress/huf_compress.c
-  ./compress/zstd_opt.c
-  ./compress/zstd_double_fast.c
-  ./compress/zstd_compress.c
-  ./compress/zstd_fast.c
-  ./compress/fse_compress.c
-  ./compress/zstdmt_compress.c
-  ./compress/hist.c
-  ./decompress/zstd_decompress.c
-  ./decompress/huf_decompress.c
-  ./deprecated/zbuff_common.c
-  ./deprecated/zbuff_compress.c
-  ./deprecated/zbuff_decompress.c
-  ./legacy/zstd_v05.c
-  ./legacy/zstd_v04.c
-  ./legacy/zstd_v06.c
-  ./legacy/zstd_v07.c
-  ./legacy/zstd_v03.c
-  ./legacy/zstd_v02.c
-  ./legacy/zstd_v01.c
-  ./dictBuilder/cover.c
-  ./dictBuilder/divsufsort.c
-  ./dictBuilder/zdict.c
-  )
-
-target_include_directories(zstd
-  PUBLIC
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/>
-  PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}/common
-    ${CMAKE_CURRENT_SOURCE_DIR}/compress
-    ${CMAKE_CURRENT_SOURCE_DIR}/decompress
-    ${CMAKE_CURRENT_SOURCE_DIR}/deprecated
-    ${CMAKE_CURRENT_SOURCE_DIR}/dictBuilder
-    ${CMAKE_CURRENT_SOURCE_DIR}/dll
-    ${CMAKE_CURRENT_SOURCE_DIR}/legacy
-  )
-
-
-
-

src/util/CMakeLists.txt

@@ -4,6 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/SZ/sz/include)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tutil ${SRC})
 TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic SZ )

src/util/inc/tscompression.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "taosdef.h"
 #include "tutil.h"
 
+
 #define COMP_OVERFLOW_BYTES 2
 #define BITS_PER_BYTE 8
 // Masks
@@ -46,7 +47,13 @@ extern int tsCompressDoubleImp(const char *const input, const int nelements, cha
 extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
 extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
 extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
+// lossy
+int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output);
+int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output);
+int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output);
+int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output);
 
+// init
 bool tsCompressInit();
 void tsCompressExit();
 
@@ -242,25 +249,28 @@ static FORCE_INLINE int tsDecompressDouble(const char *const input, int compress
   }
 }
 
+//
+//  lossy float double
+//
 static FORCE_INLINE int tsCompressFloatLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
                     char algorithm, char *const buffer, int bufferSize) {
-                      return -1;
-                    }
+  return tsCompressFloatLossyImp(input, nelements, output);
+}
 
 static FORCE_INLINE int tsDecompressFloatLossy(const char *const input, int compressedSize, const int nelements, char *const output,
                       int outputSize, char algorithm, char *const buffer, int bufferSize){
-                        return -1;
-                      }
+  return tsDecompressFloatLossyImp(input, compressedSize, nelements, output);
+}
 
 static FORCE_INLINE int tsCompressDoubleLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
                      char algorithm, char *const buffer, int bufferSize){
-                       return -1;
-                     }
+  return tsCompressDoubleLossyImp(input, nelements, output);
+}
 
 static FORCE_INLINE int tsDecompressDoubleLossy(const char *const input, int compressedSize, const int nelements, char *const output,
                        int outputSize, char algorithm, char *const buffer, int bufferSize){
-                         return -1;
-                       }
+  return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output);
+}
 
 
 static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,

src/util/src/tcompression.c

@@ -49,6 +49,7 @@
 
 #include "os.h"
 #include "lz4.h"
+#include "td_sz.h"
 #include "taosdef.h"
 #include "tscompression.h"
 #include "tulog.h"
@@ -894,49 +895,29 @@ int SZ_Init(const char *configFilePath);
 
 
 bool tsCompressInit() {
-  int i = 7*9;
-  i %= 10;
-  if (i > 10000) {
-  tsCompressFloatLossy(NULL, 0, 0, NULL, 0, 0, NULL, 0);
-  tsDecompressFloatLossy(NULL, 0, 0, NULL, 0, 0, NULL, 0);
-  tsCompressDoubleLossy(NULL, 0, 0, NULL, 0, 0, NULL, 0);
-  tsDecompressDoubleLossy(NULL, 0, 0, NULL, 0, 0, NULL, 0); 
-  tsCompressExit();
- 
-  }
-
   SZ_Init("./sz.config");
   return true;
 }
 
 void tsCompressExit(){
-  tsCompressInit();
+  
 }
 
 //
 //   ----------  float double lossy  -----------
 //
-
-/*
-static int  tsCompressFloatLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                    char algorithm, char *const buffer, int bufferSize) {
-
-    return -1;
+int tsCompressFloatLossyImp(const char * input, const int nelements, const char * output){
+  return tdszCompress(SZ_FLOAT, input, nelements, output);
 }
 
-static int tsDecompressFloatLossy(const char *const input, int compressedSize, const int nelements, char *const output,
-                      int outputSize, char algorithm, char *const buffer, int bufferSize) {
-
-    return -1;
+int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, const char * output){
+  return tdszDecompress(SZ_FLOAT, input, compressedSize, nelements, output);
 }
 
-static int tsCompressDoubleLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-    return -1;
+int tsCompressDoubleLossyImp(const char * input, const int nelements, const char * output){
+  return tdszCompress(SZ_DOUBLE, input, nelements, output);
 }
 
-static int tsDecompressDoubleLossy(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-    return -1;
-}
-*/
\ No newline at end of file
+int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output){
+  return tdszDecompress(SZ_DOUBLE, input, compressedSize, nelements, output);
+}
\ No newline at end of file