diff --git a/deps/SZ/sz/include/TightDataPointStorageD.h b/deps/SZ/sz/include/TightDataPointStorageD.h index 188dfe1556cfb2903fa5dc2ddb8a8c8ed768960c..61f05d84677869a97e4f4a823c7b8c23834394db 100644 --- a/deps/SZ/sz/include/TightDataPointStorageD.h +++ b/deps/SZ/sz/include/TightDataPointStorageD.h @@ -10,6 +10,8 @@ #ifndef _TightDataPointStorageD_H #define _TightDataPointStorageD_H +#include "pub.h" + #ifdef __cplusplus extern "C" { #endif @@ -64,7 +66,7 @@ typedef struct TightDataPointStorageD } TightDataPointStorageD; void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self); -int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength); +int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params); void new_TightDataPointStorageD(TightDataPointStorageD **self, size_t dataSeriesLength, size_t exactDataNum, diff --git a/deps/SZ/sz/include/TightDataPointStorageF.h b/deps/SZ/sz/include/TightDataPointStorageF.h index 0b61c02e22941984a06c0a069c78b081812dd842..6a6583e4e6090f55c49d99e7452e5167e0cea5f1 100644 --- a/deps/SZ/sz/include/TightDataPointStorageF.h +++ b/deps/SZ/sz/include/TightDataPointStorageF.h @@ -15,6 +15,7 @@ extern "C" { #endif #include +#include "pub.h" typedef struct TightDataPointStorageF { @@ -66,7 +67,7 @@ typedef struct TightDataPointStorageF } TightDataPointStorageF; void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self); -int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength); +int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params); void new_TightDataPointStorageF(TightDataPointStorageF **self, size_t dataSeriesLength, size_t exactDataNum, diff --git a/deps/SZ/sz/include/TightDataPointStorageI.h b/deps/SZ/sz/include/TightDataPointStorageI.h index 466a753e0e7a751d9a8d544adcaf8ac63fab9131..6a773b96e0f6837c50cb903c0886cfbfeeace5a6 100644 --- a/deps/SZ/sz/include/TightDataPointStorageI.h +++ b/deps/SZ/sz/include/TightDataPointStorageI.h @@ -15,6 +15,7 @@ extern "C" { #endif #include +#include "pub.h" typedef struct TightDataPointStorageI { diff --git a/deps/SZ/sz/include/pub.h b/deps/SZ/sz/include/pub.h new file mode 100644 index 0000000000000000000000000000000000000000..ef849d2d5646a8cf4e9ae3be20cc6c7bf23b7a5c --- /dev/null +++ b/deps/SZ/sz/include/pub.h @@ -0,0 +1,71 @@ +/** + * @file sz.h + * @author Sheng Di + * @date April, 2015 + * @brief Header file for the whole compressor. + * (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +#ifndef _PUB_H +#define _PUB_H + + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/* array meta data and compression parameters for SZ_Init_Params() */ +typedef struct sz_params +{ + int dataType; + unsigned int max_quant_intervals; //max number of quantization intervals for quantization + unsigned int quantization_intervals; + unsigned int maxRangeRadius; + int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri) + int losslessCompressor; + int sampleDistance; //2 bytes + float predThreshold; // 2 bytes + int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression + int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION + int errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR + double absErrBound; //absolute error bound + double relBoundRatio; //value range based relative error bound ratio + double psnr; //PSNR + double normErr; + double pw_relBoundRatio; //point-wise relative error bound + int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated) + int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio + + int protectValueRange; //0 or 1 + float fmin, fmax; + double dmin, dmax; + + int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep + int predictionMode; + + int accelerate_pw_rel_compression; + int plus_bits; + + int randomAccess; + int withRegression; + +} sz_params; + +typedef struct sz_exedata +{ + char optQuantMode; //opt Quantization (0: fixed ; 1: optimized) + int intvCapacity; // the number of intervals for the linear-scaling quantization + int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2) + unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t) +} sz_exedata; + +#ifdef __cplusplus +} +#endif + +#endif /* ----- #ifndef _PUB_H ----- */ diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index 8bf7fa357f5597864646fc284e218525e2bfd8e4..7e241d9951e841b3383bf3408fc7af0d6ed2a9cd 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -14,6 +14,7 @@ #include #include /* For gettimeofday(), in microseconds */ #include /* For time(), in seconds */ +#include "pub.h" #include "CompressElement.h" #include "DynamicByteArray.h" #include "DynamicIntArray.h" @@ -158,42 +159,6 @@ typedef union lfloat unsigned char byte[4]; } lfloat; -/* array meta data and compression parameters for SZ_Init_Params() */ -typedef struct sz_params -{ - int dataType; - unsigned int max_quant_intervals; //max number of quantization intervals for quantization - unsigned int quantization_intervals; - unsigned int maxRangeRadius; - int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri) - int losslessCompressor; - int sampleDistance; //2 bytes - float predThreshold; // 2 bytes - int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression - int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION - int errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR - double absErrBound; //absolute error bound - double relBoundRatio; //value range based relative error bound ratio - double psnr; //PSNR - double normErr; - double pw_relBoundRatio; //point-wise relative error bound - int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated) - int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio - - int protectValueRange; //0 or 1 - float fmin, fmax; - double dmin, dmax; - - int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep - int predictionMode; - - int accelerate_pw_rel_compression; - int plus_bits; - - int randomAccess; - int withRegression; - -} sz_params; typedef struct sz_metadata { @@ -206,13 +171,6 @@ typedef struct sz_metadata struct sz_params* conf_params; //configuration parameters } sz_metadata; -typedef struct sz_exedata -{ - char optQuantMode; //opt Quantization (0: fixed ; 1: optimized) - int intvCapacity; // the number of intervals for the linear-scaling quantization - int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2) - unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t) -} sz_exedata; /*We use a linked list to maintain time-step meta info for time-step based compression*/ typedef struct sz_tsc_metainfo @@ -296,7 +254,7 @@ void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params); void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1); -sz_metadata* SZ_getMetadata(unsigned char* bytes); +sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe); void SZ_printMetadata(sz_metadata* metadata); @@ -320,7 +278,7 @@ void SZ_decompress_ts(unsigned char *bytes, size_t byteLength); void SZ_Finalize(); void convertSZParamsToBytes(sz_params* params, unsigned char* result); -void convertBytesToSZParams(unsigned char* bytes, sz_params* params); +void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe); unsigned char* SZ_compress_customize(const char* appName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status); diff --git a/deps/SZ/sz/include/szd_double.h b/deps/SZ/sz/include/szd_double.h index 3fcf48bc6f957b8b40a98774c35f12276379d2d6..a22287c5d19f135dbc11dca6cd47db7f8d9eb3f2 100644 --- a/deps/SZ/sz/include/szd_double.h +++ b/deps/SZ/sz/include/szd_double.h @@ -25,7 +25,7 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt void decompressDataSeries_double_2D_MSST19(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps); void decompressDataSeries_double_3D_MSST19(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps); -void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data); +void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params); void getSnapshotData_double_2D(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data); void getSnapshotData_double_3D(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data); void getSnapshotData_double_4D(double** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data); @@ -34,7 +34,7 @@ void decompressDataSeries_double_3D_nonblocked_with_blocked_regression(double** size_t decompressDataSeries_double_3D_RA_block(double * data, double mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, double * unpredictable_data); -int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data); +int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/szd_float.h b/deps/SZ/sz/include/szd_float.h index d11bba4478c6264b39008845d4a7abd994f40e66..8cfa063970d6cb8853e1279b7f6b521789270efc 100644 --- a/deps/SZ/sz/include/szd_float.h +++ b/deps/SZ/sz/include/szd_float.h @@ -25,7 +25,7 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength, void decompressDataSeries_float_2D_MSST19(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps); void decompressDataSeries_float_3D_MSST19(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps); -void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data); +void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params); void getSnapshotData_float_2D(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data); void getSnapshotData_float_3D(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data); void getSnapshotData_float_4D(float** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data); @@ -33,7 +33,7 @@ void getSnapshotData_float_4D(float** data, size_t r1, size_t r2, size_t r3, siz size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data); size_t decompressDataSeries_float_2D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t block_dim_0, size_t block_dim_1, double realPrecision, int * type, float * unpredictable_data); -int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data); +int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params); size_t decompressDataSeries_float_3D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, float * unpredictable_data); diff --git a/deps/SZ/sz/src/ByteToolkit.c b/deps/SZ/sz/src/ByteToolkit.c index c3d7ad1eb5ccfcabc6861a0a7e04d48d0098108e..a715d4f252947c82f3bdeac958aab515453b121a 100644 --- a/deps/SZ/sz/src/ByteToolkit.c +++ b/deps/SZ/sz/src/ByteToolkit.c @@ -971,10 +971,10 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result) } -void convertBytesToSZParams(unsigned char* bytes, sz_params* params) +void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe) { unsigned char flag1 = bytes[0]; - exe_params->optQuantMode = (flag1 & 0x40) >> 6; + pde_exe->optQuantMode = (flag1 & 0x40) >> 6; dataEndianType = (flag1 & 0x20) >> 5; //sysEndianType = (flag1 & 0x10) >> 4; @@ -1038,7 +1038,7 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params) //params->segment_size = bytesToInt16_bigEndian(&bytes[14]); params->sol_ID = (int)(bytes[14]); - if(exe_params->optQuantMode==1) + if(pde_exe->optQuantMode==1) { params->max_quant_intervals = bytesToInt32_bigEndian(&bytes[16]); params->quantization_intervals = 0; diff --git a/deps/SZ/sz/src/TightDataPointStorageD.c b/deps/SZ/sz/src/TightDataPointStorageD.c index f30d8cdd62f20a0bc1c43cdefdac1ce74b5b9c74..eb40fa2c3a92c1339b897396292941708014264b 100644 --- a/deps/SZ/sz/src/TightDataPointStorageD.c +++ b/deps/SZ/sz/src/TightDataPointStorageD.c @@ -52,7 +52,7 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this) } -int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength) +int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params) { new_TightDataPointStorageD_Empty(this); size_t i, index = 0; @@ -92,7 +92,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi confparams_dec = (sz_params*)malloc(sizeof(sz_params)); memset(confparams_dec, 0, sizeof(sz_params)); } - convertBytesToSZParams(&(flatBytes[index]), confparams_dec); + convertBytesToSZParams(&(flatBytes[index]), confparams_dec, pde_exe); index += MetaDataByteLength_double; diff --git a/deps/SZ/sz/src/TightDataPointStorageF.c b/deps/SZ/sz/src/TightDataPointStorageF.c index aa0e0c1ce3b4d9f796e915f5f3cdc62cd6780e82..eed001fee097047a4bdaa9942bc040dd9cd8a86f 100644 --- a/deps/SZ/sz/src/TightDataPointStorageF.c +++ b/deps/SZ/sz/src/TightDataPointStorageF.c @@ -51,7 +51,7 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this) (*this)->raBytes_size = 0; } -int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength) +int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params) { new_TightDataPointStorageF_Empty(this); size_t i, index = 0; @@ -73,12 +73,12 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi (*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 int isPW_REL = (sameRByte & 0x20)>>5; //0010,0000 exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 - //confparams_dec->randomAccess = (sameRByte & 0x02) >> 1; - //confparams_dec->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it) + //pde_params->randomAccess = (sameRByte & 0x02) >> 1; + //pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it) - confparams_dec->protectValueRange = (sameRByte & 0x04)>>2; + pde_params->protectValueRange = (sameRByte & 0x04)>>2; - confparams_dec->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000 + pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000 int errorBoundMode = ABS; if(isPW_REL) @@ -88,12 +88,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi pwrErrBoundBytesL = 4; } - if(confparams_dec==NULL) - { - confparams_dec = (sz_params*)malloc(sizeof(sz_params)); - memset(confparams_dec, 0, sizeof(sz_params)); - } - convertBytesToSZParams(&(flatBytes[index]), confparams_dec); + + convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe); index += MetaDataByteLength; @@ -132,7 +128,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi byteBuf[i] = flatBytes[index++]; int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4 - confparams_dec->maxRangeRadius = max_quant_intervals/2; + pde_params->maxRangeRadius = max_quant_intervals/2; if(errorBoundMode>=PW_REL) { @@ -140,7 +136,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi radExpoL = 1; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - confparams_dec->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE + pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE for (i = 0; i < 4; i++) byteBuf[i] = flatBytes[index++]; @@ -161,7 +157,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi (*this)->reqLength = flatBytes[index++]; //1 - if(isPW_REL && confparams_dec->accelerate_pw_rel_compression) + if(isPW_REL && pde_params->accelerate_pw_rel_compression) { (*this)->plus_bits = flatBytes[index++]; (*this)->max_bits = flatBytes[index++]; diff --git a/deps/SZ/sz/src/TightDataPointStorageI.c b/deps/SZ/sz/src/TightDataPointStorageI.c index 569f57987e226dcc551ff891be1508984adec032..328e8c047a8aac25397f8ce3866342a9f223ae0c 100644 --- a/deps/SZ/sz/src/TightDataPointStorageI.c +++ b/deps/SZ/sz/src/TightDataPointStorageI.c @@ -134,7 +134,7 @@ int new_TightDataPointStorageI_fromFlatBytes(TightDataPointStorageI **this, unsi confparams_dec = (sz_params*)malloc(sizeof(sz_params)); memset(confparams_dec, 0, sizeof(sz_params)); } - convertBytesToSZParams(&(flatBytes[index]), confparams_dec); + convertBytesToSZParams(&(flatBytes[index]), confparams_dec, exe_params); /*sz_params* params = convertBytesToSZParams(&(flatBytes[index])); int mode = confparams_dec->szMode; int losslessCompressor = confparams_dec->losslessCompressor; diff --git a/deps/SZ/sz/src/sz.c b/deps/SZ/sz/src/sz.c index 0567a6fc4486bdced2180a8175e5fea292c5ce4e..53a12441b414b59a1bb45d06860ef70cf454cc24 100644 --- a/deps/SZ/sz/src/sz.c +++ b/deps/SZ/sz/src/sz.c @@ -195,54 +195,6 @@ double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, siz return newByteData; } - else if(dataType==SZ_INT64) - { - unsigned char *newByteData; - SZ_compress_args_int64(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_INT32) //int type - { - unsigned char *newByteData; - SZ_compress_args_int32(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_INT16) - { - unsigned char *newByteData; - SZ_compress_args_int16(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_INT8) - { - unsigned char *newByteData; - SZ_compress_args_int8(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_UINT64) - { - unsigned char *newByteData; - SZ_compress_args_uint64(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_UINT32) //int type - { - unsigned char *newByteData; - SZ_compress_args_uint32(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_UINT16) - { - unsigned char *newByteData; - SZ_compress_args_uint16(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } - else if(dataType==SZ_UINT8) - { - unsigned char *newByteData; - SZ_compress_args_uint8(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio); - return newByteData; - } else { printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE, SZ_INT8/16/32/64 or SZ_UINT8/16/32/64.\n"); @@ -345,13 +297,13 @@ unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, si void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { - if(confparams_dec==NULL) - confparams_dec = (sz_params*)malloc(sizeof(sz_params)); - memset(confparams_dec, 0, sizeof(sz_params)); - if(exe_params==NULL) - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); - memset(exe_params, 0, sizeof(sz_exedata)); - exe_params->SZ_SIZE_TYPE = 8; + sz_exedata de_exe; + memset(&de_exe, 0, sizeof(sz_exedata)); + de_exe.SZ_SIZE_TYPE = 8; + + sz_params de_params; + memset(&de_params, 0, sizeof(sz_params)); + int x = 1; char *y = (char*)&x; @@ -363,63 +315,15 @@ void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_ if(dataType == SZ_FLOAT) { float *newFloatData; - SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL); + SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); return newFloatData; } else if(dataType == SZ_DOUBLE) { double *newDoubleData; - SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL); + SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params); return newDoubleData; } - else if(dataType == SZ_INT8) - { - int8_t *newInt8Data; - SZ_decompress_args_int8(&newInt8Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newInt8Data; - } - else if(dataType == SZ_INT16) - { - int16_t *newInt16Data; - SZ_decompress_args_int16(&newInt16Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newInt16Data; - } - else if(dataType == SZ_INT32) - { - int32_t *newInt32Data; - SZ_decompress_args_int32(&newInt32Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newInt32Data; - } - else if(dataType == SZ_INT64) - { - int64_t *newInt64Data; - SZ_decompress_args_int64(&newInt64Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newInt64Data; - } - else if(dataType == SZ_UINT8) - { - uint8_t *newUInt8Data; - SZ_decompress_args_uint8(&newUInt8Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newUInt8Data; - } - else if(dataType == SZ_UINT16) - { - uint16_t *newUInt16Data; - SZ_decompress_args_uint16(&newUInt16Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newUInt16Data; - } - else if(dataType == SZ_UINT32) - { - uint32_t *newUInt32Data; - SZ_decompress_args_uint32(&newUInt32Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newUInt32Data; - } - else if(dataType == SZ_UINT64) - { - uint64_t *newUInt64Data; - SZ_decompress_args_uint64(&newUInt64Data, r5, r4, r3, r2, r1, bytes, byteLength); - return newUInt64Data; - } else { printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n"); @@ -521,7 +425,7 @@ size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, } -sz_metadata* SZ_getMetadata(unsigned char* bytes) +sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe) { int index = 0, i, isConstant, isLossless; size_t dataSeriesLength = 0; @@ -535,12 +439,8 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes) int isRegressionBased = (sameRByte >> 7) & 0x01; - if(exe_params==NULL) - { - exe_params = (sz_exedata *)malloc(sizeof(struct sz_exedata)); - memset(exe_params, 0, sizeof(struct sz_exedata)); - } - exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; + + pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; if(confparams_dec==NULL) { @@ -548,7 +448,7 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes) memset(confparams_dec, 0, sizeof(sz_params)); } - convertBytesToSZParams(&(bytes[index]), confparams_dec); + convertBytesToSZParams(&(bytes[index]), confparams_dec, pde_exe); /*sz_params* params = convertBytesToSZParams(&(bytes[index])); if(confparams_dec!=NULL) free(confparams_dec); @@ -983,15 +883,14 @@ int SZ_compress_ts(int cmprType, unsigned char** newByteData, size_t *outSize) void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength) { - if(confparams_dec==NULL) - confparams_dec = (sz_params*)malloc(sizeof(sz_params)); - memset(confparams_dec, 0, sizeof(sz_params)); - confparams_dec->szMode = SZ_TEMPORAL_COMPRESSION; - confparams_dec->predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE; - - if(exe_params==NULL) - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); - memset(exe_params, 0, sizeof(sz_exedata)); + sz_params de_params; + memset(&de_params, 0, sizeof(sz_params)); + de_params.szMode = SZ_TEMPORAL_COMPRESSION; + de_params.predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE; + + sz_exedata de_exe; + memset(&de_exe, 0, sizeof(sz_exedata)); + int x = 1; char *y = (char*)&x; @@ -1037,12 +936,12 @@ void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength) switch(dataType) { case SZ_FLOAT: - SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data); + SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params); memcpy(p->data, newFloatData, dataLen*sizeof(float)); free(newFloatData); break; case SZ_DOUBLE: - SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data); + SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params); memcpy(p->data, newDoubleData, dataLen*sizeof(double)); free(newDoubleData); break; @@ -1058,15 +957,13 @@ void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength) void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count, unsigned char *bytes, size_t bytesLength) { - if(confparams_dec==NULL) - confparams_dec = (sz_params*)malloc(sizeof(sz_params)); - memset(confparams_dec, 0, sizeof(sz_params)); - confparams_dec->szMode = SZ_TEMPORAL_COMPRESSION; - confparams_dec->predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE; + sz_params de_params; + memset(&de_params, 0, sizeof(sz_params)); + de_params.szMode = SZ_TEMPORAL_COMPRESSION; + de_params.predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE; - if(exe_params==NULL) - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); - memset(exe_params, 0, sizeof(sz_exedata)); + sz_exedata de_exe; + memset(&de_exe, 0, sizeof(sz_exedata)); int x = 1; char *y = (char*)&x; @@ -1113,12 +1010,12 @@ void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count switch(dataType) { case SZ_FLOAT: - SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data); + SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params); memcpy(p->data, newFloatData, dataLen*sizeof(float)); free(newFloatData); break; case SZ_DOUBLE: - SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data); + SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params); memcpy(p->data, newDoubleData, dataLen*sizeof(double)); free(newDoubleData); break; diff --git a/deps/SZ/sz/src/szd_double.c b/deps/SZ/sz/src/szd_double.c index cd3cd1c0d96cb5fea76857bfecf621d86dfbe896..7a49b879581abedbffaabcfe54952b53c3c54db5 100644 --- a/deps/SZ/sz/src/szd_double.c +++ b/deps/SZ/sz/src/szd_double.c @@ -19,7 +19,7 @@ #include "utility.h" int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, -size_t cmpSize, int compressionType, double* hist_data) +size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params) { int status = SZ_SCES; size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); @@ -66,7 +66,7 @@ size_t cmpSize, int compressionType, double* hist_data) confparams_dec->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose //TODO: convert szTmpBytes to double array. TightDataPointStorageD* tdps; - int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize); + int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); int dim = computeDimension(r5,r4,r3,r2,r1); int doubleSize = sizeof(double); @@ -86,14 +86,14 @@ size_t cmpSize, int compressionType, double* hist_data) } else if(confparams_dec->sol_ID==SZ_Transpose) { - getSnapshotData_double_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data); + getSnapshotData_double_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params); } else //confparams_dec->sol_ID==SZ { if(tdps->raBytes_size > 0) //v2.0 { if (dim == 1) - getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data); + getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); else if(dim == 2) decompressDataSeries_double_2D_nonblocked_with_blocked_regression(newData, r2, r1, tdps->raBytes, hist_data); else if(dim == 3) @@ -109,7 +109,7 @@ size_t cmpSize, int compressionType, double* hist_data) else //1.4.13 or time-based compression { if (dim == 1) - getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data); + getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); else if (dim == 2) getSnapshotData_double_2D(newData,r2,r1,tdps, errBoundMode, compressionType, hist_data); @@ -2659,7 +2659,7 @@ void decompressDataSeries_double_3D_MSST19(double** data, size_t r1, size_t r2, return; } -void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data) +void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params) { size_t i; if (tdps->allSameData) { diff --git a/deps/SZ/sz/src/szd_float.c b/deps/SZ/sz/src/szd_float.c index 2c4a1f12aef18eb6a34497eca34118f291c94759..732a9ba4957cd9d33e7b073d3624ba9cb6549c2b 100644 --- a/deps/SZ/sz/src/szd_float.c +++ b/deps/SZ/sz/src/szd_float.c @@ -48,7 +48,7 @@ void cost_end_() * @return status SUCCESSFUL (SZ_SCES) or not (other error codes) f * */ int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, -size_t cmpSize, int compressionType, float* hist_data) +size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params) { int status = SZ_SCES; size_t dataLength = computeDataLength(r5,r4,r3,r2,r1); @@ -56,37 +56,37 @@ size_t cmpSize, int compressionType, float* hist_data) //unsigned char* tmpBytes; size_t targetUncompressSize = dataLength <<2; //i.e., *4 //tmpSize must be "much" smaller than dataLength - size_t i, tmpSize = 8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + size_t i, tmpSize = 8+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE; unsigned char* szTmpBytes; if(cmpSize!=8+4+MetaDataByteLength && cmpSize!=8+8+MetaDataByteLength) //4,8 means two posibilities of SZ_SIZE_TYPE { - confparams_dec->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize); - if(confparams_dec->szMode!=SZ_TEMPORAL_COMPRESSION) + pde_params->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize); + if(pde_params->szMode!=SZ_TEMPORAL_COMPRESSION) { - if(confparams_dec->losslessCompressor!=-1) - confparams_dec->szMode = SZ_BEST_COMPRESSION; + if(pde_params->losslessCompressor!=-1) + pde_params->szMode = SZ_BEST_COMPRESSION; else - confparams_dec->szMode = SZ_BEST_SPEED; + pde_params->szMode = SZ_BEST_SPEED; } - if(confparams_dec->szMode==SZ_BEST_SPEED) + if(pde_params->szMode==SZ_BEST_SPEED) { tmpSize = cmpSize; szTmpBytes = cmpBytes; } - else if(confparams_dec->szMode==SZ_BEST_COMPRESSION || confparams_dec->szMode==SZ_DEFAULT_COMPRESSION || confparams_dec->szMode==SZ_TEMPORAL_COMPRESSION) + else if(pde_params->szMode==SZ_BEST_COMPRESSION || pde_params->szMode==SZ_DEFAULT_COMPRESSION || pde_params->szMode==SZ_TEMPORAL_COMPRESSION) { if(targetUncompressSizelosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize + tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize //szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize); //memcpy(szTmpBytes, tmpBytes, tmpSize); //free(tmpBytes); //release useless memory } else { - printf("Wrong value of confparams_dec->szMode in the double compressed bytes.\n"); + printf("Wrong value of pde_params->szMode in the double compressed bytes.\n"); status = SZ_MERR; return status; } @@ -94,11 +94,11 @@ size_t cmpSize, int compressionType, float* hist_data) else szTmpBytes = cmpBytes; - confparams_dec->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose + pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose //TODO: convert szTmpBytes to data array. TightDataPointStorageF* tdps; - int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize); + int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params); //writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt"); int dim = computeDimension(r5,r4,r3,r2,r1); @@ -117,16 +117,16 @@ size_t cmpSize, int compressionType, float* hist_data) (*newData)[i] = bytesToFloat(p); } } - else if(confparams_dec->sol_ID==SZ_Transpose) + else if(pde_params->sol_ID==SZ_Transpose) { - getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data); + getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params); } - else //confparams_dec->sol_ID==SZ + else //pde_params->sol_ID==SZ { if(tdps->raBytes_size > 0) //v2.0 { if (dim == 1) - getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data); + getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params); else if(dim == 2) decompressDataSeries_float_2D_nonblocked_with_blocked_regression(newData, r2, r1, tdps->raBytes, hist_data); else if(dim == 3) @@ -142,7 +142,7 @@ size_t cmpSize, int compressionType, float* hist_data) else //1.4.13 or time-based compression { if (dim == 1) - getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data); + getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params); else if (dim == 2) getSnapshotData_float_2D(newData,r2,r1,tdps, errBoundMode, compressionType, hist_data); else if (dim == 3) @@ -158,11 +158,11 @@ size_t cmpSize, int compressionType, float* hist_data) } //cost_start_(); - if(confparams_dec->protectValueRange) + if(pde_params->protectValueRange) { float* nd = *newData; - float min = confparams_dec->fmin; - float max = confparams_dec->fmax; + float min = pde_params->fmin; + float max = pde_params->fmax; for(i=0;iszMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE) + if(pde_params->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE) free(szTmpBytes); return status; } @@ -272,7 +272,7 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float* } #ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) + if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION) memcpy(hist_data, (*data), dataSeriesLength*sizeof(float)); #endif @@ -2683,7 +2683,7 @@ void decompressDataSeries_float_3D_MSST19(float** data, size_t r1, size_t r2, si return; } -void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data) +void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params) { size_t i; @@ -2697,7 +2697,7 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo if(errBoundMode < PW_REL) { #ifdef HAVE_TIMECMPR - if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION) + if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION) { if(compressionType == 0) //snapshot decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps); @@ -2710,7 +2710,7 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo } else { - if(confparams_dec->accelerate_pw_rel_compression) + if(pde_params->accelerate_pw_rel_compression) decompressDataSeries_float_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps); else decompressDataSeries_float_1D_pwr_pre_log(data, dataSeriesLength, tdps);