diff --git a/deps/SZ/sz/include/ByteToolkit.h b/deps/SZ/sz/include/ByteToolkit.h index 82d060483c05b69f2be04d9fb9bb20726a13420e..fee74e5b6c13710a5cb91ccc8c2a4d12712b10c8 100644 --- a/deps/SZ/sz/include/ByteToolkit.h +++ b/deps/SZ/sz/include/ByteToolkit.h @@ -36,8 +36,8 @@ int getLeftMovingCode(int kMod8); int getRightMovingSteps(int kMod8, int resiBitLength); int getRightMovingCode(int kMod8, int resiBitLength); -size_t bytesToSize(unsigned char* bytes); -void sizeToBytes(unsigned char* outBytes, size_t size); +size_t bytesToSize(unsigned char* bytes, int size_type); +void sizeToBytes(unsigned char* outBytes, size_t size, int size_type); #ifdef __cplusplus } diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index 735c38ea493944e54675feada9e6475b517759b6..38aaa6c41b4b6bdb1b68582e2b76dceed482f86e 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -186,7 +186,7 @@ size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outD size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData); -void convertSZParamsToBytes(sz_params* params, unsigned char* result); +void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode); void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe); #ifdef __cplusplus diff --git a/deps/SZ/sz/src/ByteToolkit.c b/deps/SZ/sz/src/ByteToolkit.c index 032e9a10b85df6cce587f56d946bbe846a29f43f..f9fd8ba90e679bd8781790a0b83a2990a0ea126f 100644 --- a/deps/SZ/sz/src/ByteToolkit.c +++ b/deps/SZ/sz/src/ByteToolkit.c @@ -260,30 +260,29 @@ INLINE int getRightMovingCode(int kMod8, int resiBitLength) } } -INLINE size_t bytesToSize(unsigned char* bytes) +INLINE size_t bytesToSize(unsigned char* bytes, int size_type) { size_t result = 0; - if(exe_params->SZ_SIZE_TYPE==4) + if(size_type == 4) result = bytesToInt_bigEndian(bytes);//4 else result = bytesToLong_bigEndian(bytes);//8 return result; } -INLINE void sizeToBytes(unsigned char* outBytes, size_t size) +INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type) { - if(exe_params->SZ_SIZE_TYPE==4) + if(size_type == 4) intToBytes_bigEndian(outBytes, (unsigned int)size);//4 else longToBytes_bigEndian(outBytes, (unsigned long)size);//8 } -void convertSZParamsToBytes(sz_params* params, unsigned char* result) +void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode) { //unsigned char* result = (unsigned char*)malloc(16); unsigned char buf = 0; - //flag1: exe_params->optQuantMode(1bit), dataEndianType(1bit), sysEndianType(1bit), conf_params->szMode (1bit), conf_params->gzipMode (2bits), pwrType (2bits) - buf = exe_params->optQuantMode; + buf = optQuantMode; buf = (buf << 1) | dataEndianType; buf = (buf << 1) | sysEndianType; buf = (buf << 2) | params->szMode; diff --git a/deps/SZ/sz/src/TightDataPointStorageD.c b/deps/SZ/sz/src/TightDataPointStorageD.c index a4dc2b4c79a8102e6681cbb38593a28ebbce4e16..07f84aece8981f7282a2e5747b177c9febc43dd6 100644 --- a/deps/SZ/sz/src/TightDataPointStorageD.c +++ b/deps/SZ/sz/src/TightDataPointStorageD.c @@ -41,7 +41,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi int same = sameRByte & 0x01; (*this)->isLossless = (sameRByte & 0x10)>>4; - exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; + pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //pde_params->protectValueRange = (sameRByte & 0x04)>>2; pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3; int errorBoundMode = SZ_ABS; @@ -53,9 +53,9 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi int isRegression = (sameRByte >> 7) & 0x01; unsigned char dsLengthBytes[8]; - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) dsLengthBytes[i] = flatBytes[index++]; - (*this)->dataSeriesLength = bytesToSize(dsLengthBytes); + (*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE); if((*this)->isLossless==1) { @@ -65,7 +65,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi else if(same==1) { (*this)->allSameData = 1; - //size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength_double -exe_params->SZ_SIZE_TYPE; (*this)->exactMidBytes = &(flatBytes[index]); return errorBoundMode; } @@ -74,7 +73,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi if(isRegression == 1) { - (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE; + (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE; (*this)->raBytes = &(flatBytes[index]); return errorBoundMode; } @@ -101,17 +100,17 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi byteBuf[i] = flatBytes[index++]; (*this)->realPrecision = bytesToDouble(byteBuf);//8 - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE + (*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE); - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->exactDataNum = bytesToSize(byteBuf);// ST + (*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST + (*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; if (logicLeadNumBitsNum % 8 == 0) @@ -132,8 +131,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi // todo need check length - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE + (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 + - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; (*this)->leadNumArray = &flatBytes[index]; @@ -211,7 +210,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte bytes[k++] = versionNumber; bytes[k++] = sameByte; //1 byte - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode); k = k + MetaDataByteLength_double; for(i = 0;iSZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes @@ -234,15 +233,15 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte for (i = 0; i < 8; i++)// 8 bytes[k++] = realPrecisionBytes[i]; - sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); + sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = typeArrayLengthBytes[i]; - sizeToBytes(exactLengthBytes, tdps->exactDataNum); + sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactLengthBytes[i]; - sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); + sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactMidBytesLength[i]; @@ -302,7 +301,7 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes[k++] = versionNumber; bytes[k++] = sameByte; - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode); k = k + MetaDataByteLength_double; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) diff --git a/deps/SZ/sz/src/TightDataPointStorageF.c b/deps/SZ/sz/src/TightDataPointStorageF.c index 47afc4f9dfc4abc869dc2e952385d2cb1f7079d4..cf8c45a417695109fa288bb362d83c6436d5e3b1 100644 --- a/deps/SZ/sz/src/TightDataPointStorageF.c +++ b/deps/SZ/sz/src/TightDataPointStorageF.c @@ -48,16 +48,16 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi // 2 same(1) //note that 1000,0000 is reserved for regression tag. int same = sameRByte & 0x01; //0000,0001 (*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000 - exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 + pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 int errorBoundMode = SZ_ABS; // 3 meta(2) convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe); index += MetaDataByteLength; // 4 element count(4) unsigned char dsLengthBytes[8]; - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) dsLengthBytes[i] = flatBytes[index++]; - (*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8 + (*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8 if((*this)->isLossless==1) { //(*this)->exactMidBytes = flatBytes+8; @@ -66,7 +66,6 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi else if(same==1) { (*this)->allSameData = 1; - //size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE; (*this)->exactMidBytes = &(flatBytes[index]); return errorBoundMode; } @@ -76,7 +75,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi int isRegression = (sameRByte >> 7) & 0x01; if(isRegression == 1) { - (*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE; + (*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE; (*this)->raBytes = &(flatBytes[index]); return errorBoundMode; } @@ -101,17 +100,17 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi byteBuf[i] = flatBytes[index++]; (*this)->realPrecision = bytesToDouble(byteBuf);//8 // 10 typeArray_size - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->typeArray_size = bytesToSize(byteBuf);// 4 + (*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4 // 11 exactNum - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->exactDataNum = bytesToSize(byteBuf);// ST + (*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST // 12 mid size - for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) + for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++) byteBuf[i] = flatBytes[index++]; - (*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST + (*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq // calc leadNumArray_size size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; @@ -141,8 +140,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi (*this)->residualMidBits = &flatBytes[index]; // calc residualMidBits_size - (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 - - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE + (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 + - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; @@ -211,7 +210,7 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes // 2 same bytes[k++] = sameByte; //1 byte // 3 meta - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode); k = k + MetaDataByteLength; // 4 element count for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes @@ -235,15 +234,15 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes for (i = 0; i < 8; i++)// 8 bytes[k++] = realPrecisionBytes[i]; // 10 typeArray size - sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); + sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = typeArrayLengthBytes[i]; // 11 exactDataNum leadNum calc by this , so not save leadNum - sizeToBytes(exactLengthBytes, tdps->exactDataNum); + sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactLengthBytes[i]; // 12 Mid size - sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); + sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE); for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = exactMidBytesLength[i]; // 13 typeArray @@ -306,7 +305,7 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b // 2 same flag 1 bytes bytes[k++] = sameByte; // 3 metaData 26 bytes - convertSZParamsToBytes(confparams_cpr, &(bytes[k])); + convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode); k = k + MetaDataByteLength; // 4 data Length 4 or 8 bytes for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) diff --git a/deps/SZ/sz/src/sz_double.c b/deps/SZ/sz/src/sz_double.c index 680f0ad86667ff9df63f9b7b9d56eb0c1c8459cc..f206d3de19976e5de866ebdb2db88551b717bbed 100644 --- a/deps/SZ/sz/src/sz_double.c +++ b/deps/SZ/sz/src/sz_double.c @@ -229,10 +229,10 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un else newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 - convertSZParamsToBytes(confparams_cpr, &(newByteData[k])); + convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode); k = k + MetaDataByteLength_double; - sizeToBytes(dsLengthBytes,dataLength); + sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE); for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 newByteData[k++] = dsLengthBytes[i]; diff --git a/deps/SZ/sz/src/szd_double.c b/deps/SZ/sz/src/szd_double.c index 93ef42e418550b49c8dd423065bbeb553921b9b9..50920b42b4e440e2243ad139e5dfba9948a9cd09 100644 --- a/deps/SZ/sz/src/szd_double.c +++ b/deps/SZ/sz/src/szd_double.c @@ -25,7 +25,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte //unsigned char* tmpBytes; size_t targetUncompressSize = dataLength <<3; //i.e., *8 //tmpSize must be "much" smaller than dataLength - size_t i, tmpSize = 12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; + size_t i, tmpSize = 12+MetaDataByteLength_double+8; unsigned char* szTmpBytes = NULL; bool needFree = false; @@ -50,7 +50,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte { if(targetUncompressSizelosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE); + tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+8); needFree = true; } } @@ -73,11 +73,11 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte // *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan if(sysEndianType==BIG_ENDIAN_SYSTEM) { - memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); + memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE, dataLength*doubleSize); } else { - unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; + unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE; for(i=0;ilosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize + tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+8); needFree = true; } } @@ -81,11 +81,11 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, //*newData = (float*)malloc(floatSize*dataLength); comment by tickduan if(sysEndianType==BIG_ENDIAN_SYSTEM) { - memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize); + memcpy(newData, szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE, dataLength*floatSize); } else { - unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + unsigned char* p = szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE; for(i=0;i