Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
提交 04aa94dd 编写于 作者: T tickduan
浏览文件
操作

fixed open lossy then close, read old data crash bug

上级 efdb7132
隐藏空白更改
内联 并排
Showing with 52 addition and 55 deletion
deps/SZ/sz/include/ByteToolkit.h
浏览文件 @ 04aa94dd
...@@ -36,8 +36,8 @@ int getLeftMovingCode(int kMod8); ...@@ -36,8 +36,8 @@ int getLeftMovingCode(int kMod8);
int getRightMovingSteps(int kMod8, int resiBitLength); int getRightMovingSteps(int kMod8, int resiBitLength);
int getRightMovingCode(int kMod8, int resiBitLength); int getRightMovingCode(int kMod8, int resiBitLength);
size_t bytesToSize(unsigned char* bytes); size_t bytesToSize(unsigned char* bytes, int size_type);
void sizeToBytes(unsigned char* outBytes, size_t size); void sizeToBytes(unsigned char* outBytes, size_t size, int size_type);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
deps/SZ/sz/include/sz.h
浏览文件 @ 04aa94dd
...@@ -186,7 +186,7 @@ size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outD ...@@ -186,7 +186,7 @@ size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outD
size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData); size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
void convertSZParamsToBytes(sz_params* params, unsigned char* result); void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe); void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
#ifdef __cplusplus #ifdef __cplusplus
......
deps/SZ/sz/src/ByteToolkit.c
浏览文件 @ 04aa94dd
...@@ -260,30 +260,29 @@ INLINE int getRightMovingCode(int kMod8, int resiBitLength) ...@@ -260,30 +260,29 @@ INLINE int getRightMovingCode(int kMod8, int resiBitLength)
} }
} }
INLINE size_t bytesToSize(unsigned char* bytes) INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
{ {
size_t result = 0; size_t result = 0;
if(exe_params->SZ_SIZE_TYPE==4) if(size_type == 4)
result = bytesToInt_bigEndian(bytes);//4 result = bytesToInt_bigEndian(bytes);//4
else else
result = bytesToLong_bigEndian(bytes);//8 result = bytesToLong_bigEndian(bytes);//8
return result; return result;
} }
INLINE void sizeToBytes(unsigned char* outBytes, size_t size) INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
{ {
if(exe_params->SZ_SIZE_TYPE==4) if(size_type == 4)
intToBytes_bigEndian(outBytes, (unsigned int)size);//4 intToBytes_bigEndian(outBytes, (unsigned int)size);//4
else else
longToBytes_bigEndian(outBytes, (unsigned long)size);//8 longToBytes_bigEndian(outBytes, (unsigned long)size);//8
} }
void convertSZParamsToBytes(sz_params* params, unsigned char* result) void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)
{ {
//unsigned char* result = (unsigned char*)malloc(16); //unsigned char* result = (unsigned char*)malloc(16);
unsigned char buf = 0; unsigned char buf = 0;
//flag1: exe_params->optQuantMode(1bit), dataEndianType(1bit), sysEndianType(1bit), conf_params->szMode (1bit), conf_params->gzipMode (2bits), pwrType (2bits) buf = optQuantMode;
buf = exe_params->optQuantMode;
buf = (buf << 1) | dataEndianType; buf = (buf << 1) | dataEndianType;
buf = (buf << 1) | sysEndianType; buf = (buf << 1) | sysEndianType;
buf = (buf << 2) | params->szMode; buf = (buf << 2) | params->szMode;
......
deps/SZ/sz/src/TightDataPointStorageD.c
浏览文件 @ 04aa94dd
...@@ -41,7 +41,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -41,7 +41,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
int same = sameRByte & 0x01; int same = sameRByte & 0x01;
(*this)->isLossless = (sameRByte & 0x10)>>4; (*this)->isLossless = (sameRByte & 0x10)>>4;
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2; //pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3; pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS; int errorBoundMode = SZ_ABS;
...@@ -53,9 +53,9 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -53,9 +53,9 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
int isRegression = (sameRByte >> 7) & 0x01; int isRegression = (sameRByte >> 7) & 0x01;
unsigned char dsLengthBytes[8]; unsigned char dsLengthBytes[8];
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++]; dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes); (*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);
if((*this)->isLossless==1) if((*this)->isLossless==1)
{ {
...@@ -65,7 +65,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -65,7 +65,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
else if(same==1) else if(same==1)
{ {
(*this)->allSameData = 1; (*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength_double -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]); (*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
...@@ -74,7 +73,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -74,7 +73,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
if(isRegression == 1) if(isRegression == 1)
{ {
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE; (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]); (*this)->raBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
...@@ -101,17 +100,17 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -101,17 +100,17 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8 (*this)->realPrecision = bytesToDouble(byteBuf);//8
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE (*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST (*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST (*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
if (logicLeadNumBitsNum % 8 == 0) if (logicLeadNumBitsNum % 8 == 0)
...@@ -132,8 +131,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi ...@@ -132,8 +131,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
// todo need check length // todo need check length
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
(*this)->leadNumArray = &flatBytes[index]; (*this)->leadNumArray = &flatBytes[index];
...@@ -211,7 +210,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte ...@@ -211,7 +210,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
bytes[k++] = versionNumber; bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1 byte bytes[k++] = sameByte; //1 byte
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double; k = k + MetaDataByteLength_double;
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
...@@ -234,15 +233,15 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte ...@@ -234,15 +233,15 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
for (i = 0; i < 8; i++)// 8 for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i]; bytes[k++] = realPrecisionBytes[i];
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i]; bytes[k++] = typeArrayLengthBytes[i];
sizeToBytes(exactLengthBytes, tdps->exactDataNum); sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i]; bytes[k++] = exactLengthBytes[i];
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i]; bytes[k++] = exactMidBytesLength[i];
...@@ -302,7 +301,7 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* ...@@ -302,7 +301,7 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
bytes[k++] = versionNumber; bytes[k++] = versionNumber;
bytes[k++] = sameByte; bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double; k = k + MetaDataByteLength_double;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
......
deps/SZ/sz/src/TightDataPointStorageF.c
浏览文件 @ 04aa94dd
...@@ -48,16 +48,16 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi ...@@ -48,16 +48,16 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
// 2 same(1) //note that 1000,0000 is reserved for regression tag. // 2 same(1) //note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001 int same = sameRByte & 0x01; //0000,0001
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000 (*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000 //0010,0000
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000 pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
int errorBoundMode = SZ_ABS; int errorBoundMode = SZ_ABS;
// 3 meta(2) // 3 meta(2)
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe); convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength; index += MetaDataByteLength;
// 4 element count(4) // 4 element count(4)
unsigned char dsLengthBytes[8]; unsigned char dsLengthBytes[8];
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
dsLengthBytes[i] = flatBytes[index++]; dsLengthBytes[i] = flatBytes[index++];
(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8 (*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8
if((*this)->isLossless==1) if((*this)->isLossless==1)
{ {
//(*this)->exactMidBytes = flatBytes+8; //(*this)->exactMidBytes = flatBytes+8;
...@@ -66,7 +66,6 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi ...@@ -66,7 +66,6 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
else if(same==1) else if(same==1)
{ {
(*this)->allSameData = 1; (*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]); (*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
...@@ -76,7 +75,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi ...@@ -76,7 +75,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
int isRegression = (sameRByte >> 7) & 0x01; int isRegression = (sameRByte >> 7) & 0x01;
if(isRegression == 1) if(isRegression == 1)
{ {
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE; (*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]); (*this)->raBytes = &(flatBytes[index]);
return errorBoundMode; return errorBoundMode;
} }
...@@ -101,17 +100,17 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi ...@@ -101,17 +100,17 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->realPrecision = bytesToDouble(byteBuf);//8 (*this)->realPrecision = bytesToDouble(byteBuf);//8
// 10 typeArray_size // 10 typeArray_size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->typeArray_size = bytesToSize(byteBuf);// 4 (*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4
// 11 exactNum // 11 exactNum
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->exactDataNum = bytesToSize(byteBuf);// ST (*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
// 12 mid size // 12 mid size
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++]; byteBuf[i] = flatBytes[index++];
(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST (*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq
// calc leadNumArray_size // calc leadNumArray_size
size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2; size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
...@@ -141,8 +140,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi ...@@ -141,8 +140,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->residualMidBits = &flatBytes[index]; (*this)->residualMidBits = &flatBytes[index];
// calc residualMidBits_size // calc residualMidBits_size
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 (*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size; - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
...@@ -211,7 +210,7 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes ...@@ -211,7 +210,7 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
// 2 same // 2 same
bytes[k++] = sameByte; //1 byte bytes[k++] = sameByte; //1 byte
// 3 meta // 3 meta
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength; k = k + MetaDataByteLength;
// 4 element count // 4 element count
for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
...@@ -235,15 +234,15 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes ...@@ -235,15 +234,15 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
for (i = 0; i < 8; i++)// 8 for (i = 0; i < 8; i++)// 8
bytes[k++] = realPrecisionBytes[i]; bytes[k++] = realPrecisionBytes[i];
// 10 typeArray size // 10 typeArray size
sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size); sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = typeArrayLengthBytes[i]; bytes[k++] = typeArrayLengthBytes[i];
// 11 exactDataNum leadNum calc by this , so not save leadNum // 11 exactDataNum leadNum calc by this , so not save leadNum
sizeToBytes(exactLengthBytes, tdps->exactDataNum); sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactLengthBytes[i]; bytes[k++] = exactLengthBytes[i];
// 12 Mid size // 12 Mid size
sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size); sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
bytes[k++] = exactMidBytesLength[i]; bytes[k++] = exactMidBytesLength[i];
// 13 typeArray // 13 typeArray
...@@ -306,7 +305,7 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b ...@@ -306,7 +305,7 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
// 2 same flag 1 bytes // 2 same flag 1 bytes
bytes[k++] = sameByte; bytes[k++] = sameByte;
// 3 metaData 26 bytes // 3 metaData 26 bytes
convertSZParamsToBytes(confparams_cpr, &(bytes[k])); convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength; k = k + MetaDataByteLength;
// 4 data Length 4 or 8 bytes // 4 data Length 4 or 8 bytes
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
......
deps/SZ/sz/src/sz_double.c
浏览文件 @ 04aa94dd
...@@ -229,10 +229,10 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un ...@@ -229,10 +229,10 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
else else
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &(newByteData[k])); convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode);
k = k + MetaDataByteLength_double; k = k + MetaDataByteLength_double;
sizeToBytes(dsLengthBytes,dataLength); sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE);
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
newByteData[k++] = dsLengthBytes[i]; newByteData[k++] = dsLengthBytes[i];
......
deps/SZ/sz/src/szd_double.c
浏览文件 @ 04aa94dd
...@@ -25,7 +25,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte ...@@ -25,7 +25,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
//unsigned char* tmpBytes; //unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<3; //i.e., *8 size_t targetUncompressSize = dataLength <<3; //i.e., *8
//tmpSize must be "much" smaller than dataLength //tmpSize must be "much" smaller than dataLength
size_t i, tmpSize = 12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; size_t i, tmpSize = 12+MetaDataByteLength_double+8;
unsigned char* szTmpBytes = NULL; unsigned char* szTmpBytes = NULL;
bool needFree = false; bool needFree = false;
...@@ -50,7 +50,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte ...@@ -50,7 +50,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
{ {
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES; targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE); tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+8);
needFree = true; needFree = true;
} }
} }
...@@ -73,11 +73,11 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte ...@@ -73,11 +73,11 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
// *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan // *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM) if(sysEndianType==BIG_ENDIAN_SYSTEM)
{ {
memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE, dataLength*doubleSize);
} }
else else
{ {
unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=doubleSize) for(i=0;i<dataLength;i++,p+=doubleSize)
newData[i] = bytesToDouble(p); newData[i] = bytesToDouble(p);
} }
......
deps/SZ/sz/src/szd_float.c
浏览文件 @ 04aa94dd
...@@ -57,7 +57,7 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, ...@@ -57,7 +57,7 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
{ {
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES; targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+8);
needFree = true; needFree = true;
} }
} }
...@@ -81,11 +81,11 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, ...@@ -81,11 +81,11 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
//*newData = (float*)malloc(floatSize*dataLength); comment by tickduan //*newData = (float*)malloc(floatSize*dataLength); comment by tickduan
if(sysEndianType==BIG_ENDIAN_SYSTEM) if(sysEndianType==BIG_ENDIAN_SYSTEM)
{ {
memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize); memcpy(newData, szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE, dataLength*floatSize);
} }
else else
{ {
unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; unsigned char* p = szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize) for(i=0;i<dataLength;i++,p+=floatSize)
newData[i] = bytesToFloat(p); newData[i] = bytesToFloat(p);
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或