fixed open lossy then close, read old data crash bug

deps/SZ/sz/include/ByteToolkit.h

@@ -36,8 +36,8 @@ int getLeftMovingCode(int kMod8);
 int getRightMovingSteps(int kMod8, int resiBitLength);
 int getRightMovingCode(int kMod8, int resiBitLength);
 
-size_t bytesToSize(unsigned char* bytes);
-void sizeToBytes(unsigned char* outBytes, size_t size);
+size_t bytesToSize(unsigned char* bytes, int size_type);
+void sizeToBytes(unsigned char* outBytes, size_t size, int size_type);
 
 #ifdef __cplusplus
 }

deps/SZ/sz/include/sz.h

@@ -186,7 +186,7 @@ size_t SZ_compress_args(int dataType, void *data, size_t r1, unsigned char* outD
 size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r1, unsigned char* outData);
 
 
-void convertSZParamsToBytes(sz_params* params, unsigned char* result);
+void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode);
 void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
 
 #ifdef __cplusplus

deps/SZ/sz/src/ByteToolkit.c

@@ -260,30 +260,29 @@ INLINE int getRightMovingCode(int kMod8, int resiBitLength)
 	}
 }
 
-INLINE size_t bytesToSize(unsigned char* bytes)
+INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
 {
 	size_t result = 0;
-	if(exe_params->SZ_SIZE_TYPE==4)	
+	if(size_type == 4)	
 		result = bytesToInt_bigEndian(bytes);//4		
 	else
 		result = bytesToLong_bigEndian(bytes);//8	
 	return result;
 }
 
-INLINE void sizeToBytes(unsigned char* outBytes, size_t size)
+INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
 {
-	if(exe_params->SZ_SIZE_TYPE==4)
+	if(size_type == 4)
 		intToBytes_bigEndian(outBytes, (unsigned int)size);//4
 	else
 		longToBytes_bigEndian(outBytes, (unsigned long)size);//8
 }
 
-void convertSZParamsToBytes(sz_params* params, unsigned char* result)
+void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)
 {
 	//unsigned char* result = (unsigned char*)malloc(16);
 	unsigned char buf = 0;
-	//flag1: exe_params->optQuantMode(1bit), dataEndianType(1bit), sysEndianType(1bit), conf_params->szMode (1bit), conf_params->gzipMode (2bits), pwrType (2bits)
-	buf = exe_params->optQuantMode;
+	buf = optQuantMode;
 	buf = (buf << 1) | dataEndianType;
 	buf = (buf << 1) | sysEndianType;
 	buf = (buf << 2) | params->szMode;

deps/SZ/sz/src/TightDataPointStorageD.c

@@ -41,7 +41,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 
 	int same = sameRByte & 0x01;
 	(*this)->isLossless = (sameRByte & 0x10)>>4;
-	exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
+	pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
 	//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
 	pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
 	int errorBoundMode = SZ_ABS;
@@ -53,9 +53,9 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 	int isRegression = (sameRByte >> 7) & 0x01;
 
 	unsigned char dsLengthBytes[8];
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		dsLengthBytes[i] = flatBytes[index++];
-	(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);
+	(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);
 
 	if((*this)->isLossless==1)
 	{
@@ -65,7 +65,6 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 	else if(same==1)
 	{
 		(*this)->allSameData = 1;
-		//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength_double -exe_params->SZ_SIZE_TYPE;
 		(*this)->exactMidBytes = &(flatBytes[index]);
 		return errorBoundMode;
 	}
@@ -74,7 +73,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 		
 	if(isRegression == 1)
 	{
-		(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE;
+		(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - pde_exe->SZ_SIZE_TYPE;
 		(*this)->raBytes = &(flatBytes[index]);
 		return errorBoundMode;
 	}						
@@ -101,17 +100,17 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 		byteBuf[i] = flatBytes[index++];
 	(*this)->realPrecision = bytesToDouble(byteBuf);//8
 
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 	    byteBuf[i] = flatBytes[index++];
-	(*this)->typeArray_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE	
+	(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);
 
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		byteBuf[i] = flatBytes[index++];
-	(*this)->exactDataNum = bytesToSize(byteBuf);// ST
+	(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
 
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		byteBuf[i] = flatBytes[index++];
-	(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
+	(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
 
 	size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
 	if (logicLeadNumBitsNum % 8 == 0)
@@ -132,8 +131,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 
 
     // todo need check length
-	(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 
-			- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
+	(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 
+			- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
 			- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;
 
 	(*this)->leadNumArray = &flatBytes[index];
@@ -211,7 +210,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
 	bytes[k++] = versionNumber;
 	bytes[k++] = sameByte;	//1	byte	
 	
-	convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
+	convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
 	k = k + MetaDataByteLength_double;
 	
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
@@ -234,15 +233,15 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
 	for (i = 0; i < 8; i++)// 8
 		bytes[k++] = realPrecisionBytes[i];	
 
-	sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
+	sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = typeArrayLengthBytes[i];				
 				
-	sizeToBytes(exactLengthBytes, tdps->exactDataNum);
+	sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = exactLengthBytes[i];
 
-	sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
+	sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = exactMidBytesLength[i];
 
@@ -302,7 +301,7 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
 		bytes[k++] = versionNumber;
 		bytes[k++] = sameByte;
 
-		convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
+		convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
 		k = k + MetaDataByteLength_double;
 
 		for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)

deps/SZ/sz/src/TightDataPointStorageF.c

@@ -48,16 +48,16 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
 	// 2 same(1)														      //note that 1000,0000 is reserved for regression tag.
 	int same = sameRByte & 0x01; 											//0000,0001
 	(*this)->isLossless = (sameRByte & 0x10)>>4; 							//0001,0000								//0010,0000
-	exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; 				//0100,0000	
+	pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; 				//0100,0000	
 	int errorBoundMode = SZ_ABS;
     // 3 meta(2)   
 	convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
 	index += MetaDataByteLength;
     // 4 element count(4)
 	unsigned char dsLengthBytes[8];
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		dsLengthBytes[i] = flatBytes[index++];
-	(*this)->dataSeriesLength = bytesToSize(dsLengthBytes);// 4 or 8		
+	(*this)->dataSeriesLength = bytesToSize(dsLengthBytes, pde_exe->SZ_SIZE_TYPE);// 4 or 8		
 	if((*this)->isLossless==1)
 	{
 		//(*this)->exactMidBytes = flatBytes+8;
@@ -66,7 +66,6 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
 	else if(same==1)
 	{
 		(*this)->allSameData = 1;
-		//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
 		(*this)->exactMidBytes = &(flatBytes[index]);
 		return errorBoundMode;
 	}
@@ -76,7 +75,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
     int isRegression = (sameRByte >> 7) & 0x01;
 	if(isRegression == 1)
 	{
-		(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
+		(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE;
 		(*this)->raBytes = &(flatBytes[index]);
 		return errorBoundMode;
 	}			
@@ -101,17 +100,17 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
 		byteBuf[i] = flatBytes[index++];
 	(*this)->realPrecision = bytesToDouble(byteBuf);//8
 	// 10 typeArray_size
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		byteBuf[i] = flatBytes[index++];
-	(*this)->typeArray_size = bytesToSize(byteBuf);// 4		
+	(*this)->typeArray_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// 4		
     // 11 exactNum
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		byteBuf[i] = flatBytes[index++];    
-	(*this)->exactDataNum = bytesToSize(byteBuf);// ST
+	(*this)->exactDataNum = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// ST
     // 12 mid size
-	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
+	for (i = 0; i < pde_exe->SZ_SIZE_TYPE; i++)
 		byteBuf[i] = flatBytes[index++];
-	(*this)->exactMidBytes_size = bytesToSize(byteBuf);// ST
+	(*this)->exactMidBytes_size = bytesToSize(byteBuf, pde_exe->SZ_SIZE_TYPE);// STqq
     
 	// calc leadNumArray_size
 	size_t logicLeadNumBitsNum = (*this)->exactDataNum * 2;
@@ -141,8 +140,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
 	(*this)->residualMidBits = &flatBytes[index];
 
     // calc residualMidBits_size
-	(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 
-			- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE
+	(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 
+			- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
 			- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - (*this)->typeArray_size;	
 	
 	
@@ -211,7 +210,7 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
 	// 2 same
 	bytes[k++] = sameByte;	//1	byte
 	// 3 meta
-	convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
+	convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
 	k = k + MetaDataByteLength;
 	// 4 element count
 	for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
@@ -235,15 +234,15 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
 	for (i = 0; i < 8; i++)// 8
 		bytes[k++] = realPrecisionBytes[i];		
    // 10 typeArray size
-	sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size);
+	sizeToBytes(typeArrayLengthBytes, tdps->typeArray_size, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = typeArrayLengthBytes[i];			
     // 11 exactDataNum  leadNum calc by this , so not save leadNum
-	sizeToBytes(exactLengthBytes, tdps->exactDataNum);
+	sizeToBytes(exactLengthBytes, tdps->exactDataNum, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = exactLengthBytes[i];
     // 12 Mid size
-	sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size);
+	sizeToBytes(exactMidBytesLength, tdps->exactMidBytes_size, exe_params->SZ_SIZE_TYPE);
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST
 		bytes[k++] = exactMidBytesLength[i];
 	// 13 typeArray	
@@ -306,7 +305,7 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
 		// 2 same flag 1 bytes
 		bytes[k++] = sameByte;
 		// 3 metaData 26 bytes
-		convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
+		convertSZParamsToBytes(confparams_cpr, &(bytes[k]), exe_params->optQuantMode);
 		k = k + MetaDataByteLength;
 		// 4 data Length 4 or 8 bytes	
 		for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)

deps/SZ/sz/src/sz_double.c

@@ -229,10 +229,10 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
 	else
 		newByteData[k++] = 80;	//01010000: 01000000 indicates the SZ_SIZE_TYPE=8
 
-	convertSZParamsToBytes(confparams_cpr, &(newByteData[k]));
+	convertSZParamsToBytes(confparams_cpr, &(newByteData[k]), exe_params->optQuantMode);
 	k = k + MetaDataByteLength_double;
 
-	sizeToBytes(dsLengthBytes,dataLength);
+	sizeToBytes(dsLengthBytes,dataLength, exe_params->SZ_SIZE_TYPE);
 	for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8
 		newByteData[k++] = dsLengthBytes[i];
 

deps/SZ/sz/src/szd_double.c

@@ -25,7 +25,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
 	//unsigned char* tmpBytes;
 	size_t targetUncompressSize = dataLength <<3; //i.e., *8
 	//tmpSize must be "much" smaller than dataLength
-	size_t i, tmpSize = 12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
+	size_t i, tmpSize = 12+MetaDataByteLength_double+8;
 	unsigned char* szTmpBytes = NULL;
 	bool needFree = false;
 
@@ -50,7 +50,7 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
 		{
 			if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
 				targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES; 			
-			tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE);			
+			tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+8);			
 	        needFree = true;	
 		}
 	}
@@ -73,11 +73,11 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
 		// *newData = (double*)malloc(doubleSize*dataLength); comment by tickduan
 		if(sysEndianType==BIG_ENDIAN_SYSTEM)
 		{
-			memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize);
+			memcpy(newData, szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE, dataLength*doubleSize);
 		}
 		else
 		{
-			unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE;
+			unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+pde_exe->SZ_SIZE_TYPE;
 			for(i=0;i<dataLength;i++,p+=doubleSize)
 				newData[i] = bytesToDouble(p);
 		}		

deps/SZ/sz/src/szd_float.c

@@ -57,7 +57,7 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
 		{
 			if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
 				targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES; 
-			tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);//		(unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
+			tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+8);
 	        needFree = true;
 		}
 	}
@@ -81,11 +81,11 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
 		//*newData = (float*)malloc(floatSize*dataLength);  comment by tickduan
 		if(sysEndianType==BIG_ENDIAN_SYSTEM)
 		{
-			memcpy(newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*floatSize);
+			memcpy(newData, szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE, dataLength*floatSize);
 		}
 		else
 		{
-			unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
+			unsigned char* p = szTmpBytes+4+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
 			for(i=0;i<dataLength;i++,p+=floatSize)
 				newData[i] = bytesToFloat(p);
 		}