Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
提交 3e8cfacd 编写于 作者: T tickduan
浏览文件
操作

reduce head meta from 36 to 2 bytes

上级 6438e193
隐藏空白更改
内联 并排
Showing with 117 addition and 264 deletion
deps/SZ/sz/include/conf.h
浏览文件 @ 3e8cfacd
......@@ -20,7 +20,7 @@ extern "C" {
void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile);
int SZ_LoadConf(const char* sz_cfgFile);
int checkVersion(char* version);
int checkVersion(unsigned char version);
int computeVersion(int major, int minor, int revision);
int checkVersion2(char* version);
......
deps/SZ/sz/include/dataCompression.h
浏览文件 @ 3e8cfacd
......@@ -82,7 +82,6 @@ int getPredictionCoefficients(int layers, int dimension, int **coeff_array, int
int computeBlockEdgeSize_3D(int segmentSize);
int computeBlockEdgeSize_2D(int segmentSize);
int initRandomAccessBytes(unsigned char* raBytes);
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData);
int compressExactDataArray_float(float* oriData, double precision, size_t nbEle, unsigned char** leadArray, unsigned char** midArray, unsigned char** resiArray,
......
deps/SZ/sz/include/defines.h
浏览文件 @ 3e8cfacd
......@@ -11,10 +11,8 @@
#define _SZ_DEFINES_H
#define SZ_VERNUM 0x0200
#define SZ_VER_MAJOR 2
#define SZ_VER_MINOR 1
#define SZ_VER_BUILD 12
#define SZ_VER_REVISION 0
#define SZ_VER_MAJOR 1 // curretn version
#define PASTRI 103
#define HZ 102 //deprecated
......@@ -95,9 +93,10 @@
#define SZ_DESTROY_WHOLE_VARSET 1
#define GROUP_COUNT 16 //2^{16}=65536
#define MetaDataByteLength 28
#define MetaDataByteLength_double 36 //meta data length for double type
// metaData remove some by tickduan
#define MetaDataByteLength 2 // original is 28 bytes
#define MetaDataByteLength_double 2 // original is 36 bytes
#define numOfBufferedSteps 1 //the number of time steps in the buffer
......
deps/SZ/sz/include/sz.h
浏览文件 @ 3e8cfacd
......@@ -142,7 +142,7 @@ typedef union lfloat
typedef struct sz_metadata
{
int versionNumber[3]; //only used for checking the version by calling SZ_GetMetaData()
unsigned char ver; //only used for checking the version by calling SZ_GetMetaData()
int isConstant; //only used for checking if the data are constant values by calling SZ_GetMetaData()
int isLossless; //only used for checking if the data compression was lossless, used only by calling SZ_GetMetaData()
int sizeType; //only used for checking whether the size type is "int" or "long" in the compression, used only by calling SZ_GetMetaData()
......@@ -165,7 +165,7 @@ typedef struct sz_tsc_metainfo
} sz_tsc_metadata;
extern int versionNumber[4];
extern unsigned char versionNumber;
//-------------------key global variables--------------
extern int dataEndianType; //*endian type of the data read from disk
......
deps/SZ/sz/include/sz_float.h
浏览文件 @ 3e8cfacd
......@@ -29,8 +29,6 @@ unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataL
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
......
deps/SZ/sz/src/ByteToolkit.c
浏览文件 @ 3e8cfacd
......@@ -898,65 +898,66 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
//buf = (buf << 2) | params->pwr_type; //deprecated
result[0] = buf;
//sampleDistance; //2 bytes
int16ToBytes_bigEndian(&result[1], params->sampleDistance);
//sampleDistance; //2 bytes
//int16ToBytes_bigEndian(&result[1], params->sampleDistance);
//conf_params->predThreshold; // 2 bytes
short tmp2 = params->predThreshold * 10000;
int16ToBytes_bigEndian(&result[3], tmp2);
//short tmp2 = params->predThreshold * 10000;
//int16ToBytes_bigEndian(&result[3], tmp2);
//errorBoundMode; //4bits(0.5 byte)
result[5] = params->errorBoundMode;
result[1] = params->errorBoundMode;
//data type (float, double, int8, int16, ....) //10 choices, so 4 bits
result[5] = (result[5] << 4) | (params->dataType & 0x17);
result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[5]: abs_err_bound or psnr //4 bytes
//result[9]: rel_bound_ratio or pwr_err_bound//4 bytes
/*
switch(params->errorBoundMode)
{
case SZ_ABS:
floatToBytes(&result[6], (float)(params->absErrBound)); //big_endian
memset(&result[10], 0, 4);
floatToBytes(&result[6-4], (float)(params->absErrBound)); //big_endian
memset(&result[10-4], 0, 4);
break;
case REL:
memset(&result[6], 0, 4);
floatToBytes(&result[10], (float)(params->relBoundRatio)); //big_endian
memset(&result[6-4], 0, 4);
floatToBytes(&result[10-4], (float)(params->relBoundRatio)); //big_endian
break;
case ABS_AND_REL:
case ABS_OR_REL:
floatToBytes(&result[6], (float)(params->absErrBound));
floatToBytes(&result[10], (float)(params->relBoundRatio)); //big_endian
floatToBytes(&result[6-4], (float)(params->absErrBound));
floatToBytes(&result[10-4], (float)(params->relBoundRatio)); //big_endian
break;
case PSNR:
floatToBytes(&result[6], (float)(params->psnr));
memset(&result[9], 0, 4);
floatToBytes(&result[6-4], (float)(params->psnr));
memset(&result[9-4], 0, 4);
break;
case ABS_AND_PW_REL:
case ABS_OR_PW_REL:
floatToBytes(&result[6], (float)(params->absErrBound));
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian
floatToBytes(&result[6-4], (float)(params->absErrBound));
floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break;
case REL_AND_PW_REL:
case REL_OR_PW_REL:
floatToBytes(&result[6], (float)(params->relBoundRatio));
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian
floatToBytes(&result[6-4], (float)(params->relBoundRatio));
floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break;
case PW_REL:
memset(&result[6], 0, 4);
floatToBytes(&result[10], (float)(params->pw_relBoundRatio)); //big_endian
memset(&result[6-4], 0, 4);
floatToBytes(&result[10-4], (float)(params->pw_relBoundRatio)); //big_endian
break;
}
*/
//compressor
result[14] = (unsigned char)params->sol_ID;
//int16ToBytes_bigEndian(&result[14], (short)(params->segment_size));
//result[14-4] = (unsigned char)params->sol_ID;
/* remove fmin fmax intervals
if(exe_params->optQuantMode==1)
int32ToBytes_bigEndian(&result[16], params->max_quant_intervals);
else
int32ToBytes_bigEndian(&result[16], params->quantization_intervals);
int32ToBytes_bigEndian(&result[16], params->quantization_intervals);
if(params->dataType==SZ_FLOAT)
{
......@@ -968,6 +969,7 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
doubleToBytes(&result[20], params->dmin);
doubleToBytes(&result[28], params->dmax);
}
*/
}
......@@ -996,48 +998,45 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
//params->pwr_type = (flag1 & 0x03) >> 0;
params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]);
params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
//params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]);
//params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
params->dataType = bytes[5] & 0x07;
params->errorBoundMode = (bytes[5] & 0xf0) >> 4;
params->dataType = bytes[1] & 0x07;
params->errorBoundMode = (bytes[1] & 0xf0) >> 4;
/*
switch(params->errorBoundMode)
{
case SZ_ABS:
params->absErrBound = bytesToFloat(&bytes[6]);
params->absErrBound = bytesToFloat(&bytes[6-4]);
break;
case REL:
params->relBoundRatio = bytesToFloat(&bytes[10]);
params->relBoundRatio = bytesToFloat(&bytes[10-4]);
break;
case ABS_AND_REL:
case ABS_OR_REL:
params->absErrBound = bytesToFloat(&bytes[6]);
params->relBoundRatio = bytesToFloat(&bytes[10]);
params->absErrBound = bytesToFloat(&bytes[6-4]);
params->relBoundRatio = bytesToFloat(&bytes[10-4]);
break;
case PSNR:
params->psnr = bytesToFloat(&bytes[6]);
params->psnr = bytesToFloat(&bytes[6-4]);
break;
case ABS_AND_PW_REL:
case ABS_OR_PW_REL:
params->absErrBound = bytesToFloat(&bytes[6]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10]);
params->absErrBound = bytesToFloat(&bytes[6-4]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
break;
case REL_AND_PW_REL:
case REL_OR_PW_REL:
params->relBoundRatio = bytesToFloat(&bytes[6]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10]);
params->relBoundRatio = bytesToFloat(&bytes[6-4]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
break;
case PW_REL:
params->pw_relBoundRatio = bytesToFloat(&bytes[10]);
params->pw_relBoundRatio = bytesToFloat(&bytes[10-4]);
}
//segment_size // 2 bytes
//params->segment_size = bytesToInt16_bigEndian(&bytes[14]);
params->sol_ID = (int)(bytes[14]);
params->sol_ID = (int)(bytes[14-4]);
if(pde_exe->optQuantMode==1)
{
params->max_quant_intervals = bytesToInt32_bigEndian(&bytes[16]);
......@@ -1059,5 +1058,6 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
params->dmin = bytesToDouble(&bytes[20]);
params->dmax = bytesToDouble(&bytes[28]);
}
*/
}
deps/SZ/sz/src/TightDataPointStorageD.c
浏览文件 @ 3e8cfacd
......@@ -57,24 +57,20 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
new_TightDataPointStorageD_Empty(this);
size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
char version[3];
for (i = 0; i < 3; i++)
version[i] = flatBytes[index++]; //3
unsigned char version = flatBytes[index++]; //3
unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion2(version)!=1)
if(checkVersion(version)!=1)
{
//wrong version
printf("Wrong version: \nCompressed-data version (%d.%d.%d)\n",version[0], version[1], version[2]);
printf("Current sz version: (%d.%d.%d)\n", versionNumber[0], versionNumber[1], versionNumber[2]);
printf("Please double-check if the compressed data (or file) is correct.\n");
exit(0);
printf("Wrong version: \nCompressed-data %s . program version=%d\n", version, versionNumber);
return 0;
}
int same = sameRByte & 0x01;
(*this)->isLossless = (sameRByte & 0x10)>>4;
int isPW_REL = (sameRByte & 0x20)>>5;
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
pde_params->protectValueRange = (sameRByte & 0x04)>>2;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;
int errorBoundMode = SZ_ABS;
if(isPW_REL)
......@@ -213,7 +209,8 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
if ((*this)->rtypeArray != NULL)
{
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
// -3 -2 modify to -1 -1
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
......@@ -222,7 +219,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
}
else
{
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
}
......@@ -381,8 +378,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
for(i = 0;i<3;i++)//3 bytes
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1 byte
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
......@@ -483,8 +479,7 @@ void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned ch
unsigned char pwrErrBoundBytes_sizeBytes[4];
unsigned char max_quant_intervals_Bytes[4];
for(i = 0;i<3;i++)//3
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
bytes[k++] = sameByte; //1
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
......@@ -595,15 +590,14 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
if(tdps->allSameData==1)
{
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // comment by tickduan
if(totalByteLength >= tdps->dataSeriesLength * sizeof(double))
{
return false;
}
for (i = 0; i < 3; i++)//3
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
......@@ -677,14 +671,13 @@ void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned c
sameByte = (unsigned char) (sameByte | 0x40); //0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit
if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
//if(confparams_cpr->protectValueRange)
// sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData==1)
{
size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
size_t totalByteLength = 1 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
for (i = 0; i < 3; i++)//3
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
......
deps/SZ/sz/src/TightDataPointStorageF.c
浏览文件 @ 3e8cfacd
......@@ -56,17 +56,13 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
new_TightDataPointStorageF_Empty(this);
size_t i, index = 0;
size_t pwrErrBoundBytes_size = 0, segmentL = 0, radExpoL = 0, pwrErrBoundBytesL = 0;
char version[3];
for (i = 0; i < 3; i++)
version[i] = flatBytes[index++]; //3
unsigned char version = flatBytes[index++]; //1
unsigned char sameRByte = flatBytes[index++]; //1
if(checkVersion2(version)!=1)
if(checkVersion(version) != 1)
{
//wrong version
printf("Wrong version: \nCompressed-data version (%d.%d.%d)\n",version[0], version[1], version[2]);
printf("Current sz version: (%d.%d.%d)\n", versionNumber[0], versionNumber[1], versionNumber[2]);
printf("Please double-check if the compressed data (or file) is correct.\n");
exit(0);
printf("Wrong version: \nCompressed-data version =%d , program version=%d \n", version, versionNumber);
return 0;
}
//note that 1000,0000 is reserved for regression tag.
int same = sameRByte & 0x01; //0000,0001
......@@ -76,7 +72,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
//pde_params->randomAccess = (sameRByte & 0x02) >> 1;
//pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it)
pde_params->protectValueRange = (sameRByte & 0x04)>>2;
//pde_params->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000
......@@ -108,7 +104,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
else if(same==1)
{
(*this)->allSameData = 1;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
//size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 1 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE;
(*this)->exactMidBytes = &(flatBytes[index]);
return errorBoundMode;
}
......@@ -116,7 +112,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->allSameData = 0;
if(isRegression == 1)
{
(*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE;
(*this)->raBytes = &(flatBytes[index]);
return errorBoundMode;
}
......@@ -214,7 +210,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
if ((*this)->rtypeArray != NULL)
{
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
// modify from -3 -1 to -1 -1 tickduan
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 4 - (*this)->rtypeArray_size
- minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size
- (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
......@@ -223,7 +220,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
}
else
{
(*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 4 - 1 - 8
- exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size
- (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1;
}
......@@ -384,15 +381,14 @@ void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes
unsigned char max_quant_intervals_Bytes[4];
// 1 version
for(i = 0;i<3;i++)//3 bytes
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
// 2 same
bytes[k++] = sameByte; //1 byte
// 3 meta
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
k = k + MetaDataByteLength;
// 4 element count
for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
for(i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 bytes
bytes[k++] = dsLengthBytes[i];
intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
// 5 max_quant_intervals length
......@@ -495,15 +491,15 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
sameByte = (unsigned char) (sameByte | 0x40); // 0100,0000, the 6th bit
if(confparams_cpr->errorBoundMode == PW_REL && confparams_cpr->accelerate_pw_rel_compression)
sameByte = (unsigned char) (sameByte | 0x08); //0000,1000
if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
//if(confparams_cpr->protectValueRange)
// sameByte = (unsigned char) (sameByte | 0x04); //0000,0100
if(tdps->allSameData == 1 )
{
//
// same format
//
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); // not need malloc comment by tickduan
// check output buffer enough
if(totalByteLength >= tdps->dataSeriesLength * sizeof(float) )
......@@ -512,9 +508,8 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
return false;
}
// 1 version 3 bytes
for (i = 0; i < 3; i++)//3
bytes[k++] = versionNumber[i];
// 1 version 1 byte
bytes[k++] = versionNumber;
// 2 same flag 1 bytes
bytes[k++] = sameByte;
// 3 metaData 26 bytes
......@@ -545,7 +540,8 @@ bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* b
minLogValueSize = 4;
}
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
// version(1) + samebyte(1)
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
......@@ -596,11 +592,10 @@ void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned ch
if(tdps->allSameData==1)
{
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size;
//*bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength);
for (i = 0; i < 3; i++)//3
bytes[k++] = versionNumber[i];
bytes[k++] = versionNumber;
bytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(bytes[k]));
......@@ -624,7 +619,7 @@ void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned ch
pwrBoundArrayL = 4;
}
size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
size_t totalByteLength = 1 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 4 + 1 + 8
+ exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE
+ tdps->typeArray_size + tdps->leadNumArray_size
+ tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size;
......
deps/SZ/sz/src/conf.c
浏览文件 @ 3e8cfacd
......@@ -120,7 +120,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
confparams_cpr->errorBoundMode = SZ_ABS;
confparams_cpr->psnr = 90;
confparams_cpr->absErrBound = 1E-20;
confparams_cpr->absErrBound = 1E-8;
confparams_cpr->relBoundRatio = 1E-8;
confparams_cpr->accelerate_pw_rel_compression = 1;
......@@ -410,33 +410,11 @@ int SZ_LoadConf(const char* sz_cfgFile) {
return SZ_SUCCESS;
}
int checkVersion(char* version)
int checkVersion(unsigned char version)
{
int i = 0;
for(;i<3;i++)
if(version[i]!=versionNumber[i])
return 0;
return 1;
return version <= versionNumber;
}
inline int computeVersion(int major, int minor, int revision)
{
return major*10000+minor*100+revision;
}
int checkVersion2(char* version)
{
int major = version[0];
int minor = version[1];
int revision = version[2];
int preVersion = 20108;
int givenVersion = computeVersion(major, minor, revision);
//int currentVersion = computeVersion(SZ_VER_MAJOR, SZ_VER_MINOR, SZ_VER_REVISION);
if(givenVersion < preVersion) //only for old version (older than 2.1.8), we will check whether version is consistent exactly.
return checkVersion(version);
return 1;
}
void initSZ_TSC()
{
......
deps/SZ/sz/src/dataCompression.c
浏览文件 @ 3e8cfacd
......@@ -678,32 +678,6 @@ int computeBlockEdgeSize_3D(int segmentSize)
//return (int)(pow(segmentSize, 1.0/3)+1);
}
//convert random-access version based bytes to output bytes
int initRandomAccessBytes(unsigned char* raBytes)
{
int k = 0, i = 0;
for (i = 0; i < 3; i++)//3
raBytes[k++] = versionNumber[i];
int sameByte = 0x80; //indicating this is regression-based compression mode
if(exe_params->SZ_SIZE_TYPE==8)
sameByte = (unsigned char) (sameByte | 0x40); // 01000000, the 6th bit
if(confparams_cpr->randomAccess)
sameByte = (unsigned char) (sameByte | 0x02); // 00000010, random access
//sameByte = sameByte | (confparams_cpr->szMode << 1);
if(confparams_cpr->protectValueRange)
sameByte = (unsigned char) (sameByte | 0x04); //00000100, protect value range
raBytes[k++] = sameByte;
convertSZParamsToBytes(confparams_cpr, &(raBytes[k]));
if(confparams_cpr->dataType==SZ_FLOAT)
k = k + MetaDataByteLength;
else if(confparams_cpr->dataType==SZ_DOUBLE)
k = k + MetaDataByteLength_double;
return k;
}
//The following functions are float-precision version of dealing with the unpredictable data points
int generateLossyCoefficients_float(float* oriData, double precision, size_t nbEle, int* reqBytesLength, int* resiBitsLength, float* medianValue, float* decData)
{
......
deps/SZ/sz/src/sz.c
浏览文件 @ 3e8cfacd
......@@ -25,7 +25,7 @@
//#include "CurveFillingCompressStorage.h"
int versionNumber[4] = {SZ_VER_MAJOR,SZ_VER_MINOR,SZ_VER_BUILD,SZ_VER_REVISION};
unsigned char versionNumber = SZ_VER_MAJOR;
int SZ_SIZE_TYPE_DEFUALT = 4;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
......
deps/SZ/sz/src/sz_double.c
浏览文件 @ 3e8cfacd
......@@ -232,13 +232,12 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un
{
int doubleSize = sizeof(double);
size_t k = 0, i;
size_t totalByteLength = 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
size_t totalByteLength = 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);
unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i];
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)//1
newByteData[k++] = 16; //00010000
......
deps/SZ/sz/src/sz_double_pwr.c
浏览文件 @ 3e8cfacd
......@@ -320,13 +320,12 @@ size_t dataLength, size_t *outSize, double min, double max)
{
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 3 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength);
size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
//*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i];
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)
{
......@@ -650,7 +649,7 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
......@@ -712,7 +711,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByt
free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
if(*outSize>1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
......@@ -746,7 +745,7 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char*
free(signs);
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageD(tdps);
......
deps/SZ/sz/src/sz_float.c
浏览文件 @ 3e8cfacd
......@@ -103,7 +103,7 @@ unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, doub
}
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f)
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f)
{
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
......@@ -259,41 +259,6 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_
return tdps;
}
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize)
{
int floatSize=sizeof(float);
size_t k = 0, i;
size_t totalByteLength = 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);
unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i];
if(exe_params->SZ_SIZE_TYPE==4)//1
newByteData[k++] = 16; //00010000
else
newByteData[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8
convertSZParamsToBytes(confparams_cpr, &(newByteData[k]));
k = k + MetaDataByteLength;
sizeToBytes(dsLengthBytes,dataLength); //SZ_SIZE_TYPE: 4 or 8
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else
{
unsigned char* p = newByteData+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
for(i=0;i<dataLength;i++,p+=floatSize)
floatToBytes(p, oriData[i]);
}
*outSize = totalByteLength;
}
// compress core algorithm if success return true else return false
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f)
......@@ -312,9 +277,8 @@ bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float
}
// check compressed size large than original
if(*outSize > 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
if(*outSize > 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
{
//SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
return false;
}
......@@ -610,8 +574,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize >= dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
}
//cost_end(" sz_first_compress");
......
deps/SZ/sz/src/sz_float_pwr.c
浏览文件 @ 3e8cfacd
......@@ -151,7 +151,7 @@ unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength,
}
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision,
size_t dataLength, size_t *outSize, float min, float max)
size_t dataLength, size_t *outSize, float min, float max)
{
size_t pwrLength = dataLength%confparams_cpr->segment_size==0?dataLength/confparams_cpr->segment_size:dataLength/confparams_cpr->segment_size+1;
float* pwrErrBound = (float*)malloc(sizeof(float)*pwrLength);
......@@ -309,13 +309,13 @@ size_t dataLength, size_t *outSize, float min, float max)
{
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 3 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength);
size_t totalByteLength = 1 + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
//*newByteData = (unsigned char*)malloc(totalByteLength); comment by tickduan
unsigned char dsLengthBytes[exe_params->SZ_SIZE_TYPE];
intToBytes_bigEndian(dsLengthBytes, dataLength);//4
for (i = 0; i < 3; i++)//3
newByteData[k++] = versionNumber[i];
newByteData[k++] = versionNumber;
if(exe_params->SZ_SIZE_TYPE==4)
{
......@@ -328,7 +328,6 @@ size_t dataLength, size_t *outSize, float min, float max)
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//4 or 8
newByteData[k++] = dsLengthBytes[i];
if(sysEndianType==BIG_ENDIAN_SYSTEM)
memcpy(newByteData+4+exe_params->SZ_SIZE_TYPE, oriData, dataLength*floatSize);
else
......@@ -639,9 +638,6 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}
......@@ -702,9 +698,7 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByte
free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}
......@@ -737,8 +731,6 @@ void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char*
free(signs);
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}
deps/SZ/sz/src/szd_double.c
浏览文件 @ 3e8cfacd
......@@ -64,8 +64,9 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
}
else
szTmpBytes = cmpBytes;
pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
// calc postion
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to double array.
TightDataPointStorageD* tdps;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
......@@ -102,23 +103,6 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
}
}
if(pde_params->protectValueRange)
{
double* nd = newData;
double min = pde_params->dmin;
double max = pde_params->dmax;
for(i=0;i<dataLength;i++)
{
double v = nd[i];
if(v <= max && v >= min)
continue;
if(v < min)
nd[i] = min;
else if(v > max)
nd[i] = max;
}
}
free_TightDataPointStorageD2(tdps);
if(pde_params->szMode!=SZ_BEST_SPEED && cmpSize!=12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE)
free(szTmpBytes);
......
deps/SZ/sz/src/szd_float.c
浏览文件 @ 3e8cfacd
......@@ -91,7 +91,8 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
else
szTmpBytes = cmpBytes;
pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
// calc sol_ID
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(1bytes), samebyte(1byte), [14-4]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to data array.
TightDataPointStorageF* tdps;
......@@ -113,10 +114,6 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
newData[i] = bytesToFloat(p);
}
}
else if(pde_params->sol_ID==SZ_Transpose)
{
getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //pde_params->sol_ID==SZ
{
if(tdps->raBytes_size > 0) //v2.0
......@@ -130,22 +127,6 @@ int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes,
}
//cost_start_();
if(pde_params->protectValueRange)
{
float* nd = newData;
float min = pde_params->fmin;
float max = pde_params->fmax;
for(i=0;i<dataLength;i++)
{
float v = nd[i];
if(v <= max && v >= min)
continue;
if(v < min)
nd[i] = min;
else if(v > max)
nd[i] = max;
}
}
//cost_end_();
//printf("totalCost_=%f\n", totalCost_);
free_TightDataPointStorageF2(tdps);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或