提交 f3834b67 编写于 作者: T tickduan

fix muli-thread

上级 a0996872
......@@ -10,6 +10,8 @@
#ifndef _TightDataPointStorageD_H
#define _TightDataPointStorageD_H
#include "pub.h"
#ifdef __cplusplus
extern "C" {
#endif
......@@ -64,7 +66,7 @@ typedef struct TightDataPointStorageD
} TightDataPointStorageD;
void new_TightDataPointStorageD_Empty(TightDataPointStorageD **self);
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength);
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
void new_TightDataPointStorageD(TightDataPointStorageD **self,
size_t dataSeriesLength, size_t exactDataNum,
......
......@@ -15,6 +15,7 @@ extern "C" {
#endif
#include <stdio.h>
#include "pub.h"
typedef struct TightDataPointStorageF
{
......@@ -66,7 +67,7 @@ typedef struct TightDataPointStorageF
} TightDataPointStorageF;
void new_TightDataPointStorageF_Empty(TightDataPointStorageF **self);
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength);
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **self, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params);
void new_TightDataPointStorageF(TightDataPointStorageF **self,
size_t dataSeriesLength, size_t exactDataNum,
......
......@@ -15,6 +15,7 @@ extern "C" {
#endif
#include <stdio.h>
#include "pub.h"
typedef struct TightDataPointStorageI
{
......
/**
* @file sz.h
* @author Sheng Di
* @date April, 2015
* @brief Header file for the whole compressor.
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _PUB_H
#define _PUB_H
#include <stdio.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* array meta data and compression parameters for SZ_Init_Params() */
typedef struct sz_params
{
int dataType;
unsigned int max_quant_intervals; //max number of quantization intervals for quantization
unsigned int quantization_intervals;
unsigned int maxRangeRadius;
int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri)
int losslessCompressor;
int sampleDistance; //2 bytes
float predThreshold; // 2 bytes
int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
int errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
double absErrBound; //absolute error bound
double relBoundRatio; //value range based relative error bound ratio
double psnr; //PSNR
double normErr;
double pw_relBoundRatio; //point-wise relative error bound
int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated)
int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
int protectValueRange; //0 or 1
float fmin, fmax;
double dmin, dmax;
int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
int predictionMode;
int accelerate_pw_rel_compression;
int plus_bits;
int randomAccess;
int withRegression;
} sz_params;
typedef struct sz_exedata
{
char optQuantMode; //opt Quantization (0: fixed ; 1: optimized)
int intvCapacity; // the number of intervals for the linear-scaling quantization
int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
} sz_exedata;
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _PUB_H ----- */
......@@ -14,6 +14,7 @@
#include <stdint.h>
#include <sys/time.h> /* For gettimeofday(), in microseconds */
#include <time.h> /* For time(), in seconds */
#include "pub.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
......@@ -158,42 +159,6 @@ typedef union lfloat
unsigned char byte[4];
} lfloat;
/* array meta data and compression parameters for SZ_Init_Params() */
typedef struct sz_params
{
int dataType;
unsigned int max_quant_intervals; //max number of quantization intervals for quantization
unsigned int quantization_intervals;
unsigned int maxRangeRadius;
int sol_ID;// it's SZ or SZ_Transpose, unless the setting is PASTRI compression mode (./configure --enable-pastri)
int losslessCompressor;
int sampleDistance; //2 bytes
float predThreshold; // 2 bytes
int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression
int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION
int errorBoundMode; //4bits (0.5byte), //ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR
double absErrBound; //absolute error bound
double relBoundRatio; //value range based relative error bound ratio
double psnr; //PSNR
double normErr;
double pw_relBoundRatio; //point-wise relative error bound
int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated)
int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio
int protectValueRange; //0 or 1
float fmin, fmax;
double dmin, dmax;
int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep
int predictionMode;
int accelerate_pw_rel_compression;
int plus_bits;
int randomAccess;
int withRegression;
} sz_params;
typedef struct sz_metadata
{
......@@ -206,13 +171,6 @@ typedef struct sz_metadata
struct sz_params* conf_params; //configuration parameters
} sz_metadata;
typedef struct sz_exedata
{
char optQuantMode; //opt Quantization (0: fixed ; 1: optimized)
int intvCapacity; // the number of intervals for the linear-scaling quantization
int intvRadius; // the number of intervals for the radius of the quantization range (intvRadius=intvCapacity/2)
unsigned int SZ_SIZE_TYPE; //the length (# bytes) of the size_t in the system at runtime //4 or 8: sizeof(size_t)
} sz_exedata;
/*We use a linked list to maintain time-step meta info for time-step based compression*/
typedef struct sz_tsc_metainfo
......@@ -296,7 +254,7 @@ void SZ_Create_ParamsExe(sz_params** conf_params, sz_exedata** exe_params);
void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength, void* decompressed_array, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
sz_metadata* SZ_getMetadata(unsigned char* bytes);
sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe);
void SZ_printMetadata(sz_metadata* metadata);
......@@ -320,7 +278,7 @@ void SZ_decompress_ts(unsigned char *bytes, size_t byteLength);
void SZ_Finalize();
void convertSZParamsToBytes(sz_params* params, unsigned char* result);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params);
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe);
unsigned char* SZ_compress_customize(const char* appName, void* userPara, int dataType, void* data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, size_t *outSize, int *status);
......
......@@ -25,7 +25,7 @@ void decompressDataSeries_double_1D_MSST19(double** data, size_t dataSeriesLengt
void decompressDataSeries_double_2D_MSST19(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps);
void decompressDataSeries_double_3D_MSST19(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps);
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data);
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
void getSnapshotData_double_2D(double** data, size_t r1, size_t r2, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data);
void getSnapshotData_double_3D(double** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data);
void getSnapshotData_double_4D(double** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data);
......@@ -34,7 +34,7 @@ void decompressDataSeries_double_3D_nonblocked_with_blocked_regression(double**
size_t decompressDataSeries_double_3D_RA_block(double * data, double mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, double * unpredictable_data);
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data);
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
#ifdef __cplusplus
}
......
......@@ -25,7 +25,7 @@ void decompressDataSeries_float_1D_MSST19(float** data, size_t dataSeriesLength,
void decompressDataSeries_float_2D_MSST19(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps);
void decompressDataSeries_float_3D_MSST19(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps);
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data);
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
void getSnapshotData_float_2D(float** data, size_t r1, size_t r2, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data);
void getSnapshotData_float_3D(float** data, size_t r1, size_t r2, size_t r3, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data);
void getSnapshotData_float_4D(float** data, size_t r1, size_t r2, size_t r3, size_t r4, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data);
......@@ -33,7 +33,7 @@ void getSnapshotData_float_4D(float** data, size_t r1, size_t r2, size_t r3, siz
size_t decompressDataSeries_float_1D_RA_block(float * data, float mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data);
size_t decompressDataSeries_float_2D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t block_dim_0, size_t block_dim_1, double realPrecision, int * type, float * unpredictable_data);
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data);
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
size_t decompressDataSeries_float_3D_RA_block(float * data, float mean, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, double realPrecision, int * type, float * unpredictable_data);
......
......@@ -971,10 +971,10 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
}
void convertBytesToSZParams(unsigned char* bytes, sz_params* params)
void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata* pde_exe)
{
unsigned char flag1 = bytes[0];
exe_params->optQuantMode = (flag1 & 0x40) >> 6;
pde_exe->optQuantMode = (flag1 & 0x40) >> 6;
dataEndianType = (flag1 & 0x20) >> 5;
//sysEndianType = (flag1 & 0x10) >> 4;
......@@ -1038,7 +1038,7 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params)
//params->segment_size = bytesToInt16_bigEndian(&bytes[14]);
params->sol_ID = (int)(bytes[14]);
if(exe_params->optQuantMode==1)
if(pde_exe->optQuantMode==1)
{
params->max_quant_intervals = bytesToInt32_bigEndian(&bytes[16]);
params->quantization_intervals = 0;
......
......@@ -52,7 +52,7 @@ void new_TightDataPointStorageD_Empty(TightDataPointStorageD **this)
}
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength)
int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
{
new_TightDataPointStorageD_Empty(this);
size_t i, index = 0;
......@@ -92,7 +92,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
}
convertBytesToSZParams(&(flatBytes[index]), confparams_dec);
convertBytesToSZParams(&(flatBytes[index]), confparams_dec, pde_exe);
index += MetaDataByteLength_double;
......
......@@ -51,7 +51,7 @@ void new_TightDataPointStorageF_Empty(TightDataPointStorageF **this)
(*this)->raBytes_size = 0;
}
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength)
int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsigned char* flatBytes, size_t flatBytesLength, sz_exedata* pde_exe, sz_params* pde_params)
{
new_TightDataPointStorageF_Empty(this);
size_t i, index = 0;
......@@ -73,12 +73,12 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->isLossless = (sameRByte & 0x10)>>4; //0001,0000
int isPW_REL = (sameRByte & 0x20)>>5; //0010,0000
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4; //0100,0000
//confparams_dec->randomAccess = (sameRByte & 0x02) >> 1;
//confparams_dec->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it)
//pde_params->randomAccess = (sameRByte & 0x02) >> 1;
//pde_params->szMode = (sameRByte & 0x06) >> 1; //0000,0110 (in fact, this szMode could be removed because convertSZParamsToBytes will overwrite it)
confparams_dec->protectValueRange = (sameRByte & 0x04)>>2;
pde_params->protectValueRange = (sameRByte & 0x04)>>2;
confparams_dec->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000
pde_params->accelerate_pw_rel_compression = (sameRByte & 0x08) >> 3;//0000,1000
int errorBoundMode = ABS;
if(isPW_REL)
......@@ -88,12 +88,8 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
pwrErrBoundBytesL = 4;
}
if(confparams_dec==NULL)
{
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
}
convertBytesToSZParams(&(flatBytes[index]), confparams_dec);
convertBytesToSZParams(&(flatBytes[index]), pde_params, pde_exe);
index += MetaDataByteLength;
......@@ -132,7 +128,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
byteBuf[i] = flatBytes[index++];
int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4
confparams_dec->maxRangeRadius = max_quant_intervals/2;
pde_params->maxRangeRadius = max_quant_intervals/2;
if(errorBoundMode>=PW_REL)
{
......@@ -140,7 +136,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
radExpoL = 1;
for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)
byteBuf[i] = flatBytes[index++];
confparams_dec->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
pde_params->segment_size = (*this)->segment_size = bytesToSize(byteBuf);// exe_params->SZ_SIZE_TYPE
for (i = 0; i < 4; i++)
byteBuf[i] = flatBytes[index++];
......@@ -161,7 +157,7 @@ int new_TightDataPointStorageF_fromFlatBytes(TightDataPointStorageF **this, unsi
(*this)->reqLength = flatBytes[index++]; //1
if(isPW_REL && confparams_dec->accelerate_pw_rel_compression)
if(isPW_REL && pde_params->accelerate_pw_rel_compression)
{
(*this)->plus_bits = flatBytes[index++];
(*this)->max_bits = flatBytes[index++];
......
......@@ -134,7 +134,7 @@ int new_TightDataPointStorageI_fromFlatBytes(TightDataPointStorageI **this, unsi
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
}
convertBytesToSZParams(&(flatBytes[index]), confparams_dec);
convertBytesToSZParams(&(flatBytes[index]), confparams_dec, exe_params);
/*sz_params* params = convertBytesToSZParams(&(flatBytes[index]));
int mode = confparams_dec->szMode;
int losslessCompressor = confparams_dec->losslessCompressor;
......
......@@ -195,54 +195,6 @@ double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, siz
return newByteData;
}
else if(dataType==SZ_INT64)
{
unsigned char *newByteData;
SZ_compress_args_int64(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_INT32) //int type
{
unsigned char *newByteData;
SZ_compress_args_int32(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_INT16)
{
unsigned char *newByteData;
SZ_compress_args_int16(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_INT8)
{
unsigned char *newByteData;
SZ_compress_args_int8(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_UINT64)
{
unsigned char *newByteData;
SZ_compress_args_uint64(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_UINT32) //int type
{
unsigned char *newByteData;
SZ_compress_args_uint32(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_UINT16)
{
unsigned char *newByteData;
SZ_compress_args_uint16(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else if(dataType==SZ_UINT8)
{
unsigned char *newByteData;
SZ_compress_args_uint8(&newByteData, data, r5, r4, r3, r2, r1, outSize, errBoundMode, absErrBound, relBoundRatio);
return newByteData;
}
else
{
printf("Error: dataType can only be SZ_FLOAT, SZ_DOUBLE, SZ_INT8/16/32/64 or SZ_UINT8/16/32/64.\n");
......@@ -345,13 +297,13 @@ unsigned char *SZ_compress_rev(int dataType, void *data, void *reservedValue, si
void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1)
{
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
exe_params->SZ_SIZE_TYPE = 8;
sz_exedata de_exe;
memset(&de_exe, 0, sizeof(sz_exedata));
de_exe.SZ_SIZE_TYPE = 8;
sz_params de_params;
memset(&de_params, 0, sizeof(sz_params));
int x = 1;
char *y = (char*)&x;
......@@ -363,63 +315,15 @@ void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_
if(dataType == SZ_FLOAT)
{
float *newFloatData;
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL);
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
return newFloatData;
}
else if(dataType == SZ_DOUBLE)
{
double *newDoubleData;
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL);
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, bytes, byteLength, 0, NULL, &de_exe, &de_params);
return newDoubleData;
}
else if(dataType == SZ_INT8)
{
int8_t *newInt8Data;
SZ_decompress_args_int8(&newInt8Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newInt8Data;
}
else if(dataType == SZ_INT16)
{
int16_t *newInt16Data;
SZ_decompress_args_int16(&newInt16Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newInt16Data;
}
else if(dataType == SZ_INT32)
{
int32_t *newInt32Data;
SZ_decompress_args_int32(&newInt32Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newInt32Data;
}
else if(dataType == SZ_INT64)
{
int64_t *newInt64Data;
SZ_decompress_args_int64(&newInt64Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newInt64Data;
}
else if(dataType == SZ_UINT8)
{
uint8_t *newUInt8Data;
SZ_decompress_args_uint8(&newUInt8Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newUInt8Data;
}
else if(dataType == SZ_UINT16)
{
uint16_t *newUInt16Data;
SZ_decompress_args_uint16(&newUInt16Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newUInt16Data;
}
else if(dataType == SZ_UINT32)
{
uint32_t *newUInt32Data;
SZ_decompress_args_uint32(&newUInt32Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newUInt32Data;
}
else if(dataType == SZ_UINT64)
{
uint64_t *newUInt64Data;
SZ_decompress_args_uint64(&newUInt64Data, r5, r4, r3, r2, r1, bytes, byteLength);
return newUInt64Data;
}
else
{
printf("Error: data type cannot be the types other than SZ_FLOAT or SZ_DOUBLE\n");
......@@ -521,7 +425,7 @@ size_t SZ_decompress_args(int dataType, unsigned char *bytes, size_t byteLength,
}
sz_metadata* SZ_getMetadata(unsigned char* bytes)
sz_metadata* SZ_getMetadata(unsigned char* bytes, sz_exedata* pde_exe)
{
int index = 0, i, isConstant, isLossless;
size_t dataSeriesLength = 0;
......@@ -535,12 +439,8 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes)
int isRegressionBased = (sameRByte >> 7) & 0x01;
if(exe_params==NULL)
{
exe_params = (sz_exedata *)malloc(sizeof(struct sz_exedata));
memset(exe_params, 0, sizeof(struct sz_exedata));
}
exe_params->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
pde_exe->SZ_SIZE_TYPE = ((sameRByte & 0x40)>>6)==1?8:4;
if(confparams_dec==NULL)
{
......@@ -548,7 +448,7 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes)
memset(confparams_dec, 0, sizeof(sz_params));
}
convertBytesToSZParams(&(bytes[index]), confparams_dec);
convertBytesToSZParams(&(bytes[index]), confparams_dec, pde_exe);
/*sz_params* params = convertBytesToSZParams(&(bytes[index]));
if(confparams_dec!=NULL)
free(confparams_dec);
......@@ -983,15 +883,14 @@ int SZ_compress_ts(int cmprType, unsigned char** newByteData, size_t *outSize)
void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength)
{
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
confparams_dec->szMode = SZ_TEMPORAL_COMPRESSION;
confparams_dec->predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE;
sz_params de_params;
memset(&de_params, 0, sizeof(sz_params));
de_params.szMode = SZ_TEMPORAL_COMPRESSION;
de_params.predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE;
sz_exedata de_exe;
memset(&de_exe, 0, sizeof(sz_exedata));
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
int x = 1;
char *y = (char*)&x;
......@@ -1037,12 +936,12 @@ void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength)
switch(dataType)
{
case SZ_FLOAT:
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data);
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params);
memcpy(p->data, newFloatData, dataLen*sizeof(float));
free(newFloatData);
break;
case SZ_DOUBLE:
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data);
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params);
memcpy(p->data, newDoubleData, dataLen*sizeof(double));
free(newDoubleData);
break;
......@@ -1058,15 +957,13 @@ void SZ_decompress_ts(unsigned char *bytes, size_t bytesLength)
void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count, unsigned char *bytes, size_t bytesLength)
{
if(confparams_dec==NULL)
confparams_dec = (sz_params*)malloc(sizeof(sz_params));
memset(confparams_dec, 0, sizeof(sz_params));
confparams_dec->szMode = SZ_TEMPORAL_COMPRESSION;
confparams_dec->predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE;
sz_params de_params;
memset(&de_params, 0, sizeof(sz_params));
de_params.szMode = SZ_TEMPORAL_COMPRESSION;
de_params.predictionMode = SZ_PREVIOUS_VALUE_ESTIMATE;
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
sz_exedata de_exe;
memset(&de_exe, 0, sizeof(sz_exedata));
int x = 1;
char *y = (char*)&x;
......@@ -1113,12 +1010,12 @@ void SZ_decompress_ts_select_var(unsigned char* var_ids, unsigned char var_count
switch(dataType)
{
case SZ_FLOAT:
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data);
SZ_decompress_args_float(&newFloatData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params);
memcpy(p->data, newFloatData, dataLen*sizeof(float));
free(newFloatData);
break;
case SZ_DOUBLE:
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data);
SZ_decompress_args_double(&newDoubleData, r5, r4, r3, r2, r1, cmpBytes, cmpSize, multisteps->compressionType, multisteps->hist_data, &de_exe, &de_params);
memcpy(p->data, newDoubleData, dataLen*sizeof(double));
free(newDoubleData);
break;
......
......@@ -19,7 +19,7 @@
#include "utility.h"
int SZ_decompress_args_double(double** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, double* hist_data)
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
{
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
......@@ -66,7 +66,7 @@ size_t cmpSize, int compressionType, double* hist_data)
confparams_dec->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to double array.
TightDataPointStorageD* tdps;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize);
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
int dim = computeDimension(r5,r4,r3,r2,r1);
int doubleSize = sizeof(double);
......@@ -86,14 +86,14 @@ size_t cmpSize, int compressionType, double* hist_data)
}
else if(confparams_dec->sol_ID==SZ_Transpose)
{
getSnapshotData_double_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data);
getSnapshotData_double_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //confparams_dec->sol_ID==SZ
{
if(tdps->raBytes_size > 0) //v2.0
{
if (dim == 1)
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data);
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
else if(dim == 2)
decompressDataSeries_double_2D_nonblocked_with_blocked_regression(newData, r2, r1, tdps->raBytes, hist_data);
else if(dim == 3)
......@@ -109,7 +109,7 @@ size_t cmpSize, int compressionType, double* hist_data)
else //1.4.13 or time-based compression
{
if (dim == 1)
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data);
getSnapshotData_double_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
else
if (dim == 2)
getSnapshotData_double_2D(newData,r2,r1,tdps, errBoundMode, compressionType, hist_data);
......@@ -2659,7 +2659,7 @@ void decompressDataSeries_double_3D_MSST19(double** data, size_t r1, size_t r2,
return;
}
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data)
void getSnapshotData_double_1D(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
{
size_t i;
if (tdps->allSameData) {
......
......@@ -48,7 +48,7 @@ void cost_end_()
* @return status SUCCESSFUL (SZ_SCES) or not (other error codes) f
* */
int SZ_decompress_args_float(float** newData, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, float* hist_data)
size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
{
int status = SZ_SCES;
size_t dataLength = computeDataLength(r5,r4,r3,r2,r1);
......@@ -56,37 +56,37 @@ size_t cmpSize, int compressionType, float* hist_data)
//unsigned char* tmpBytes;
size_t targetUncompressSize = dataLength <<2; //i.e., *4
//tmpSize must be "much" smaller than dataLength
size_t i, tmpSize = 8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE;
size_t i, tmpSize = 8+MetaDataByteLength+pde_exe->SZ_SIZE_TYPE;
unsigned char* szTmpBytes;
if(cmpSize!=8+4+MetaDataByteLength && cmpSize!=8+8+MetaDataByteLength) //4,8 means two posibilities of SZ_SIZE_TYPE
{
confparams_dec->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize);
if(confparams_dec->szMode!=SZ_TEMPORAL_COMPRESSION)
pde_params->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize);
if(pde_params->szMode!=SZ_TEMPORAL_COMPRESSION)
{
if(confparams_dec->losslessCompressor!=-1)
confparams_dec->szMode = SZ_BEST_COMPRESSION;
if(pde_params->losslessCompressor!=-1)
pde_params->szMode = SZ_BEST_COMPRESSION;
else
confparams_dec->szMode = SZ_BEST_SPEED;
pde_params->szMode = SZ_BEST_SPEED;
}
if(confparams_dec->szMode==SZ_BEST_SPEED)
if(pde_params->szMode==SZ_BEST_SPEED)
{
tmpSize = cmpSize;
szTmpBytes = cmpBytes;
}
else if(confparams_dec->szMode==SZ_BEST_COMPRESSION || confparams_dec->szMode==SZ_DEFAULT_COMPRESSION || confparams_dec->szMode==SZ_TEMPORAL_COMPRESSION)
else if(pde_params->szMode==SZ_BEST_COMPRESSION || pde_params->szMode==SZ_DEFAULT_COMPRESSION || pde_params->szMode==SZ_TEMPORAL_COMPRESSION)
{
if(targetUncompressSize<MIN_ZLIB_DEC_ALLOMEM_BYTES) //Considering the minimum size
targetUncompressSize = MIN_ZLIB_DEC_ALLOMEM_BYTES;
tmpSize = sz_lossless_decompress(confparams_dec->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
tmpSize = sz_lossless_decompress(pde_params->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE);// (unsigned long)targetUncompressSize+8: consider the total length under lossless compression mode is actually 3+4+1+targetUncompressSize
//szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize);
//memcpy(szTmpBytes, tmpBytes, tmpSize);
//free(tmpBytes); //release useless memory
}
else
{
printf("Wrong value of confparams_dec->szMode in the double compressed bytes.\n");
printf("Wrong value of pde_params->szMode in the double compressed bytes.\n");
status = SZ_MERR;
return status;
}
......@@ -94,11 +94,11 @@ size_t cmpSize, int compressionType, float* hist_data)
else
szTmpBytes = cmpBytes;
confparams_dec->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
pde_params->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to data array.
TightDataPointStorageF* tdps;
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize);
int errBoundMode = new_TightDataPointStorageF_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
//writeByteData(tdps->typeArray, tdps->typeArray_size, "decompress-typebytes.tbt");
int dim = computeDimension(r5,r4,r3,r2,r1);
......@@ -117,16 +117,16 @@ size_t cmpSize, int compressionType, float* hist_data)
(*newData)[i] = bytesToFloat(p);
}
}
else if(confparams_dec->sol_ID==SZ_Transpose)
else if(pde_params->sol_ID==SZ_Transpose)
{
getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data);
getSnapshotData_float_1D(newData,dataLength,tdps, errBoundMode, 0, hist_data, pde_params);
}
else //confparams_dec->sol_ID==SZ
else //pde_params->sol_ID==SZ
{
if(tdps->raBytes_size > 0) //v2.0
{
if (dim == 1)
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data);
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, 0, hist_data, pde_params);
else if(dim == 2)
decompressDataSeries_float_2D_nonblocked_with_blocked_regression(newData, r2, r1, tdps->raBytes, hist_data);
else if(dim == 3)
......@@ -142,7 +142,7 @@ size_t cmpSize, int compressionType, float* hist_data)
else //1.4.13 or time-based compression
{
if (dim == 1)
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data);
getSnapshotData_float_1D(newData,r1,tdps, errBoundMode, compressionType, hist_data, pde_params);
else if (dim == 2)
getSnapshotData_float_2D(newData,r2,r1,tdps, errBoundMode, compressionType, hist_data);
else if (dim == 3)
......@@ -158,11 +158,11 @@ size_t cmpSize, int compressionType, float* hist_data)
}
//cost_start_();
if(confparams_dec->protectValueRange)
if(pde_params->protectValueRange)
{
float* nd = *newData;
float min = confparams_dec->fmin;
float max = confparams_dec->fmax;
float min = pde_params->fmin;
float max = pde_params->fmax;
for(i=0;i<dataLength;i++)
{
float v = nd[i];
......@@ -177,7 +177,7 @@ size_t cmpSize, int compressionType, float* hist_data)
//cost_end_();
//printf("totalCost_=%f\n", totalCost_);
free_TightDataPointStorageF2(tdps);
if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE)
if(pde_params->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE)
free(szTmpBytes);
return status;
}
......@@ -272,7 +272,7 @@ void decompressDataSeries_float_1D(float** data, size_t dataSeriesLength, float*
}
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION)
memcpy(hist_data, (*data), dataSeriesLength*sizeof(float));
#endif
......@@ -2683,7 +2683,7 @@ void decompressDataSeries_float_3D_MSST19(float** data, size_t r1, size_t r2, si
return;
}
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data)
void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params)
{
size_t i;
......@@ -2697,7 +2697,7 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo
if(errBoundMode < PW_REL)
{
#ifdef HAVE_TIMECMPR
if(confparams_dec->szMode == SZ_TEMPORAL_COMPRESSION)
if(pde_params->szMode == SZ_TEMPORAL_COMPRESSION)
{
if(compressionType == 0) //snapshot
decompressDataSeries_float_1D(data, dataSeriesLength, hist_data, tdps);
......@@ -2710,7 +2710,7 @@ void getSnapshotData_float_1D(float** data, size_t dataSeriesLength, TightDataPo
}
else
{
if(confparams_dec->accelerate_pw_rel_compression)
if(pde_params->accelerate_pw_rel_compression)
decompressDataSeries_float_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps);
else
decompressDataSeries_float_1D_pwr_pre_log(data, dataSeriesLength, tdps);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册