Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
“33c8ddd49c56763588c67dfd25c547d8578cd3e7”上不存在“git@gitcode.net:qq_37101384/tdengine.git”
提交 3e24cb15 编写于 作者: T tickduan
浏览文件
操作

remove pwr_ reduce to 41s

上级 3e8cfacd
展开全部 隐藏空白更改
内联 并排
Showing with 214 addition and 4545 deletion
deps/SZ/sz/include/CompressElement.h
浏览文件 @ 3e24cb15
......@@ -45,7 +45,6 @@ typedef struct LossyCompressionElement
int residualMidBits;
} LossyCompressionElement;
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
short computeGroupNum_float(float value);
short computeGroupNum_double(double value);
......
deps/SZ/sz/include/TightDataPointStorageD.h
浏览文件 @ 3e24cb15
......@@ -33,10 +33,7 @@ typedef struct TightDataPointStorageD
size_t exactDataNum;
double reservedValue;
unsigned char* rtypeArray;
size_t rtypeArray_size;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
......@@ -54,9 +51,6 @@ typedef struct TightDataPointStorageD
unsigned char isLossless; //a mark to denote whether it's lossless compression (1 is yes, 0 is no)
size_t segment_size;
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
......@@ -75,22 +69,11 @@ void new_TightDataPointStorageD(TightDataPointStorageD **self,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
void new_TightDataPointStorageD2(TightDataPointStorageD **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, double medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
double realPrecision, double medianValue, char reqLength,
unsigned int intervals, unsigned char radExpo);
void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageD(TightDataPointStorageD *tdps);
void free_TightDataPointStorageD2(TightDataPointStorageD *tdps);
......
deps/SZ/sz/include/TightDataPointStorageF.h
浏览文件 @ 3e24cb15
......@@ -33,13 +33,10 @@ typedef struct TightDataPointStorageF
size_t exactDataNum;
float reservedValue;
unsigned char* rtypeArray;
size_t rtypeArray_size;
float minLogValue;
unsigned char* typeArray; //its size is dataSeriesLength/4 (or xxx/4+1)
size_t typeArray_size;
size_t typeArray_size;
unsigned char* leadNumArray; //its size is exactDataNum/4 (or exactDataNum/4+1)
size_t leadNumArray_size;
......@@ -56,8 +53,6 @@ typedef struct TightDataPointStorageF
size_t segment_size;
unsigned char* pwrErrBoundBytes;
int pwrErrBoundBytes_size;
unsigned char* raBytes;
size_t raBytes_size;
......@@ -77,24 +72,10 @@ void new_TightDataPointStorageF(TightDataPointStorageF **self,
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char resiBitLength,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
/**
* This function is designed for first-version of the point-wise relative error bound (developed by Sheng Di for TPDS18 paper)
*
* */
void new_TightDataPointStorageF2(TightDataPointStorageF **self,
size_t dataSeriesLength, size_t exactDataNum,
int* type, unsigned char* exactMidBytes, size_t exactMidBytes_size,
unsigned char* leadNumIntArray, //leadNumIntArray contains readable numbers....
unsigned char* resiMidBits, size_t resiMidBits_size,
unsigned char* resiBitLength, size_t resiBitLengthSize,
double realPrecision, float medianValue, char reqLength, unsigned int intervals,
unsigned char* pwrErrBoundBytes, size_t pwrErrBoundBytes_size, unsigned char radExpo);
unsigned char radExpo);
void convertTDPStoBytes_float(TightDataPointStorageF* tdps, unsigned char* bytes, unsigned char* dsLengthBytes, unsigned char sameByte);
bool convertTDPStoFlatBytes_float(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void convertTDPStoFlatBytes_float_args(TightDataPointStorageF *tdps, unsigned char* bytes, size_t *size);
void free_TightDataPointStorageF(TightDataPointStorageF *tdps);
void free_TightDataPointStorageF2(TightDataPointStorageF *tdps);
......
deps/SZ/sz/include/dataCompression.h
浏览文件 @ 3e24cb15
......@@ -30,21 +30,9 @@ extern "C" {
//dataCompression.c
int computeByteSizePerIntValue(long valueRangeSize);
long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize);
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue);
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue);
float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero);
double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero);
double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
float computeRangeSize_float_subblock(float* oriData, float* valueRangeSize, float* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1);
double min_d(double a, double b);
double max_d(double a, double b);
float min_f(float a, float b);
......@@ -56,22 +44,10 @@ void symTransform_8bytes(unsigned char data[8]);
void symTransform_2bytes(unsigned char data[2]);
void symTransform_4bytes(unsigned char data[4]);
void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes);
void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes);
void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes);
void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes);
void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes);
void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes);
void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes);
void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes);
void compressSingleFloatValue(FloatValueCompressElement *vce, float tgtValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength);
void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength);
int compIdenticalLeadingBytesCount_double(unsigned char* preBytes, unsigned char* curBytes);
int compIdenticalLeadingBytesCount_float(unsigned char* preBytes, unsigned char* curBytes);
......
deps/SZ/sz/include/defines.h
浏览文件 @ 3e24cb15
......@@ -10,8 +10,8 @@
#ifndef _SZ_DEFINES_H
#define _SZ_DEFINES_H
#define SZ_VERNUM 0x0200
#define SZ_VER_MAJOR 1 // curretn version
// define data format version
#define DATA_FROMAT_VER1 1 // curretn version
#define PASTRI 103
......@@ -88,6 +88,7 @@
#define SZ_BERR -6 //bound-mode error (should be only SZ_ABS, REL, ABS_AND_REL, ABS_OR_REL, or PW_REL)
#define SZ_LITTER_ELEMENT -7
#define SZ_ALGORITHM_ERR -8
#define SZ_FORMAT_ERR -9
#define SZ_MAINTAIN_VAR_DATA 0
#define SZ_DESTROY_WHOLE_VARSET 1
......
deps/SZ/sz/include/sz.h
浏览文件 @ 3e24cb15
......@@ -19,7 +19,6 @@
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "VarSet.h"
#include "Huffman.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "conf.h"
......@@ -30,8 +29,6 @@
#include "sz_double.h"
#include "szd_float.h"
#include "szd_double.h"
#include "sz_float_pwr.h"
#include "sz_double_pwr.h"
#include "sz_opencl.h"
#include "callZlib.h"
#include "rw.h"
......@@ -185,9 +182,6 @@ extern sz_tsc_metadata *sz_tsc;
extern pastri_params pastri_par;
#endif
//sz.h
HuffmanTree* SZ_Reset();
void SZ_Finalize();
int SZ_Init(const char *configFilePath);
......
deps/SZ/sz/include/sz_double.h
浏览文件 @ 3e24cb15
......@@ -20,22 +20,18 @@ extern "C" {
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize);
void computeReqLength_double(double realPrecision, short radExpo, int* reqLength, double* medianValue);
short computeReqLength_double_MSST19(double realPrecision);
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char* newByteData, size_t *outSize);
bool SZ_compress_args_double_NoCkRngeNoGzip_1D( unsigned char* newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData, size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize);
......
deps/SZ/sz/include/sz_double_pwr.h 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file sz_double.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_double.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Double_PWR_H
#define _SZ_Double_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdbool.h>
void compute_segment_precisions_double_1D(double *oriData, size_t dataLength, double* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_double_1D_pwr(double *oriData, size_t dataLength, double* pwrErrBound);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void createRangeGroups_double(double** posGroups, double** negGroups, int** posFlags, int** negFlags);
void compressGroupIDArray_double(char* groupID, TightDataPointStorageD* tdps);
TightDataPointStorageD* SZ_compress_double_1D_MDQ_pwrGroup(double* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, double *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, double valueRangeSize, double medianValue_f, size_t *outSize);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, double *oriData, double globalPrecision, size_t dataLength, size_t *outSize, double min, double max);
void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, double *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, double valueRangeSize, double medianValue_f,
unsigned char* signs, bool* positive, double min, double max, double nearZero);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Double_PWR_H ----- */
deps/SZ/sz/include/sz_float.h
浏览文件 @ 3e24cb15
......@@ -17,15 +17,12 @@ extern "C" {
unsigned char* SZ_skip_compress_float(float* data, size_t dataLength, size_t* outSize);
void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue);
short computeReqLength_float_MSST19(double realPrecision);
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_and_compute_dense_position_float_1D(float *oriData, size_t dataLength, double realPrecision, float * dense_pos);
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_f);
......@@ -40,9 +37,6 @@ void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, s
unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);
......
deps/SZ/sz/include/sz_float_pwr.h 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file sz_float.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the sz_float.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZ_Float_PWR_H
#define _SZ_Float_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdbool.h>
void compute_segment_precisions_float_1D(float *oriData, size_t dataLength, float* pwrErrBound, unsigned char* pwrErrBoundBytes, double globalPrecision);
unsigned int optimize_intervals_float_1D_pwr(float *oriData, size_t dataLength, float* pwrErrBound);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr(unsigned char* newByteData, float *oriData, double globalPrecision, size_t dataLength, size_t *outSize, float min, float max);
void createRangeGroups_float(float** posGroups, float** negGroups, int** posFlags, int** negFlags);
void compressGroupIDArray_float(char* groupID, TightDataPointStorageF* tdps);
int* generateGroupLowerBounds();
TightDataPointStorageF* SZ_compress_float_1D_MDQ_pwrGroup(float* oriData, size_t dataLength, int errBoundMode,
double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwrgroup(unsigned char* newByteData, float *oriData,
size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio, float valueRangeSize, float medianValue_f, size_t *outSize);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float min, float max);
void SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(unsigned char* newByteData, float *oriData, double pwrErrRatio, size_t dataLength, size_t *outSize, float valueRangeSize, float medianValue_f,
unsigned char* signs, bool* positive, float min, float max, float nearZero);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZ_Float_PWR_H ----- */
deps/SZ/sz/include/szd_double.h
浏览文件 @ 3e24cb15
......@@ -18,8 +18,6 @@ extern "C" {
void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params);
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
......
deps/SZ/sz/include/szd_double_pwr.h 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file szd_double_pwr.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_double_pwr.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Double_PWR_H
#define _SZD_Double_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
void decompressDataSeries_double_1D_pwr(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwrgroup(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwr_pre_log(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double** data, size_t dataSeriesLength, TightDataPointStorageD* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Double_PWR_H ----- */
deps/SZ/sz/include/szd_double_ts.h 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file szd_double_ts.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_double_ts.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Double_TS_H
#define _SZD_Double_TS_H
#ifdef __cplusplus
extern "C" {
#endif
#include "TightDataPointStorageD.h"
void decompressDataSeries_double_1D_ts(double** data, size_t dataSeriesLength, double* hist_data, TightDataPointStorageD* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Double_TS_H ----- */
deps/SZ/sz/include/szd_float.h
浏览文件 @ 3e24cb15
......@@ -18,8 +18,6 @@ extern "C" {
void decompressDataSeries_float_1D(float* data, size_t dataSeriesLength, float* hist_data, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void getSnapshotData_float_1D(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps, int errBoundMode, int compressionType, float* hist_data, sz_params* pde_params);
int SZ_decompress_args_float(float* newData, size_t r1, unsigned char* cmpBytes, size_t cmpSize, int compressionType, float* hist_data, sz_exedata* pde_exe, sz_params* pde_params);
......
deps/SZ/sz/include/szd_float_pwr.h 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file szd_float_pwr.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the szd_float_pwr.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _SZD_Float_PWR_H
#define _SZD_Float_PWR_H
#ifdef __cplusplus
extern "C" {
#endif
void decompressDataSeries_float_1D_pwr(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength);
void decompressDataSeries_float_1D_pwrgroup(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
void decompressDataSeries_float_1D_pwr_pre_log_MSST19(float* data, size_t dataSeriesLength, TightDataPointStorageF* tdps);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _SZD_Float_PWR_H ----- */
deps/SZ/sz/src/ByteToolkit.c
浏览文件 @ 3e24cb15
......@@ -906,10 +906,10 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result)
//int16ToBytes_bigEndian(&result[3], tmp2);
//errorBoundMode; //4bits(0.5 byte)
result[1] = params->errorBoundMode;
//result[1] = params->errorBoundMode;
//data type (float, double, int8, int16, ....) //10 choices, so 4 bits
result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[1] = (result[1] << 4) | (params->dataType & 0x17);
//result[5]: abs_err_bound or psnr //4 bytes
//result[9]: rel_bound_ratio or pwr_err_bound//4 bytes
......@@ -1001,10 +1001,10 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params, sz_exedata*
//params->sampleDistance = bytesToInt16_bigEndian(&bytes[1]);
//params->predThreshold = 1.0*bytesToInt16_bigEndian(&bytes[3])/10000.0;
/*
params->dataType = bytes[1] & 0x07;
params->errorBoundMode = (bytes[1] & 0xf0) >> 4;
/*
switch(params->errorBoundMode)
{
case SZ_ABS:
......
deps/SZ/sz/src/CompressElement.c
浏览文件 @ 3e24cb15
......@@ -16,31 +16,6 @@
#include <sz.h>
#include <CompressElement.h>
char* decompressGroupIDArray(unsigned char* bytes, size_t dataLength)
{
HuffmanTree* huffmanTree = SZ_Reset(); //create a default huffman tree
int* standGroupID = (int*)malloc(dataLength*sizeof(int));
decode_withTree(huffmanTree, bytes, dataLength, standGroupID);
SZ_ReleaseHuffman(huffmanTree);
char* groupID = (char*)malloc(dataLength*sizeof(char));
size_t i = 0;
int lastGroupIDValue = 0, curStandIDValue = 0, curGroupIDValue = 0;
int offset = 2*(GROUP_COUNT + 2);
curGroupIDValue = groupID[0] = standGroupID[0] - GROUP_COUNT;
lastGroupIDValue = curGroupIDValue;
for(i=1;i<dataLength;i++)
{
curStandIDValue = standGroupID[i];
curGroupIDValue = curStandIDValue + lastGroupIDValue - offset;
lastGroupIDValue = curGroupIDValue;
groupID[i] = curGroupIDValue;
}
free(standGroupID);
return groupID;
}
inline short computeGroupNum_float(float value)
{
......
deps/SZ/sz/src/dataCompression.c
浏览文件 @ 3e24cb15
......@@ -18,83 +18,7 @@
#include "CompressElement.h"
#include "dataCompression.h"
int computeByteSizePerIntValue(long valueRangeSize)
{
if(valueRangeSize<=256)
return 1;
else if(valueRangeSize<=65536)
return 2;
else if(valueRangeSize<=4294967296) //2^32
return 4;
else
return 8;
}
long computeRangeSize_int(void* oriData, int dataType, size_t size, int64_t* valueRangeSize)
{
size_t i = 0;
long max = 0, min = 0;
if(dataType==SZ_UINT8)
{
unsigned char* data = (unsigned char*)oriData;
unsigned char data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT8)
{
char* data = (char*)oriData;
char data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT16)
{
unsigned short* data = (unsigned short*)oriData;
unsigned short data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT16)
{
short* data = (short*)oriData;
short data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT32)
{
unsigned int* data = (unsigned int*)oriData;
unsigned int data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT32)
{
int* data = (int*)oriData;
int data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_UINT64)
{
unsigned long* data = (unsigned long*)oriData;
unsigned long data_;
min = data[0], max = min;
computeMinMax(data);
}
else if(dataType == SZ_INT64)
{
long* data = (long *)oriData;
long data_;
min = data[0], max = min;
computeMinMax(data);
}
*valueRangeSize = max - min;
return min;
}
float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize, float* medianValue)
{
......@@ -115,34 +39,6 @@ float computeRangeSize_float(float* oriData, size_t size, float* valueRangeSize,
return min;
}
float computeRangeSize_float_MSST19(float* oriData, size_t size, float* valueRangeSize, float* medianValue, unsigned char * signs, bool* positive, float* nearZero)
{
size_t i = 0;
float min = oriData[0];
float max = min;
*nearZero = min;
for(i=1;i<size;i++)
{
float data = oriData[i];
if(data <0){
signs[i] = 1;
*positive = false;
}
if(oriData[i] != 0 && fabsf(oriData[i]) < fabsf(*nearZero)){
*nearZero = oriData[i];
}
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double computeRangeSize_double(double* oriData, size_t size, double* valueRangeSize, double* medianValue)
{
size_t i = 0;
......@@ -162,94 +58,6 @@ double computeRangeSize_double(double* oriData, size_t size, double* valueRangeS
return min;
}
double computeRangeSize_double_MSST19(double* oriData, size_t size, double* valueRangeSize, double* medianValue, unsigned char * signs, bool* positive, double* nearZero)
{
size_t i = 0;
double min = oriData[0];
double max = min;
*nearZero = min;
for(i=1;i<size;i++)
{
double data = oriData[i];
if(data <0){
signs[i] = 1;
*positive = false;
}
if(oriData[i] != 0 && fabs(oriData[i]) < fabs(*nearZero)){
*nearZero = oriData[i];
}
if(min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
float computeRangeSize_float_subblock(float* oriData, float* valueRangeSize, float* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1)
{
size_t i1, i2, i3, i4, i5;
size_t index_start = s5*(r4*r3*r2*r1) + s4*(r3*r2*r1) + s3*(r2*r1) + s2*r1 + s1;
float min = oriData[index_start];
float max = min;
for (i5 = s5; i5 <= e5; i5++)
for (i4 = s4; i4 <= e4; i4++)
for (i3 = s3; i3 <= e3; i3++)
for (i2 = s2; i2 <= e2; i2++)
for (i1 = s1; i1 <= e1; i1++)
{
size_t index = i5*(r4*r3*r2*r1) + i4*(r3*r2*r1) + i3*(r2*r1) + i2*r1 + i1;
float data = oriData[index];
if (min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double computeRangeSize_double_subblock(double* oriData, double* valueRangeSize, double* medianValue,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1,
size_t s5, size_t s4, size_t s3, size_t s2, size_t s1,
size_t e5, size_t e4, size_t e3, size_t e2, size_t e1)
{
size_t i1, i2, i3, i4, i5;
size_t index_start = s5*(r4*r3*r2*r1) + s4*(r3*r2*r1) + s3*(r2*r1) + s2*r1 + s1;
double min = oriData[index_start];
double max = min;
for (i5 = s5; i5 <= e5; i5++)
for (i4 = s4; i4 <= e4; i4++)
for (i3 = s3; i3 <= e3; i3++)
for (i2 = s2; i2 <= e2; i2++)
for (i1 = s1; i1 <= e1; i1++)
{
size_t index = i5*(r4*r3*r2*r1) + i4*(r3*r2*r1) + i3*(r2*r1) + i2*r1 + i1;
double data = oriData[index];
if (min>data)
min = data;
else if(max<data)
max = data;
}
*valueRangeSize = max - min;
*medianValue = min + *valueRangeSize/2;
return min;
}
double min_d(double a, double b)
{
if(a<b)
......@@ -388,66 +196,6 @@ inline void symTransform_4bytes(unsigned char data[4])
data[2] = tmp;
}
inline void compressInt8Value(int8_t tgtValue, int8_t minValue, int byteSize, unsigned char* bytes)
{
uint8_t data = tgtValue - minValue;
memcpy(bytes, &data, byteSize); //byteSize==1
}
inline void compressInt16Value(int16_t tgtValue, int16_t minValue, int byteSize, unsigned char* bytes)
{
uint16_t data = tgtValue - minValue;
unsigned char tmpBytes[2];
int16ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 2 - byteSize, byteSize);
}
inline void compressInt32Value(int32_t tgtValue, int32_t minValue, int byteSize, unsigned char* bytes)
{
uint32_t data = tgtValue - minValue;
unsigned char tmpBytes[4];
int32ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 4 - byteSize, byteSize);
}
inline void compressInt64Value(int64_t tgtValue, int64_t minValue, int byteSize, unsigned char* bytes)
{
uint64_t data = tgtValue - minValue;
unsigned char tmpBytes[8];
int64ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 8 - byteSize, byteSize);
}
inline void compressUInt8Value(uint8_t tgtValue, uint8_t minValue, int byteSize, unsigned char* bytes)
{
uint8_t data = tgtValue - minValue;
memcpy(bytes, &data, byteSize); //byteSize==1
}
inline void compressUInt16Value(uint16_t tgtValue, uint16_t minValue, int byteSize, unsigned char* bytes)
{
uint16_t data = tgtValue - minValue;
unsigned char tmpBytes[2];
int16ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 2 - byteSize, byteSize);
}
inline void compressUInt32Value(uint32_t tgtValue, uint32_t minValue, int byteSize, unsigned char* bytes)
{
uint32_t data = tgtValue - minValue;
unsigned char tmpBytes[4];
int32ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 4 - byteSize, byteSize);
}
inline void compressUInt64Value(uint64_t tgtValue, uint64_t minValue, int byteSize, unsigned char* bytes)
{
uint64_t data = tgtValue - minValue;
unsigned char tmpBytes[8];
int64ToBytes_bigEndian(tmpBytes, data);
memcpy(bytes, tmpBytes + 8 - byteSize, byteSize);
}
inline void compressSingleFloatValue(FloatValueCompressElement *vce, float oriValue, float precision, float medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{
......@@ -472,52 +220,6 @@ inline void compressSingleFloatValue(FloatValueCompressElement *vce, float oriVa
vce->resiBitsLength = resiBitsLength;
}
void compressSingleFloatValue_MSST19(FloatValueCompressElement *vce, float tgtValue, float precision, int reqLength, int reqBytesLength, int resiBitsLength)
{
float normValue = tgtValue;
lfloat lfBuf;
lfBuf.value = normValue;
int ignBytesLength = 32 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
int tmp_int = lfBuf.ivalue;
intToBytes_bigEndian(vce->curBytes, tmp_int);
lfBuf.ivalue = (lfBuf.ivalue >> ignBytesLength) << ignBytesLength;
//float tmpValue = lfBuf.value;
vce->data = lfBuf.value;
vce->curValue = tmp_int;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
void compressSingleDoubleValue_MSST19(DoubleValueCompressElement *vce, double tgtValue, double precision, int reqLength, int reqBytesLength, int resiBitsLength)
{
ldouble lfBuf;
lfBuf.value = tgtValue;
int ignBytesLength = 64 - reqLength;
if(ignBytesLength<0)
ignBytesLength = 0;
long tmp_long = lfBuf.lvalue;
longToBytes_bigEndian(vce->curBytes, tmp_long);
lfBuf.lvalue = (lfBuf.lvalue >> ignBytesLength) << ignBytesLength;
//float tmpValue = lfBuf.value;
vce->data = lfBuf.value;
vce->curValue = tmp_long;
vce->reqBytesLength = reqBytesLength;
vce->resiBitsLength = resiBitsLength;
}
void compressSingleDoubleValue(DoubleValueCompressElement *vce, double tgtValue, double precision, double medianValue,
int reqLength, int reqBytesLength, int resiBitsLength)
{
......
deps/SZ/sz/src/sz.c
浏览文件 @ 3e24cb15
......@@ -19,13 +19,12 @@
#include "TightDataPointStorageF.h"
#include "zlib.h"
#include "rw.h"
#include "Huffman.h"
#include "conf.h"
#include "utility.h"
//#include "CurveFillingCompressStorage.h"
unsigned char versionNumber = SZ_VER_MAJOR;
unsigned char versionNumber = DATA_FROMAT_VER1;
int SZ_SIZE_TYPE_DEFUALT = 4;
int dataEndianType = LITTLE_ENDIAN_DATA; //*endian type of the data read from disk
......@@ -48,10 +47,7 @@ sz_tsc_metadata *sz_tsc = NULL;
pastri_params pastri_par;
#endif
HuffmanTree* SZ_Reset()
{
return createDefaultHuffmanTree();
}
int SZ_Init(const char *configFilePath)
{
......@@ -191,7 +187,15 @@ void SZ_Finalize()
}
}
void modulePath(char *buf, int size)
{
char path[1024];
sprintf(path, "/proc/%d/exe", getpid());
readlink(path, buf, size);
char* pos = strrchr(buf, '/');
if(pos)
pos[1]=0;
}
struct timeval startTime;
......@@ -200,6 +204,7 @@ struct timeval costStart; /*only used for recording the cost*/
double totalCost = 0;
void cost_start()
{
totalCost = 0;
......
deps/SZ/sz/src/sz_double.c
浏览文件 @ 3e24cb15
......@@ -19,12 +19,9 @@
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "sz_double.h"
#include "sz_double_pwr.h"
#include "szd_double.h"
#include "szd_double_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "sz_double_ts.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
......@@ -51,11 +48,6 @@ inline void computeReqLength_double(double realPrecision, short radExpo, int* re
}
}
inline short computeReqLength_double_MSST19(double realPrecision)
{
short reqExpo = getPrecisionReqLength_double(realPrecision);
return 12-reqExpo;
}
unsigned int optimize_intervals_double_1D(double *oriData, size_t dataLength, double realPrecision)
{
......@@ -117,9 +109,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
......@@ -141,8 +132,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
......@@ -166,7 +157,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
double interval = 2*realPrecision;
double recip_realPrecision = 1/realPrecision;
for(i=2;i<dataLength;i++)
for(i=2; i < dataLength; i++)
{
//printf("%.30G\n",last3CmprsData[0]);
curData = spaceFillingValue[i];
......@@ -190,8 +181,8 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
continue;
}
//unpredictable data processing
type[i] = 0;
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
......@@ -199,7 +190,6 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
//listAdd_double(last3CmprsData, vce->data);
pred = vce->data;
}//end of for
......@@ -212,7 +202,7 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
// printf("exactDataNum=%d, expSegmentsInBytes_size=%d, exactMidByteArray->size=%d\n",
// exactDataNum, expSegmentsInBytes_size, exactMidByteArray->size);
......@@ -283,171 +273,16 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize,
return true;
}
/*MSST19*/
TightDataPointStorageD* SZ_compress_double_1D_MDQ_MSST19(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_f)
{
//struct ClockPoint clockPointBuild;
//TimeDurationStart("build", &clockPointBuild);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_opt_MSST19(oriData, dataLength, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals);
double inv = 2.0-pow(2, -(confparams_cpr->plus_bits));
for(int i=0; i<quantization_intervals; i++){
double test = pow((1+realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
}
struct TopLevelTableWideInterval levelTable;
MultiLevelCacheTableWideIntervalBuild(&levelTable, precisionTable, quantization_intervals, realPrecision, confparams_cpr->plus_bits);
size_t i;
int reqLength;
double medianValue = medianValue_f;
//double medianInverse = 1 / medianValue_f;
//short radExpo = getExponent_double(realPrecision);
reqLength = computeReqLength_double_MSST19(realPrecision);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, dataLength/2/8);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, dataLength/2);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
double last3CmprsData[3] = {0};
//size_t miss=0, hit=0;
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleDoubleValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
//add the second data
type[1] = 0;
compressSingleDoubleValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_double(last3CmprsData, vce->data);
//miss++;
int state;
//double checkRadius;
double curData;
double pred = vce->data;
double predRelErrRatio;
const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex;
const uint64_t range = top - base;
const int bits = levelTable.bits;
uint64_t* const buffer = (uint64_t*)&predRelErrRatio;
const int shift = 52-bits;
uint64_t expoIndex, mantiIndex;
uint16_t* tables[range+1];
for(int i=0; i<=range; i++){
tables[i] = levelTable.subTables[i].table;
}
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
predRelErrRatio = curData / pred;
expoIndex = ((*buffer & 0x7fffffffffffffff) >> 52) - base;
if(expoIndex <= range){
mantiIndex = (*buffer & 0x000fffffffffffff) >> shift;
state = tables[expoIndex][mantiIndex];
}else{
state = 0;
}
if(state)
{
type[i] = state;
pred *= precisionTable[state];
//hit++;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
pred = vce->data;
//miss++;
}//end of for
// printf("miss:%d, hit:%d\n", miss, hit);
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
tdps->plus_bits = confparams_cpr->plus_bits;
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
free(precisionTable);
freeTopLevelTableWideInterval(&levelTable);
return tdps;
}
void SZ_compress_args_double_withinRange(unsigned char* newByteData, double *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageD* tdps = (TightDataPointStorageD*) malloc(sizeof(TightDataPointStorageD));
tdps->rtypeArray = NULL;
tdps->typeArray = NULL;
tdps->leadNumArray = NULL;
tdps->residualMidBits = NULL;
tdps->allSameData = 1;
tdps->dataSeriesLength = dataLength;
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*8);
tdps->pwrErrBoundBytes = NULL;
tdps->isLossless = 0;
double value = oriData[0];
doubleToBytes(tdps->exactMidBytes, value);
......@@ -472,12 +307,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
double valueRangeSize = 0, medianValue = 0;
unsigned char * signs = NULL;
bool positive = true;
double nearZero = 0.0;
double min = 0;
if(params->pw_relBoundRatio < 0.000009999)
params->accelerate_pw_rel_compression = 0;
// check at least elements count
if(dataLength <= MIN_NUM_OF_ELEMENTS)
......@@ -486,14 +317,8 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
return SZ_LITTER_ELEMENT;
}
if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression == 1)
{
signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
min = computeRangeSize_double_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero);
}
else
min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue);
double max = min+valueRangeSize;
params->dmin = min;
params->dmax = max;
......@@ -517,9 +342,7 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
params->absErrBound = realPrecision;
}
if(valueRangeSize <= realPrecision)
{
if(params->errorBoundMode>=PW_REL && params->accelerate_pw_rel_compression == 1)
free(signs);
{
SZ_compress_args_double_withinRange(newByteData, oriData, dataLength, outSize);
}
else
......@@ -532,25 +355,15 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
tmpByteData = (unsigned char*)malloc(r1*sizeof(double)*1.2);
}
if(params->errorBoundMode>=PW_REL)
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768)
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max);
}
else
{
if(!SZ_compress_args_double_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(twoStage)
free(tmpByteData);
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
return SZ_ALGORITHM_ERR;
}
//if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1)
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tmpByteData, &tmpOutSize);
//
//Call Gzip to do the further compression.
......@@ -569,58 +382,6 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa
return status;
}
unsigned int optimize_intervals_double_1D_opt_MSST19(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0;
double pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance;
double * data_pos = oriData + 2;
double divider = log2(1+realPrecision)*2;
int tempIndex = 0;
while(data_pos - oriData < dataLength){
if(*data_pos == 0){
data_pos += confparams_cpr->sampleDistance;
continue;
}
tempIndex++;
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs((double)*data_pos / pred_value);
radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<64)
powerOf2 = 64;
free(intervals);
return powerOf2;
}
unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
......
deps/SZ/sz/src/sz_double_pwr.c 已删除 100644 → 0
浏览文件 @ 3e8cfacd
此差异已折叠。
deps/SZ/sz/src/sz_double_ts.c 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file sz_double_ts.c
* @author Sheng Di and Dingwen Tao
* @date Aug, 2016
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <math.h>
#include "sz.h"
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "TightDataPointStorageD.h"
#include "zlib.h"
#include "rw.h"
#include "sz_double_ts.h"
unsigned int optimize_intervals_double_1D_ts(double *oriData, size_t dataLength, double* preData, double realPrecision)
{
size_t i = 0, radiusIndex;
double pred_value = 0, pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = dataLength/confparams_cpr->sampleDistance;
for(i=2;i<dataLength;i++)
{
if(i%confparams_cpr->sampleDistance==0)
{
pred_value = preData[i];
pred_err = fabs(pred_value - oriData[i]);
radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
}
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
TightDataPointStorageD* SZ_compress_double_1D_MDQ_ts(double *oriData, size_t dataLength, sz_multisteps* multisteps,
double realPrecision, double valueRangeSize, double medianValue_d)
{
double* preStepData = (double*)(multisteps->hist_data);
//store the decompressed data
//double* decData = (double*)malloc(sizeof(double)*dataLength);
//memset(decData, 0, sizeof(double)*dataLength);
double* decData = preStepData;
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_double_1D_ts(oriData, dataLength, preStepData, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
updateQuantizationInfo(quantization_intervals);
size_t i;
int reqLength;
double medianValue = medianValue_d;
short radExpo = getExponent_double(valueRangeSize/2);
computeReqLength_double(realPrecision, radExpo, &reqLength, &medianValue);
int* type = (int*) malloc(dataLength*sizeof(int));
double* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, DynArrayInitLen);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, DynArrayInitLen);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[8];
longToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
DoubleValueCompressElement *vce = (DoubleValueCompressElement*)malloc(sizeof(DoubleValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[0], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[0] = vce->data;
//add the second data
type[1] = 0;
compressSingleDoubleValue(vce, spaceFillingValue[1], realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[1] = vce->data;
int state = 0;
double checkRadius = 0;
double curData = 0;
double pred = 0;
double predAbsErr = 0;
checkRadius = (exe_params->intvCapacity-1)*realPrecision;
double interval = 2*realPrecision;
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
pred = preStepData[i];
predAbsErr = fabs(curData - pred);
if(predAbsErr<=checkRadius)
{
state = (predAbsErr/realPrecision+1)/2;
if(curData>=pred)
{
type[i] = exe_params->intvRadius+state;
pred = pred + state*interval;
}
else //curData<pred
{
type[i] = exe_params->intvRadius-state;
pred = pred - state*interval;
}
decData[i] = pred;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleDoubleValue(vce, curData, realPrecision, medianValue, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Double(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,8);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
decData[i] = vce->data;
}//end of for
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageD* tdps;
new_TightDataPointStorageD(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
//memcpy(preStepData, decData, dataLength*sizeof(double)); //update the data
//free(decData);
return tdps;
}
deps/SZ/sz/src/sz_float.c
浏览文件 @ 3e24cb15
......@@ -19,9 +19,7 @@
#include "DynamicByteArray.h"
#include "TightDataPointStorageF.h"
#include "sz_float.h"
#include "sz_float_pwr.h"
#include "szd_float.h"
#include "szd_float_pwr.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
......@@ -53,11 +51,6 @@ void computeReqLength_float(double realPrecision, short rangeExpo, int* reqLengt
}
}
inline short computeReqLength_float_MSST19(double realPrecision)
{
short reqExpo = getPrecisionReqLength_float(realPrecision);
return 9-reqExpo;
}
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision)
{
......@@ -240,7 +233,7 @@ TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
realPrecision, medianValue, (char)reqLength, quantization_intervals, 0);
//sdi:Debug
/* int sum =0;
......@@ -286,176 +279,16 @@ bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float
return true;
}
/*MSST19*/
TightDataPointStorageF* SZ_compress_float_1D_MDQ_MSST19(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f)
{
//struct ClockPoint clockPointBuild;
//TimeDurationStart("build", &clockPointBuild);
unsigned int quantization_intervals;
if(exe_params->optQuantMode==1)
quantization_intervals = optimize_intervals_float_1D_opt_MSST19(oriData, dataLength, realPrecision);
else
quantization_intervals = exe_params->intvCapacity;
//updateQuantizationInfo(quantization_intervals);
int intvRadius = quantization_intervals/2;
double* precisionTable = (double*)malloc(sizeof(double) * quantization_intervals);
double inv = 2.0-pow(2, -(confparams_cpr->plus_bits));
for(int i=0; i<quantization_intervals; i++){
double test = pow((1+realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
// if(i>30000 && i<40000)
// printf("%d %.30G\n", i, test);
}
//float smallest_precision = precisionTable[0], largest_precision = precisionTable[quantization_intervals-1];
struct TopLevelTableWideInterval levelTable;
MultiLevelCacheTableWideIntervalBuild(&levelTable, precisionTable, quantization_intervals, realPrecision, confparams_cpr->plus_bits);
size_t i;
int reqLength;
float medianValue = medianValue_f;
//float medianInverse = 1 / medianValue_f;
//short radExpo = getExponent_float(valueRangeSize/2);
reqLength = computeReqLength_float_MSST19(realPrecision);
int* type = (int*) malloc(dataLength*sizeof(int));
float* spaceFillingValue = oriData; //
DynamicIntArray *exactLeadNumArray;
new_DIA(&exactLeadNumArray, dataLength/2/8);
DynamicByteArray *exactMidByteArray;
new_DBA(&exactMidByteArray, dataLength/2);
DynamicIntArray *resiBitArray;
new_DIA(&resiBitArray, DynArrayInitLen);
unsigned char preDataBytes[4];
intToBytes_bigEndian(preDataBytes, 0);
int reqBytesLength = reqLength/8;
int resiBitsLength = reqLength%8;
float last3CmprsData[3] = {0};
//size_t miss=0, hit=0;
FloatValueCompressElement *vce = (FloatValueCompressElement*)malloc(sizeof(FloatValueCompressElement));
LossyCompressionElement *lce = (LossyCompressionElement*)malloc(sizeof(LossyCompressionElement));
//add the first data
type[0] = 0;
compressSingleFloatValue_MSST19(vce, spaceFillingValue[0], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//miss++;
//add the second data
type[1] = 0;
compressSingleFloatValue_MSST19(vce, spaceFillingValue[1], realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
listAdd_float(last3CmprsData, vce->data);
//miss++;
int state;
//double checkRadius;
float curData;
float pred = vce->data;
double predRelErrRatio;
const uint64_t top = levelTable.topIndex, base = levelTable.baseIndex;
const uint64_t range = top - base;
const int bits = levelTable.bits;
uint64_t* const buffer = (uint64_t*)&predRelErrRatio;
const int shift = 52-bits;
uint64_t expoIndex, mantiIndex;
uint16_t* tables[range+1];
for(int i=0; i<=range; i++){
tables[i] = levelTable.subTables[i].table;
}
for(i=2;i<dataLength;i++)
{
curData = spaceFillingValue[i];
predRelErrRatio = curData / pred;
expoIndex = ((*buffer & 0x7fffffffffffffff) >> 52) - base;
if(expoIndex <= range){
mantiIndex = (*buffer & 0x000fffffffffffff) >> shift;
state = tables[expoIndex][mantiIndex];
}else{
state = 0;
}
if(state)
{
type[i] = state;
pred *= precisionTable[state];
//hit++;
continue;
}
//unpredictable data processing
type[i] = 0;
compressSingleFloatValue_MSST19(vce, curData, realPrecision, reqLength, reqBytesLength, resiBitsLength);
updateLossyCompElement_Float(vce->curBytes, preDataBytes, reqBytesLength, resiBitsLength, lce);
memcpy(preDataBytes,vce->curBytes,4);
addExactData(exactMidByteArray, exactLeadNumArray, resiBitArray, lce);
pred = vce->data;
//miss++;
#ifdef HAVE_TIMECMPR
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
decData[i] = vce->data;
#endif
}//end of for
// printf("miss:%d, hit:%d\n", miss, hit);
size_t exactDataNum = exactLeadNumArray->size;
TightDataPointStorageF* tdps;
new_TightDataPointStorageF(&tdps, dataLength, exactDataNum,
type, exactMidByteArray->array, exactMidByteArray->size,
exactLeadNumArray->array,
resiBitArray->array, resiBitArray->size,
resiBitsLength,
realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0);
tdps->plus_bits = confparams_cpr->plus_bits;
//free memory
free_DIA(exactLeadNumArray);
free_DIA(resiBitArray);
free(type);
free(vce);
free(lce);
free(exactMidByteArray); //exactMidByteArray->array has been released in free_TightDataPointStorageF(tdps);
free(precisionTable);
freeTopLevelTableWideInterval(&levelTable);
return tdps;
}
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize)
{
TightDataPointStorageF* tdps = (TightDataPointStorageF*) malloc(sizeof(TightDataPointStorageF));
tdps->rtypeArray = NULL;
tdps->typeArray = NULL;
tdps->leadNumArray = NULL;
tdps->residualMidBits = NULL;
tdps->allSameData = 1;
tdps->dataSeriesLength = dataLength;
tdps->exactMidBytes = (unsigned char*)malloc(sizeof(unsigned char)*4);
tdps->pwrErrBoundBytes = NULL;
tdps->isLossless = 0;
float value = oriData[0];
floatToBytes(tdps->exactMidBytes, value);
......@@ -487,9 +320,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
printf("error, input elements count=%d less than %d, so need not do compress.\n", dataLength, MIN_NUM_OF_ELEMENTS);
return SZ_LITTER_ELEMENT;
}
if(params->pw_relBoundRatio < 0.000009999)
params->accelerate_pw_rel_compression = 0;
float valueRangeSize = 0, medianValue = 0;
unsigned char * signs = NULL;
......@@ -497,17 +327,7 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
float nearZero = 0.0;
float min = 0;
// calc min
if(params->errorBoundMode == PW_REL && params->accelerate_pw_rel_compression)
{
signs = (unsigned char *) malloc(dataLength);
memset(signs, 0, dataLength);
min = computeRangeSize_float_MSST19(oriData, dataLength, &valueRangeSize, &medianValue, signs, &positive, &nearZero);
free(signs); // tickduan move to here
signs = NULL;
}
else
min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
min = computeRangeSize_float(oriData, dataLength, &valueRangeSize, &medianValue);
// calc max
float max = min+valueRangeSize;
......@@ -557,23 +377,13 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
tmpByteData = (unsigned char*)malloc(r1*sizeof(float) + 1024);
}
if(params->errorBoundMode>=PW_REL)
// compress core algorithm
if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
if(params->accelerate_pw_rel_compression && params->maxRangeRadius <= 32768)
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log_MSST19(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, valueRangeSize, medianValue, signs, &positive, min, max, nearZero);
else
SZ_compress_args_float_NoCkRngeNoGzip_1D_pwr_pre_log(tmpByteData, oriData, params->pw_relBoundRatio, r1, &tmpOutSize, min, max);
}
else
{
// compress core algorithm
if(!SZ_compress_args_float_NoCkRngeNoGzip_1D(tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue))
{
*outSize = 0;
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
*outSize = 0;
if(twoStage)
free(tmpByteData);
return SZ_ALGORITHM_ERR;
}
//cost_end(" sz_first_compress");
......@@ -597,58 +407,6 @@ int SZ_compress_args_float(float *oriData, size_t r1, unsigned char* newByteData
return SZ_SUCCESS;
}
unsigned int optimize_intervals_float_1D_opt_MSST19(float *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
float pred_value = 0;
double pred_err;
size_t *intervals = (size_t*)malloc(confparams_cpr->maxRangeRadius*sizeof(size_t));
memset(intervals, 0, confparams_cpr->maxRangeRadius*sizeof(size_t));
size_t totalSampleSize = 0;//dataLength/confparams_cpr->sampleDistance;
float * data_pos = oriData + 2;
float divider = log2(1+realPrecision)*2;
int tempIndex = 0;
while(data_pos - oriData < dataLength){
if(*data_pos == 0){
data_pos += confparams_cpr->sampleDistance;
continue;
}
tempIndex++;
totalSampleSize++;
pred_value = data_pos[-1];
pred_err = fabs((double)*data_pos / pred_value);
radiusIndex = (unsigned long)fabs(log2(pred_err)/divider+0.5);
if(radiusIndex>=confparams_cpr->maxRangeRadius)
radiusIndex = confparams_cpr->maxRangeRadius - 1;
intervals[radiusIndex]++;
data_pos += confparams_cpr->sampleDistance;
}
//compute the appropriate number
size_t targetCount = totalSampleSize*confparams_cpr->predThreshold;
size_t sum = 0;
for(i=0;i<confparams_cpr->maxRangeRadius;i++)
{
sum += intervals[i];
if(sum>targetCount)
break;
}
if(i>=confparams_cpr->maxRangeRadius)
i = confparams_cpr->maxRangeRadius-1;
unsigned int accIntervals = 2*(i+1);
unsigned int powerOf2 = roundUpToPowerOf2(accIntervals);
if(powerOf2<32)
powerOf2 = 32;
free(intervals);
return powerOf2;
}
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision)
{
size_t i = 0, radiusIndex;
......
deps/SZ/sz/src/sz_float_pwr.c 已删除 100644 → 0
浏览文件 @ 3e8cfacd
此差异已折叠。
deps/SZ/sz/src/szd_double.c
浏览文件 @ 3e24cb15
......@@ -13,10 +13,8 @@
#include "szd_double.h"
#include "TightDataPointStorageD.h"
#include "sz.h"
#include "Huffman.h"
#include "szd_double_pwr.h"
#include "szd_double_ts.h"
#include "utility.h"
#include "Huffman.h"
int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpBytes,
size_t cmpSize, int compressionType, double* hist_data, sz_exedata* pde_exe, sz_params* pde_params)
......@@ -68,8 +66,12 @@ int SZ_decompress_args_double(double* newData, size_t r1, unsigned char* cmpByte
// calc postion
//pde_params->sol_ID = szTmpBytes[1+3-2+14-4]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose
//TODO: convert szTmpBytes to double array.
TightDataPointStorageD* tdps;
TightDataPointStorageD* tdps = NULL;
int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize, pde_exe, pde_params);
if(tdps == NULL)
{
return SZ_FORMAT_ERR;
}
int dim = r1;
int doubleSize = sizeof(double);
......@@ -124,7 +126,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
......@@ -143,8 +144,10 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
resiBitsLength = tdps->reqLength%8;
medianValue = tdps->medianValue;
int type_;
for (i = 0; i < dataSeriesLength; i++) {
for (i = 0; i < dataSeriesLength; i++)
{
type_ = type[i];
switch (type_) {
case 0:
......@@ -203,108 +206,6 @@ void decompressDataSeries_double_1D(double* data, size_t dataSeriesLength, doubl
return;
}
/*MSST19*/
void decompressDataSeries_double_1D_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
//updateQuantizationInfo(tdps->intervals);
int intvRadius = tdps->intervals/2;
int intvCapacity = tdps->intervals;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
//double interval = tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree_MSST19(huffmanTree, tdps->typeArray, dataSeriesLength, type, tdps->max_bits);
//decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqBytesLength, resiBitsLength, resiBits;
unsigned char leadingNum;
double exactData, predValue = 0;
reqBytesLength = tdps->reqLength/8;
resiBitsLength = tdps->reqLength%8;
//float threshold = tdps->minLogValue;
double* precisionTable = (double*)malloc(sizeof(double) * intvCapacity);
double inv = 2.0-pow(2, -(tdps->plus_bits));
for(int i=0; i<intvCapacity; i++){
double test = pow((1+tdps->realPrecision), inv*(i - intvRadius));
precisionTable[i] = test;
}
int type_;
for (i = 0; i < dataSeriesLength; i++) {
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
data[i] = exactData;
memcpy(preBytes,curBytes,8);
predValue = data[i];
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
//predValue = data[i-1];
predValue = fabs(predValue) * precisionTable[type_];
data[i] = predValue;
break;
}
}
free(precisionTable);
free(leadNum);
free(type);
return;
}
void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps, int errBoundMode, int compressionType, double* hist_data, sz_params* pde_params)
{
......@@ -315,24 +216,10 @@ void getSnapshotData_double_1D(double* data, size_t dataSeriesLength, TightDataP
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
for (i = 0; i < dataSeriesLength; i++)
data[i] = value;
} else {
if (tdps->rtypeArray == NULL) {
if(errBoundMode < PW_REL)
{
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
}
else
{
if(confparams_dec->accelerate_pw_rel_compression)
decompressDataSeries_double_1D_pwr_pre_log_MSST19(data, dataSeriesLength, tdps);
else
decompressDataSeries_double_1D_pwr_pre_log(data, dataSeriesLength, tdps);
//decompressDataSeries_double_1D_pwrgroup(data, dataSeriesLength, tdps);
}
return;
} else {
//TODO
}
}
else
{
decompressDataSeries_double_1D(data, dataSeriesLength, hist_data, tdps);
}
}
deps/SZ/sz/src/szd_double_pwr.c 已删除 100644 → 0
浏览文件 @ 3e8cfacd
/**
* @file szd_double_pwr.c
* @author Sheng Di, Dingwen Tao, Xin Liang, Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang
* @date Feb, 2019
* @brief
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include "TightDataPointStorageD.h"
#include "CompressElement.h"
#include "sz.h"
#include "Huffman.h"
#include "sz_double_pwr.h"
#include "utility.h"
//#include "rw.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wchar-subscripts"
void decompressDataSeries_double_1D_pwr(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
updateQuantizationInfo(tdps->intervals);
unsigned char tmpPrecBytes[8] = {0}; //used when needing to convert bytes to double values
unsigned char* bp = tdps->pwrErrBoundBytes;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
unsigned char* leadNum;
double interval = 0;// = (double)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqLength = 0, reqBytesLength = 0, resiBitsLength = 0, resiBits = 0;
unsigned char leadingNum;
double medianValue, exactData, predValue = 0, realPrecision = 0;
medianValue = tdps->medianValue;
int type_, updateReqLength = 0;
for (i = 0; i < dataSeriesLength; i++)
{
if(i%tdps->segment_size==0)
{
tmpPrecBytes[0] = *(bp++);
tmpPrecBytes[1] = *(bp++);
memset(&tmpPrecBytes[2], 0, 6*sizeof(unsigned char));
realPrecision = bytesToDouble(tmpPrecBytes);
interval = realPrecision*2;
updateReqLength = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
data[i] = exactData + medianValue;
memcpy(preBytes,curBytes,8);
break;
default:
//predValue = 2 * data[i-1] - data[i-2];
predValue = data[i-1];
data[i] = predValue + (type_-exe_params->intvRadius)*interval;
break;
}
//printf("%.30G\n",data[i]);
}
free(leadNum);
free(type);
return;
}
void decompressDataSeries_double_1D_pwrgroup(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
double *posGroups, *negGroups, *groups;
double pos_01_group, neg_01_group;
int *posFlags, *negFlags;
updateQuantizationInfo(tdps->intervals);
unsigned char* leadNum;
double interval;// = (double)tdps->realPrecision*2;
convertByteArray2IntArray_fast_2b(tdps->exactDataNum, tdps->leadNumArray, tdps->leadNumArray_size, &leadNum);
//*data = (double*)malloc(sizeof(double)*dataSeriesLength); comment by tickduan
int* type = (int*)malloc(dataSeriesLength*sizeof(int));
HuffmanTree* huffmanTree = createHuffmanTree(tdps->stateNum);
decode_withTree(huffmanTree, tdps->typeArray, dataSeriesLength, type);
SZ_ReleaseHuffman(huffmanTree);
createRangeGroups_double(&posGroups, &negGroups, &posFlags, &negFlags);
double realGroupPrecision;
double realPrecision = tdps->realPrecision;
char* groupID = decompressGroupIDArray(tdps->pwrErrBoundBytes, tdps->dataSeriesLength);
//note that the groupID values here are [1,2,3,....,18] or [-1,-2,...,-18]
double* groupErrorBounds = generateGroupErrBounds(confparams_dec->errorBoundMode, realPrecision, confparams_dec->pw_relBoundRatio);
exe_params->intvRadius = generateGroupMaxIntervalCount(groupErrorBounds);
size_t nbBins = (size_t)(1/confparams_dec->pw_relBoundRatio + 0.5);
if(nbBins%2==1)
nbBins++;
exe_params->intvRadius = nbBins;
unsigned char preBytes[8];
unsigned char curBytes[8];
memset(preBytes, 0, 8);
size_t curByteIndex = 0;
int reqLength, reqBytesLength = 0, resiBitsLength = 0, resiBits;
unsigned char leadingNum;
double medianValue, exactData, curValue, predValue;
medianValue = tdps->medianValue;
size_t i, j, k = 0, p = 0, l = 0; // k is to track the location of residual_bit
// in resiMidBits, p is to track the
// byte_index of resiMidBits, l is for
// leadNum
int type_, updateReqLength = 0;
char rawGrpID = 0, indexGrpID = 0;
for (i = 0; i < dataSeriesLength; i++)
{
rawGrpID = groupID[i];
if(rawGrpID >= 2)
{
groups = posGroups;
indexGrpID = rawGrpID - 2;
}
else if(rawGrpID <= -2)
{
groups = negGroups;
indexGrpID = -rawGrpID - 2; }
else if(rawGrpID == 1)
{
groups = &pos_01_group;
indexGrpID = 0;
}
else //rawGrpID == -1
{
groups = &neg_01_group;
indexGrpID = 0;
}
type_ = type[i];
switch (type_) {
case 0:
// compute resiBits
if(updateReqLength==0)
{
computeReqLength_double(realPrecision, tdps->radExpo, &reqLength, &medianValue);
reqBytesLength = reqLength/8;
resiBitsLength = reqLength%8;
updateReqLength = 1;
}
resiBits = 0;
if (resiBitsLength != 0) {
int kMod8 = k % 8;
int rightMovSteps = getRightMovingSteps(kMod8, resiBitsLength);
if (rightMovSteps > 0) {
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code) >> rightMovSteps;
} else if (rightMovSteps < 0) {
int code1 = getLeftMovingCode(kMod8);
int code2 = getRightMovingCode(kMod8, resiBitsLength);
int leftMovSteps = -rightMovSteps;
rightMovSteps = 8 - leftMovSteps;
resiBits = (tdps->residualMidBits[p] & code1) << leftMovSteps;
p++;
resiBits = resiBits
| ((tdps->residualMidBits[p] & code2) >> rightMovSteps);
} else // rightMovSteps == 0
{
int code = getRightMovingCode(kMod8, resiBitsLength);
resiBits = (tdps->residualMidBits[p] & code);
p++;
}
k += resiBitsLength;
}
// recover the exact data
memset(curBytes, 0, 8);
leadingNum = leadNum[l++];
memcpy(curBytes, preBytes, leadingNum);
for (j = leadingNum; j < reqBytesLength; j++)
curBytes[j] = tdps->exactMidBytes[curByteIndex++];
if (resiBitsLength != 0) {
unsigned char resiByte = (unsigned char) (resiBits << (8 - resiBitsLength));
curBytes[reqBytesLength] = resiByte;
}
exactData = bytesToDouble(curBytes);
exactData = exactData + medianValue;
data[i] = exactData;
memcpy(preBytes,curBytes,8);
groups[indexGrpID] = exactData;
break;
default:
predValue = groups[indexGrpID]; //Here, groups[indexGrpID] is the previous value.
realGroupPrecision = groupErrorBounds[indexGrpID];
interval = realGroupPrecision*2;
curValue = predValue + (type_-exe_params->intvRadius)*interval;
//groupNum = computeGroupNum_double(curValue);
if((curValue>0&&rawGrpID<0)||(curValue<0&&rawGrpID>0))
curValue = 0;
//else
//{
// realGrpID = fabs(rawGrpID)-2;
// if(groupNum<realGrpID)
// curValue = rawGrpID>0?pow(2,realGrpID):-pow(2,realGrpID);
// else if(groupNum>realGrpID)
// curValue = rawGrpID>0?pow(2,groupNum):-pow(2,groupNum);
//}
data[i] = curValue;
groups[indexGrpID] = curValue;
break;
}
}
free(leadNum);
free(type);
free(posGroups);
free(negGroups);
free(posFlags);
free(negFlags);
free(groupErrorBounds);
free(groupID);
}
void decompressDataSeries_double_1D_pwr_pre_log(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps) {
decompressDataSeries_double_1D(data, dataSeriesLength, NULL, tdps);
double threshold = tdps->minLogValue;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs;
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
if(signs[i]) data[i] = -(data[i]);
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
else data[i] = exp2(data[i]);
}
}
}
void decompressDataSeries_double_1D_pwr_pre_log_MSST19(double* data, size_t dataSeriesLength, TightDataPointStorageD* tdps)
{
decompressDataSeries_double_1D_MSST19(data, dataSeriesLength, tdps);
double threshold = tdps->minLogValue;
uint64_t* ptr;
if(tdps->pwrErrBoundBytes_size > 0){
unsigned char * signs = NULL;
if(tdps->pwrErrBoundBytes_size==0)
{
signs = (unsigned char*)malloc(dataSeriesLength);
memset(signs, 0, dataSeriesLength);
}
else
sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold && data[i] >= 0){
data[i] = 0;
continue;
}
if(signs[i]){
ptr = (uint64_t*)data + i;
*ptr |= 0x8000000000000000;
}
}
free(signs);
}
else{
for(size_t i=0; i<dataSeriesLength; i++){
if(data[i] < threshold) data[i] = 0;
}
}
}
#pragma GCC diagnostic pop
deps/SZ/sz/src/szd_float.c
浏览文件 @ 3e24cb15
此差异已折叠。
deps/SZ/sz/src/szd_float_pwr.c 已删除 100644 → 0
浏览文件 @ 3e8cfacd
此差异已折叠。
src/kit/taospack/taospack.c
浏览文件 @ 3e24cb15
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或