Skip to content

T
TDengine

loser646 / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 1
Fork 0
T
TDengine

体验新版 GitCode，发现更多精彩内容 >>

提交 f9bf4244 编写于 作者: T tickduan
浏览文件
操作

reduce files to smaller and delete no use files and function

上级 7c1ae052
展开全部 隐藏空白更改
内联 并排
Showing with 33 addition and 9139 deletion
deps/SZ/sz/Makefile.am 已删除 100644 → 0
浏览文件 @ 7c1ae052
#AM_CFLAGS = -I./include -I../zlib
#LDFLAGS=-fPIC -shared
AUTOMAKE_OPTIONS=foreign
if FORTRAN
include_HEADERS=include/MultiLevelCacheTable.h include/MultiLevelCacheTableWideInterval.h include/CacheTable.h include/defines.h\
include/CompressElement.h include/DynamicDoubleArray.h include/rw.h include/conf.h include/dataCompression.h\
include/dictionary.h include/DynamicFloatArray.h include/VarSet.h include/sz.h include/Huffman.h include/ByteToolkit.h include/szf.h\
include/sz_float.h include/sz_double.h include/callZlib.h include/iniparser.h include/TypeManager.h\
include/sz_int8.h include/sz_int16.h include/sz_int32.h include/sz_int64.h include/szd_int8.h include/szd_int16.h include/szd_int32.h include/szd_int64.h\
include/sz_uint8.h include/sz_uint16.h include/sz_uint32.h include/sz_uint64.h include/szd_uint8.h include/szd_uint16.h include/szd_uint32.h include/szd_uint64.h\
include/sz_float_pwr.h include/sz_double_pwr.h include/szd_float.h include/szd_double.h include/szd_float_pwr.h include/szd_double_pwr.h\
include/sz_float_ts.h include/szd_float_ts.h include/sz_double_ts.h include/szd_double_ts.h include/utility.h include/sz_opencl.h\
include/DynamicByteArray.h include/DynamicIntArray.h include/TightDataPointStorageI.h include/TightDataPointStorageD.h include/TightDataPointStorageF.h\
include/pastriD.h include/pastriF.h include/pastriGeneral.h include/pastri.h include/exafelSZ.h include/ArithmeticCoding.h include/sz_omp.h include/sz_stats.h sz.mod rw.mod
lib_LTLIBRARIES=libSZ.la
libSZ_la_CFLAGS=-I./include -I../zlib/ -I../zstd/
if TIMECMPR
libSZ_la_CFLAGS+=-DHAVE_TIMECMPR
endif
if RANDOMACCESS
libSZ_la_CFLAGS+=-DHAVE_RANDOMACCESS
endif
if OPENMP
libSZ_la_CFLAGS+=-fopenmp
endif
libSZ_la_LDFLAGS = -version-info 2:1:0
libSZ_la_LIDADD=../zlib/.libs/libzlib.a ../zstd/.libs/libzstd.a
libSZ_la_SOURCES=src/MultiLevelCacheTable.c src/MultiLevelCacheTableWideInterval.c \
src/ByteToolkit.c src/dataCompression.c src/DynamicIntArray.c src/iniparser.c src/szf.c \
src/CompressElement.c src/DynamicByteArray.c src/rw.c src/utility.c\
src/TightDataPointStorageI.c src/TightDataPointStorageD.c src/TightDataPointStorageF.c \
src/conf.c src/DynamicDoubleArray.c src/rwf.c src/TypeManager.c \
src/dictionary.c src/DynamicFloatArray.c src/VarSet.c src/callZlib.c src/Huffman.c \
src/sz_float.c src/sz_double.c src/sz_int8.c src/sz_int16.c src/sz_int32.c src/sz_int64.c\
src/sz_uint8.c src/sz_uint16.c src/sz_uint32.c src/sz_uint64.c src/szd_uint8.c src/szd_uint16.c src/szd_uint32.c src/szd_uint64.c\
src/szd_float.c src/szd_double.c src/szd_int8.c src/szd_int16.c src/szd_int32.c src/szd_int64.c src/sz.c\
src/sz_float_pwr.c src/sz_double_pwr.c src/szd_float_pwr.c src/szd_double_pwr.c src/ArithmeticCoding.c src/CacheTable.c\
src/sz_interface.F90 src/rw_interface.F90 src/exafelSZ.c
libSZ_la_LINK=$(AM_V_CC)$(LIBTOOL) --tag=FC --mode=link $(FCLD) $(libSZ_la_CFLAGS) -O3 $(libSZ_la_LDFLAGS) -o $(lib_LTLIBRARIES)
else
include_HEADERS=include/MultiLevelCacheTable.h include/MultiLevelCacheTableWideInterval.h include/CacheTable.h include/defines.h\
include/CompressElement.h include/DynamicDoubleArray.h include/rw.h include/conf.h include/dataCompression.h\
include/dictionary.h include/DynamicFloatArray.h include/VarSet.h include/sz.h include/Huffman.h include/ByteToolkit.h\
include/sz_float.h include/sz_double.h include/callZlib.h include/iniparser.h include/TypeManager.h\
include/sz_int8.h include/sz_int16.h include/sz_int32.h include/sz_int64.h include/szd_int8.h include/szd_int16.h include/szd_int32.h include/szd_int64.h\
include/sz_uint8.h include/sz_uint16.h include/sz_uint32.h include/sz_uint64.h include/szd_uint8.h include/szd_uint16.h include/szd_uint32.h include/szd_uint64.h\
include/sz_float_pwr.h include/sz_double_pwr.h include/szd_float.h include/szd_double.h include/szd_float_pwr.h include/szd_double_pwr.h\
include/sz_float_ts.h include/szd_float_ts.h include/sz_double_ts.h include/szd_double_ts.h include/utility.h include/sz_opencl.h\
include/DynamicByteArray.h include/DynamicIntArray.h include/TightDataPointStorageI.h include/TightDataPointStorageD.h include/TightDataPointStorageF.h\
include/pastriD.h include/pastriF.h include/pastriGeneral.h include/pastri.h include/exafelSZ.h include/ArithmeticCoding.h include/sz_omp.h include/sz_stats.h
lib_LTLIBRARIES=libSZ.la
libSZ_la_CFLAGS=-I./include -I../zlib -I../zstd/
if WRITESTATS
libSZ_la_CFLAGS+=-DHAVE_WRITESTATS
endif
if TIMECMPR
libSZ_la_CFLAGS+=-DHAVE_TIMECMPR
endif
if RANDOMACCESS
libSZ_la_CFLAGS+=-DHAVE_RANDOMACCESS
endif
if OPENMP
libSZ_la_CFLAGS+=-fopenmp
endif
libSZ_la_LDFLAGS = -version-info 1:4:0
libSZ_la_LIDADD=../zlib/.libs/libzlib.a ../zlib/.libs/libzstd.a
libSZ_la_SOURCES=src/MultiLevelCacheTable.c src/MultiLevelCacheTableWideInterval.c \
src/ByteToolkit.c src/dataCompression.c src/DynamicIntArray.c src/iniparser.c\
src/CompressElement.c src/DynamicByteArray.c src/rw.c src/utility.c\
src/TightDataPointStorageI.c src/TightDataPointStorageD.c src/TightDataPointStorageF.c \
src/conf.c src/DynamicDoubleArray.c src/TypeManager.c \
src/dictionary.c src/DynamicFloatArray.c src/VarSet.c src/callZlib.c src/Huffman.c \
src/sz_float.c src/sz_double.c src/sz_int8.c src/sz_int16.c src/sz_int32.c src/sz_int64.c\
src/sz_uint8.c src/sz_uint16.c src/sz_uint32.c src/sz_uint64.c src/szd_uint8.c src/szd_uint16.c src/szd_uint32.c src/szd_uint64.c\
src/szd_float.c src/szd_double.c src/szd_int8.c src/szd_int16.c src/szd_int32.c src/szd_int64.c src/sz.c\
src/sz_float_pwr.c src/sz_double_pwr.c src/szd_float_pwr.c src/szd_double_pwr.c src/ArithmeticCoding.c src/exafelSZ.c src/CacheTable.c
if PASTRI
libSZ_la_SOURCES+=src/pastri.c
endif
if OPENMP
libSZ_la_SOURCES+=src/sz_omp.c
endif
if TIMECMPR
libSZ_la_SOURCES+=src/sz_float_ts.c src/szd_float_ts.c src/sz_double_ts.c src/szd_double_ts.c
endif
if WRITESTATS
libSZ_la_SOURCES+=src/sz_stats.c
endif
libSZ_la_LINK= $(AM_V_CC)$(LIBTOOL) --tag=CC --mode=link $(CCLD) $(libSZ_la_CFLAGS) -O3 $(libSZ_la_LDFLAGS) -o $(lib_LTLIBRARIES)
endif
deps/SZ/sz/Makefile.in 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/include/ArithmeticCoding.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file ArithmeticCoding.h
* @author Sheng Di
* @date Dec, 2018
* @brief Header file for the ArithmeticCoding.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _ArithmeticCoding_H
#define _ArithmeticCoding_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#define ONE_FOURTH (0x40000000000) //44 bits are absolutely enough to deal with a large dataset (support at most 16TB per process)
#define ONE_HALF (0x80000000000)
#define THREE_FOURTHS (0xC0000000000)
#define MAX_CODE (0xFFFFFFFFFFF)
#define MAX_INTERVALS 1048576 //the limit to the arithmetic coding (at most 2^(20) intervals)
typedef struct Prob {
size_t low;
size_t high;
int state;
} Prob;
typedef struct AriCoder
{
int numOfRealStates; //the # real states menas the number of states after the optimization of # intervals
int numOfValidStates; //the # valid states means the number of non-zero frequency cells (some states/codes actually didn't appear)
size_t total_frequency;
Prob* cumulative_frequency; //used to encode data more efficiencly
} AriCoder;
void output_bit_1(unsigned int* buf);
void output_bit_0(unsigned int* buf);
unsigned int output_bit_1_plus_pending(int pending_bits);
unsigned int output_bit_0_plus_pending(int pending_bits);
AriCoder *createAriCoder(int numOfStates, int *s, size_t length);
void freeAriCoder(AriCoder *ariCoder);
void ari_init(AriCoder *ariCoder, int *s, size_t length);
unsigned int pad_ariCoder(AriCoder* ariCoder, unsigned char** out);
int unpad_ariCoder(AriCoder** ariCoder, unsigned char* bytes);
unsigned char get_bit(unsigned char* p, int offset);
void ari_encode(AriCoder *ariCoder, int *s, size_t length, unsigned char *out, size_t *outSize);
void ari_decode(AriCoder *ariCoder, unsigned char *s, size_t s_len, size_t targetLength, int *out);
Prob* getCode(AriCoder *ariCoder, size_t scaled_value);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _ArithmeticCoding_H ----- */
deps/SZ/sz/include/CacheTable.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file CacheTable.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef SZ_MASTER_CACHETABLE_H
#define SZ_MASTER_CACHETABLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "stdio.h"
#include "stdint.h"
#include <math.h>
extern double* g_CacheTable;
extern uint32_t * g_InverseTable;
extern uint32_t baseIndex;
extern uint32_t topIndex;
extern int bits;
int doubleGetExpo(double d);
int CacheTableGetRequiredBits(double precision, int quantization_intervals);
uint32_t CacheTableGetIndex(float value, int bits);
uint64_t CacheTableGetIndexDouble(double value, int bits);
int CacheTableIsInBoundary(uint32_t index);
void CacheTableBuild(double * table, int count, double smallest, double largest, double precision, int quantization_intervals);
uint32_t CacheTableFind(uint32_t index);
void CacheTableFree();
#ifdef __cplusplus
}
#endif
#endif //SZ_MASTER_CACHETABLE_H
deps/SZ/sz/include/DynamicDoubleArray.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file DynamicDoubleArray.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for Dynamic Double Array.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _DynamicDoubleArray_H
#define _DynamicDoubleArray_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
typedef struct DynamicDoubleArray
{
double* array;
size_t size;
double capacity;
} DynamicDoubleArray;
void new_DDA(DynamicDoubleArray **dda, size_t cap);
void convertDDAtoDoubles(DynamicDoubleArray *dba, double **data);
void free_DDA(DynamicDoubleArray *dda);
double getDDA_Data(DynamicDoubleArray *dda, size_t pos);
void addDDA_Data(DynamicDoubleArray *dda, double value);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _DynamicDoubleArray_H ----- */
deps/SZ/sz/include/DynamicFloatArray.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file DynamicFloatArray.h
* @author Sheng Di
* @date April, 2016
* @brief Header file for Dynamic Float Array.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _DynamicFloatArray_H
#define _DynamicFloatArray_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
typedef struct DynamicFloatArray
{
float* array;
size_t size;
size_t capacity;
} DynamicFloatArray;
void new_DFA(DynamicFloatArray **dfa, size_t cap);
void convertDFAtoFloats(DynamicFloatArray *dfa, float **data);
void free_DFA(DynamicFloatArray *dfa);
float getDFA_Data(DynamicFloatArray *dfa, size_t pos);
void addDFA_Data(DynamicFloatArray *dfa, float value);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _DynamicFloatArray_H ----- */
deps/SZ/sz/include/MultiLevelCacheTable.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file MultiLevelCacheTable.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _MULTILEVELCACHETABLE_H
#define _MULTILEVELCACHETABLE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <memory.h>
#include <stdlib.h>
#include "stdio.h"
typedef struct SubLevelTable{
uint32_t baseIndex;
uint32_t topIndex;
uint32_t* table;
uint8_t expoIndex;
} SubLevelTable;
typedef struct TopLevelTable{
uint8_t bits;
uint8_t baseIndex;
uint8_t topIndex;
struct SubLevelTable* subTables;
float bottomBoundary;
float topBoundary;
} TopLevelTable;
uint8_t MLCT_GetExpoIndex(float value);
uint8_t MLCT_GetRequiredBits(float precision);
uint32_t MLCT_GetMantiIndex(float value, int bits);
float MLTC_RebuildFloat(uint8_t expo, uint32_t manti, int bits);
void MultiLevelCacheTableBuild(struct TopLevelTable* topTable, float* precisionTable, int count, float precision);
uint32_t MultiLevelCacheTableGetIndex(float value, struct TopLevelTable* topLevelTable);
void MultiLevelCacheTableFree(struct TopLevelTable* table);
#ifdef __cplusplus
}
#endif
#endif //_MULTILEVELCACHETABLE_H
deps/SZ/sz/include/MultiLevelCacheTableWideInterval.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file MultiLevelCacheTableWideInterval.h
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Header file for MultiLevelCacheTableWideInterval.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _MULTILEVELCACHETABLEWIDEINTERVAL_H
#define _MULTILEVELCACHETABLEWIDEINTERVAL_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <memory.h>
#include <stdlib.h>
#include "stdio.h"
typedef struct SubLevelTableWideInterval{
uint64_t baseIndex;
uint64_t topIndex;
uint16_t* table;
uint16_t expoIndex;
} SubLevelTableWideInterval;
typedef struct TopLevelTableWideInterval{
uint16_t bits;
uint16_t baseIndex;
uint16_t topIndex;
struct SubLevelTableWideInterval* subTables;
double bottomBoundary;
double topBoundary;
} TopLevelTableWideInterval;
void freeTopLevelTableWideInterval(struct TopLevelTableWideInterval* topTable);
uint16_t MLCTWI_GetExpoIndex(double value);
uint16_t MLCTWI_GetRequiredBits(double precision);
uint64_t MLCTWI_GetMantiIndex(double value, int bits);
double MLTCWI_RebuildDouble(uint16_t expo, uint64_t manti, int bits);
void MultiLevelCacheTableWideIntervalBuild(struct TopLevelTableWideInterval* topTable, double* precisionTable, int count, double precision, int plus_bits);
uint32_t MultiLevelCacheTableWideIntervalGetIndex(double value, struct TopLevelTableWideInterval* topLevelTable);
void MultiLevelCacheTableWideIntervalFree(struct TopLevelTableWideInterval* table);
#ifdef __cplusplus
}
#endif
#endif //_MULTILEVELCACHETABLEWIDEINTERVAL_H
deps/SZ/sz/include/TypeManager.h
浏览文件 @ f9bf4244
......@@ -18,19 +18,12 @@ extern "C" {
#include <stdint.h>
//TypeManager.c
size_t convertIntArray2ByteArray_fast_1b(unsigned char* intArray, size_t intArrayLength, unsigned char **result);
size_t convertIntArray2ByteArray_fast_1b_to_result(unsigned char* intArray, size_t intArrayLength, unsigned char *result);
void convertByteArray2IntArray_fast_1b(size_t intArrayLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
size_t convertIntArray2ByteArray_fast_2b_inplace(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char *result);
void convertByteArray2IntArray_fast_2b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_3b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
void convertByteArray2IntArray_fast_3b(size_t stepLength, unsigned char* byteArray, size_t byteArrayLength, unsigned char **intArray);
size_t convertIntArray2ByteArray_fast_2b(unsigned char* timeStepType, size_t timeStepTypeLength, unsigned char **result);
int getLeftMovingSteps(size_t k, unsigned char resiBitLength);
size_t convertIntArray2ByteArray_fast_dynamic(unsigned char* timeStepType, unsigned char resiBitLength, size_t nbEle, unsigned char **bytes);
size_t convertIntArray2ByteArray_fast_dynamic2(unsigned char* timeStepType, unsigned char* resiBitLength, size_t resiBitLengthLength, unsigned char **bytes);
int computeBitNumRequired(size_t dataLength);
void decompressBitArraybySimpleLZ77(int** result, unsigned char* bytes, size_t bytesLength, size_t totalLength, int validLength);
#ifdef __cplusplus
}
......
deps/SZ/sz/include/VarSet.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file VarSet.h
* @author Sheng Di
* @date July, 2016
* @brief Header file for the Variable.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _VarSet_H
#define _VarSet_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
typedef struct sz_multisteps
{
char compressionType;
int predictionMode;
int lastSnapshotStep; //the previous snapshot step
unsigned int currentStep; //current time step of the execution/simulation
//void* ori_data; //original data pointer, which serve as the key for retrieving hist_data
void* hist_data; //historical data in past time steps
} sz_multisteps;
typedef struct SZ_Variable
{
unsigned char var_id;
char* varName;
char compressType; //102 means HZ; 101 means SZ
int dataType; //SZ_FLOAT or SZ_DOUBLE
size_t r5;
size_t r4;
size_t r3;
size_t r2;
size_t r1;
int errBoundMode;
double absErrBound;
double relBoundRatio;
double pwRelBoundRatio;
void* data;
sz_multisteps *multisteps;
unsigned char* compressedBytes;
size_t compressedSize;
struct SZ_Variable* next;
} SZ_Variable;
typedef struct SZ_VarSet
{
unsigned short count;
struct SZ_Variable *header;
struct SZ_Variable *lastVar;
} SZ_VarSet;
void free_Variable_keepOriginalData(SZ_Variable* v);
void free_Variable_keepCompressedBytes(SZ_Variable* v);
void free_Variable_all(SZ_Variable* v);
void SZ_batchAddVar(int var_id, char* varName, int dataType, void* data,
int errBoundMode, double absErrBound, double relBoundRatio, double pwRelBoundRatio,
size_t r5, size_t r4, size_t r3, size_t r2, size_t r1);
int SZ_batchDelVar_vset(SZ_VarSet* vset, char* varName);
int SZ_batchDelVar(char* varName);
int SZ_batchDelVar_ID_vset(SZ_VarSet* vset, int var_id);
int SZ_batchDelVar_ID(int var_id);
SZ_Variable* SZ_searchVar(char* varName);
void* SZ_getVarData(char* varName, size_t *r5, size_t *r4, size_t *r3, size_t *r2, size_t *r1);
void free_VarSet_vset(SZ_VarSet *vset, int mode);
void SZ_freeVarSet(int mode);
void free_multisteps(sz_multisteps* multisteps);
int checkVarID(unsigned char cur_var_id, unsigned char* var_ids, int var_count);
SZ_Variable* SZ_getVariable(int var_id);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _VarSet_H ----- */
deps/SZ/sz/include/conf.h
浏览文件 @ f9bf4244
......@@ -20,11 +20,8 @@ extern "C" {
void updateQuantizationInfo(int quant_intervals);
int SZ_ReadConf(const char* sz_cfgFile);
int SZ_LoadConf(const char* sz_cfgFile);
int checkVersion(unsigned char version);
int computeVersion(int major, int minor, int revision);
int checkVersion2(char* version);
void initSZ_TSC();
unsigned int roundUpToPowerOf2(unsigned int base);
double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range);
double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle);
......
deps/SZ/sz/include/pastri.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
//CHECK:
//What happens when ECQBits==1, or ECQBits==0 or ECQBits<0?
//Rounding? Scale originalEb by 0.99?
//Possible improvement: Change GAMESS format: {i i i i d} -> {i}{i}{i}{i}{d}
//Possible improvement: Optimize bookkeeping bits
//Possible improvement: Guess the type (C/UC, Sparse/Not)
//Possible improvement: Get rid of writing/reading some of the indexes to in/out buffers
//Possible improvement: Get rid of all debug stuff, including Makefile debug flags
//Possible improvement: Get rid of "compressedBytes"
//Possible improvement: SparseCompressed, ECQBits=2: 1's and -1's can be represented by just 0 and 1, instead 10 and 11.
//Possible improvement: SparseCompressed, ECQBits>2: Again: 1: 10, -1:11, Others: 0XX...XX
//Possible improvement: WriteBitsFast: maybe remove some masks?
//Possible improvement: WriteBitsFast: Get rid of multiple calls!
//Possible improvement: UCSparse: Indexes use 64 bits. It can be lowered to _1DIdxBits
//Possible improvement: Parameters: Smaller data sizes may be possible!
#ifndef PASTRI_H
#define PASTRI_H
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <math.h>
#include <assert.h> //Just for debugging purposes!
//#define DATASIZE 8 //Bytes per input data point.
//We have only 1 double per data point, so it is 8 bytes.
#define MAX_PS_SIZE 100
#define MAX_BLOCK_SIZE 10000
#define MAX_BUFSIZE 160000 //Should be a multiple of 8
#define D_W 0 //Debug switch: Write (input block)
#define D_R 0 //Debug switch: Read (compressed block)
#define D_G 0 //Debug switch: General
#define D_G2 0 //Debug switch: General 2 (a little more detail)
#define D_C 0 //Debug switch: C
//#define DEBUG 1 //Debug switch
//#define BOOKKEEPINGBITS 0 //Currently unused
//#define BOOKKEEPINGBITS 120 //Includes: mode, indexOffsets, compressedBytes, Pb_, ECQBits_ (8+64+32+8+8)
//BOOKKEEPINGBITS is defined here, because if P & S is going to be used, they appear just after the bookkeeping part.
//This allows us to write P and S directly onto using outBuf.
// IMPORTANT NOTE:
//Read/Write up to 56 bits.
//More than that is not supported!
/********************************************************************/
//Datatype Declarations:
/********************************************************************/
typedef struct pastri_params{
double originalEb; //Error Bound entered by the user
double usedEb; //Error Bound used during compression/deceompression
int numBlocks; //Number of blocks to be compressed
int dataSize; //8(=Double) or 4(=Float)
int bf[4]; //Orbital types (basis function types). Typically in range [0,3]
int idxRange[4]; //Ranges of indexes. idxRange[i]=(bf[i]+1)*(bf[i]+2)/2;
int sbSize; //=idxRange[2]*idxRange[3];
int sbNum; //=idxRange[0]*idxRange[1];
int bSize; //=sbSize*sbNum;
//uint16_t idxOffset[4]; //Index offset values. No longer used.
}pastri_params;
//Block-specific stuff:
typedef struct pastri_blockParams{
uint16_t nonZeros;
//int ECQ0s; //= p->bSize - numOutliers //OR: p->bSize=ECQ0s+ECQ1s+ECQOthers
int ECQ1s;
int ECQOthers;
int numOutliers; //=ECQ1s+ECQOthers
int patternBits;
int scaleBits;
double binSize;
double scalesBinSize;
uint64_t ECQExt;
int ECQBits;
int _1DIdxBits;
}pastri_blockParams;
typedef union u_UI64I64D{
uint64_t ui64;
int64_t i64;
double d;
} u_UI64I64D;
/********************************************************************/
//Function Prototypes:
/********************************************************************/
void SZ_pastriReadParameters(char paramsFilename[512],pastri_params *paramsPtr);
//Read the basic PaSTRI parameters from a file, speficied by paramsFilename.
void SZ_pastriPreprocessParameters(pastri_params *p);
//Using basic PaSTRI parameters, generate the others.
//For example, block and sub-block sizes are generated by using basis function types.
void SZ_pastriCompressBatch(pastri_params *p,unsigned char *originalBuf, unsigned char** compressedBufP,size_t *compressedBytes);
//INPUTS: p, originalBuf
//OUTPUTS: compressedBufP, compressedBytes
//Using the inputs, compressedBufP is allocated and populated by the compressed data. Compressed size is written into compressedBytes.
//Parameters are also stored at the beginning part of the compressedBuf
void SZ_pastriDecompressBatch(unsigned char*compressedBuf, pastri_params *p, unsigned char** decompressedBufP ,size_t *decompressedBytes);
//INPUTS: compressedBuf
//OUTPUTS: p, decompressedBufP, decompressedBytes
//First, parameters are read from compressedBuf and written into p.
//Then, decompressedBufP is allocated and populated by the decompressed data. Decompressed size is written into decompressedBytes.
void SZ_pastriCheckBatch(pastri_params *p,unsigned char*originalBuf,unsigned char*decompressedBuf);
//INPUTS: p, originalBuf, decompressedBuf
//OUTPUTS: None (Just some on-screen messages)
//Compares originalBuf with decompressedBuf. Checks whether the absolute error condition is satisfied or not.
/********************************************************************/
//Other Includes:
/********************************************************************/
#include "pastriGeneral.h" //General tools
#include "pastriD.h" //Compression/Decompression for Double data
#include "pastriF.h" //Compression/Decompression for Float data
#endif
deps/SZ/sz/include/pastriD.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/include/pastriF.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/include/pastriGeneral.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
#ifndef PASTRIGENERAL_H
#define PASTRIGENERAL_H
static inline double abs_FastD(double x){
u_UI64I64D u1;
u1.d=x;
//(*((uint64_t *)(&x)))&=(int64_t)0x7FFFFFFFFFFFFFFF;
u1.ui64&=(int64_t)0x7FFFFFFFFFFFFFFF;
return u1.d;
}
static inline int64_t abs_FastI64(int64_t x){
return (x^((x&(int64_t)0x8000000000000000)>>63))+((x&(int64_t)0x8000000000000000)!=0);
}
/*
int abs(int x) {
int mask = (x >> (sizeof(int) * CHAR_BIT - 1));
return (x + mask) ^ mask;
}
*/
//Returns the min. bits needed to represent x.
//Same as: ceil(log2(abs(x)))
//Actually to be completely safe, it correspond to: ceil(log2(abs(i)+1))+0.1
//+0.1 was for fixing rounding errors
//REMEMBER: To represent the whole range [-x:x], the number of bits required is bitsNeeded(x)+1
static inline int bitsNeeded_double(double x){
u_UI64I64D u1;
u1.d=x;
return (((u1.ui64<<1)>>53)-1022) & (((x!=0)<<31)>>31);
}
//Returns the min. bits needed to represent x.
//Same as: ceil(log2(abs(x)))
//NEEDS OPTIMIZATION!
static inline int bitsNeeded_float(float x){
u_UI64I64D u1;
u1.d=x; //Casting to Double!
return (((u1.ui64<<1)>>53)-1022) & (((x!=0)<<31)>>31);
}
static inline int bitsNeeded_UI64(uint64_t x){
int shift;
int res=0;
//Get the absolute value of x:
//x=(x^((x&(int64_t)0x8000000000000000)>>63))+((x&(int64_t)0x8000000000000000)!=0);
//x=abs_FastI64(x);
//printf("%d\n",(x&(uint64_t)0xFFFFFFFF00000000)!=0);
shift=(((x&(uint64_t)0xFFFFFFFF00000000)!=0)*32);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x00000000FFFF0000)!=0);
shift=(((x&(uint64_t)0x00000000FFFF0000)!=0)*16);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x000000000000FF00)!=0);
shift=(((x&(uint64_t)0x000000000000FF00)!=0)*8);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x00000000000000F0)!=0);
shift=(((x&(uint64_t)0x00000000000000F0)!=0)*4);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x000000000000000C)!=0);
shift=(((x&(uint64_t)0x000000000000000C)!=0)*2);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x0000000000000002)!=0);
shift=((x&(uint64_t)0x0000000000000002)!=0);
x>>=shift;
res+=shift;
//printf("%d\n",(x&(uint64_t)0x0000000000000001)!=0);
shift=((x&(uint64_t)0x0000000000000001)!=0);
x>>=shift;
res+=shift;
//printf("BITS NEEDED: %d\n",res);
return res;
}
static inline int bitsNeeded_I64(int64_t x){
uint64_t ux;
ux=abs_FastI64(x);
return bitsNeeded_UI64(ux);
}
//Implementations(They are inline, so they should be in this header file)
static inline int myEndianType(){ //Should work for most cases. May not work at mixed endian systems.
uint64_t n=1;
if (*(unsigned char*)&n == 1){
//cout<<"Little-Endian"<<endl;
return 0; //0 for little endian
}
else{
//cout<<"Big-Endian"<<endl;
return 1; //1 for big endian
}
}
static inline void flipBytes_UI64(uint64_t *dataPtr){
unsigned char*tempA;
char temp8b;
tempA=(unsigned char*)dataPtr;
temp8b=tempA[7];
tempA[7]=tempA[0];
tempA[0]=temp8b;
temp8b=tempA[6];
tempA[6]=tempA[1];
tempA[1]=temp8b;
temp8b=tempA[5];
tempA[5]=tempA[2];
tempA[2]=temp8b;
temp8b=tempA[4];
tempA[4]=tempA[3];
tempA[3]=temp8b;
return;
}
//WARNING: readBits works properly only on Little Endian machines! (For Big Endians, some modifications are needed)
static inline uint64_t readBits_UI64(unsigned char* buffer,uint64_t *bitPosPtr,char numBits){ // numBits must be in range [0:56]
uint64_t mask = ((uint64_t)0x0000000000000001<<numBits)-1;
//cout<<"bitPos:"<<(*bitPosPtr)<<"\tbitPos>>3:"<<(*bitPosPtr>>3)<<endl;
uint64_t temp64b = *(uint64_t*)(buffer + ( *bitPosPtr >> 3));
//NOTE: bitPos>>3 is the same as bitPos/8
temp64b >>= (*bitPosPtr) & (uint64_t)0x0000000000000007;
//cout<<endl;
//cout<<"bitpos>>3:"<<(bitPos>>3)<<" bitPos&0x7:"<<(bitPos & 0x00000007)<<" bitPos%8:"<<(bitPos%8)<<endl;
//cout<<"Read:"<<(temp64b & mask)<<" temp64b:"<<temp64b<<" Mask:"<<mask<<" numBits:"<<numBits<<endl;
(*bitPosPtr) += numBits;
return (temp64b & mask);
}
static inline int64_t readBits_I64(unsigned char* buffer,uint64_t *bitPosPtr,char numBits){ // numBits must be in range [0:56]
int64_t val;
val=readBits_UI64(buffer,bitPosPtr,numBits);//Read value
int64_t shiftAmount=64-numBits;
val=(val<<shiftAmount)>>shiftAmount;//Sign correction
return val;
}
//WARNING: readBits_EndianSafe is not tested on Big-Endian machines
static inline uint64_t readBits_EndianSafe(unsigned char* buffer,uint64_t *bitPosPtr,char numBits){ // numBits must be in range [0:56]
uint64_t mask = ((uint64_t)0x0000000000000001<<numBits)-1;
uint64_t temp64b = *(uint64_t*)(buffer + ((*bitPosPtr)>>3));
//NOTE: (*bitPosPtr)>>3 is the same as (*bitPosPtr)/8
if(myEndianType())
flipBytes_UI64(&temp64b);
temp64b >>= (*bitPosPtr) & (uint64_t)0x0000000000000007;
(*bitPosPtr) += numBits;
return temp64b & mask;
}
//WARNING: writeBits_Fast works properly only on Little Endian machines! (For Big Endians, some modifications are needed)
//The buffer should be initialized as 0's for this to work!
//Also, the range of data is not checked!(If data exceeds numBits, it may be cause problems)
static inline void writeBits_Fast(unsigned char* buffer,uint64_t *bitPosPtr,char numBits,int64_t data){
//if(DEBUG){printf("writeBits_Fast: data:0x%lx %ld\n",data,data);} //DEBUG
//if(DEBUG){printf("writeBits_Fast: numBits:0x%lx %ld\n",numBits,numBits);} //DEBUG
uint64_t mask = ((uint64_t)0x0000000000000001<<numBits)-1;
//if(DEBUG){printf("writeBits_Fast: mask:0x%lx %ld\n",mask,mask);} //DEBUG
//if(DEBUG){printf("writeBits_Fast: data&mask:0x%lx %ld\n",((*(uint64_t*)&data)&mask),((*(uint64_t*)&data)&mask));} //DEBUG
//if(DEBUG){printf("writeBits_Fast: buffer_O:0x%lx\n",*(uint64_t*)(buffer + ((*bitPosPtr)>>3)));} //DEBUG
*(uint64_t*)(buffer + ((*bitPosPtr)>>3)) |= ((*(uint64_t*)&data)&mask) << ((*bitPosPtr) & (uint64_t)0x0000000000000007);
//if(DEBUG){printf("writeBits_Fast: buffer_N:0x%lx\n",*(uint64_t*)(buffer + ((*bitPosPtr)>>3)));} //DEBUG
(*bitPosPtr) += numBits;
}
//WARNING: writeBits_EndianSafe is not tested on Big-Endian machines
static inline void writeBits_EndianSafe(unsigned char* buffer,uint64_t *bitPosPtr,char numBits,uint64_t data){
uint64_t mask = ((uint64_t)0x0000000000000001<<numBits)-1;
data=data&mask;
uint64_t temp64b_inBuffer=*(uint64_t*)(buffer + ((*bitPosPtr)>>3));
uint64_t temp64b_outBuffer=data << ((*bitPosPtr) & (uint64_t)0x0000000000000007);
if(myEndianType()){
flipBytes_UI64(&temp64b_inBuffer);
}
temp64b_outBuffer |= temp64b_inBuffer;
if(myEndianType()){
flipBytes_UI64(&temp64b_outBuffer);
}
*(uint64_t*)(buffer + ((*bitPosPtr)>>3))=temp64b_outBuffer; // "|=" may also work
(*bitPosPtr) += numBits;
}
#endif
deps/SZ/sz/include/rw.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file io.h
* @author Sheng Di
* @date April, 2015
* @brief Header file for the whole io interface.
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _IO_H
#define _IO_H
#include <stdio.h>
#include <stdint.h>
#ifdef _WIN32
#define PATH_SEPARATOR ';'
#else
#define PATH_SEPARATOR ':'
#endif
#ifdef __cplusplus
extern "C" {
#endif
int checkFileExistance(char* filePath);
float** create2DArray_float(size_t m, size_t n);
void free2DArray_float(float** data, size_t m);
float*** create3DArray_float(size_t p, size_t m, size_t n);
void free3DArray_float(float*** data, size_t p, size_t m);
double** create2DArray_double(size_t m, size_t n);
void free2DArray_double(double** data, size_t m);
double*** create3DArray_double(size_t p, size_t m, size_t n);
void free3DArray_double(double*** data, size_t p, size_t m);
size_t checkFileSize(char *srcFilePath, int *status);
unsigned char *readByteData(char *srcFilePath, size_t *byteLength, int *status);
double *readDoubleData(char *srcFilePath, size_t *nbEle, int *status);
int8_t *readInt8Data(char *srcFilePath, size_t *nbEle, int *status);
int16_t *readInt16Data(char *srcFilePath, size_t *nbEle, int *status);
uint16_t *readUInt16Data(char *srcFilePath, size_t *nbEle, int *status);
int32_t *readInt32Data(char *srcFilePath, size_t *nbEle, int *status);
uint32_t *readUInt32Data(char *srcFilePath, size_t *nbEle, int *status);
int64_t *readInt64Data(char *srcFilePath, size_t *nbEle, int *status);
uint64_t *readUInt64Data(char *srcFilePath, size_t *nbEle, int *status);
float *readFloatData(char *srcFilePath, size_t *nbEle, int *status);
unsigned short* readShortData(char *srcFilePath, size_t *dataLength, int *status);
double *readDoubleData_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
int8_t *readInt8Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
int16_t *readInt16Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
uint16_t *readUInt16Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
int32_t *readInt32Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
uint32_t *readUInt32Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
int64_t *readInt64Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
uint64_t *readUInt64Data_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
float *readFloatData_systemEndian(char *srcFilePath, size_t *nbEle, int *status);
void writeByteData(unsigned char *bytes, size_t byteLength, char *tgtFilePath, int *status);
void writeDoubleData(double *data, size_t nbEle, char *tgtFilePath, int *status);
void writeFloatData(float *data, size_t nbEle, char *tgtFilePath, int *status);
void writeDataSZ(void *data, int dataType, size_t nbEle, char *tgtFilePath, int *status);
void writeFloatData_inBytes(float *data, size_t nbEle, char* tgtFilePath, int *status);
void writeDoubleData_inBytes(double *data, size_t nbEle, char* tgtFilePath, int *status);
void writeShortData_inBytes(short *states, size_t stateLength, char *tgtFilePath, int *status);
void writeUShortData_inBytes(unsigned short *states, size_t stateLength, char *tgtFilePath, int *status);
void writeIntData_inBytes(int *states, size_t stateLength, char *tgtFilePath, int *status);
void writeUIntData_inBytes(unsigned int *states, size_t stateLength, char *tgtFilePath, int *status);
void writeLongData_inBytes(int64_t *states, size_t stateLength, char *tgtFilePath, int *status);
void writeULongData_inBytes(uint64_t *states, size_t stateLength, char *tgtFilePath, int *status);
void writeStrings(int nbStr, char *str[], char *tgtFilePath, int *status);
//void convertToPFM_float(float *data, size_t r5, size_t r4, size_t r3, size_t r2, size_t r1, int endianType, char *tgtFilePath, int *status);
void checkfilesizec_(char *srcFilePath, int *len, size_t *filesize);
void readbytefile_(char *srcFilePath, int *len, unsigned char *bytes, size_t *byteLength);
void readdoublefile_(char *srcFilePath, int *len, double *data, size_t *nbEle);
void readfloatfile_(char *srcFilePath, int *len, float *data, size_t *nbEle);
void writebytefile_(unsigned char *bytes, size_t *byteLength, char *tgtFilePath, int *len);
void writedoublefile_(double *data, size_t *nbEle, char *tgtFilePath, int *len);
void writefloatfile_(float *data, size_t *nbEle, char *tgtFilePath, int *len);
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _IO_H ----- */
deps/SZ/sz/include/sz.h
浏览文件 @ f9bf4244
......@@ -18,7 +18,6 @@
#include "CompressElement.h"
#include "DynamicByteArray.h"
#include "DynamicIntArray.h"
#include "VarSet.h"
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "conf.h"
......@@ -29,15 +28,8 @@
#include "sz_double.h"
#include "szd_float.h"
#include "szd_double.h"
#include "sz_opencl.h"
#include "callZlib.h"
#include "rw.h"
#include "pastri.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
#include "sz_stats.h"
#ifdef _WIN32
#define PATH_SEPARATOR ';'
......@@ -172,15 +164,6 @@ extern sz_params *confparams_cpr;
extern sz_params *confparams_dec;
extern sz_exedata *exe_params;
//------------------------------------------------
extern SZ_VarSet* sz_varset;
extern sz_multisteps *multisteps; //compression based on multiple time steps (time-dimension based compression)
extern sz_tsc_metadata *sz_tsc;
//for pastri
#ifdef PASTRI
extern pastri_params pastri_par;
#endif
void SZ_Finalize();
......
deps/SZ/sz/include/sz_float.h
浏览文件 @ f9bf4244
......@@ -6,7 +6,6 @@
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "DynamicFloatArray.h"
#ifndef _SZ_Float_H
#define _SZ_Float_H
......@@ -14,13 +13,11 @@
#ifdef __cplusplus
extern "C" {
#endif
unsigned char* SZ_skip_compress_float(float* data, size_t dataLength, size_t* outSize);
void computeReqLength_float(double realPrecision, short radExpo, int* reqLength, float* medianValue);
unsigned int optimize_intervals_float_1D(float *oriData, size_t dataLength, double realPrecision);
unsigned int optimize_intervals_and_compute_dense_position_float_1D(float *oriData, size_t dataLength, double realPrecision, float * dense_pos);
unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength, double realPrecision);
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
......@@ -29,13 +26,6 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_
bool SZ_compress_args_float_NoCkRngeNoGzip_1D( unsigned char* newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
size_t SZ_compress_float_1D_MDQ_RA_block(float * block_ori_data, float * mean, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, float * unpredictable_data);
size_t SZ_compress_float_1D_MDQ_RA_block_1D_pred(float * block_ori_data, float * mean, float dense_pos, size_t dim_0, size_t block_dim_0, double realPrecision, int * type, DynamicFloatArray * unpredictable_data);
void SZ_blocked_regression(float * block_ori_data, size_t dim_0, size_t dim_1, size_t dim_2, size_t block_dim_0, size_t block_dim_1, size_t block_dim_2, float *params);
unsigned char * SZ_compress_float_1D_MDQ_RA(float *oriData, size_t r1, double realPrecision, size_t * comp_size);
void SZ_compress_args_float_withinRange(unsigned char* newByteData, float *oriData, size_t dataLength, size_t *outSize);
......
deps/SZ/sz/include/sz_opencl.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
//make header C++/C inter-operable
#ifdef __cplusplus
extern "C" {
#endif
#ifndef SZ_OPENCL_H
#define SZ_OPENCL_H
#include<stddef.h>
//opaque pointer for opencl state
struct sz_opencl_state;
/**
* creates an opencl state for multiple uses of the compressor or
* returns an error code.
*
* \post if return code is SZ_NCES, the state object may only be passed to
* sz_opencl_release or sz_opencl_error_* otherwise it may be used in any
* sz_opencl_* function.
*
* \param[out] state the sz opencl state
* \return SZ_SUCCESS for success or SZ_NCES on error
*/
int sz_opencl_init(struct sz_opencl_state** state);
/**
* deinitializes an opencl state
*
* \param[in] state the sz opencl state
* \return SZ_SUCCESS
*/
int sz_opencl_release(struct sz_opencl_state** state);
/**
* returns a human readable error message for the last error recieved by state
*
* \param[in] state the sz opencl state
* \return a pointer to a string that describes the error
*/
const char* sz_opencl_error_msg(struct sz_opencl_state* state);
/**
* returns a numeric code for the last error recieved by state
*
* \param[in] state the sz opencl state
* \return the numeric error code
*/
int sz_opencl_error_code(struct sz_opencl_state* state);
/**
* confirms that the sz opencl state is ready to use by performing a vector addition
*
* \param[in] state the sz opencl state
* \return SZ_SUCCESS if the opencl implementation is functioning
*/
int sz_opencl_check(struct sz_opencl_state*);
unsigned char* sz_compress_float3d_opencl(float* data, size_t r1, size_t r2, size_t r3, double, size_t* out_size);
#endif /* SZ_OPENCL_H */
//make header C++/C inter-operable
#ifdef __cplusplus
}
#endif
deps/SZ/sz/include/sz_stats.h 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file ByteToolkit.h
* @author Sheng Di
* @date July, 2017
* @brief Header file for the ByteToolkit.c.
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef _STATS_H
#define _STATS_H
#include <stdint.h>
#include <stdio.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct sz_stats
{
int use_mean;
size_t blockSize;
float lorenzoPercent;
float regressionPercent;
size_t lorenzoBlocks;
size_t regressionBlocks;
size_t totalBlocks;
//size_t huffmanTreeHeight;
size_t huffmanTreeSize; //before the final zstd
size_t huffmanCodingSize; //before the final zstd
float huffmanCompressionRatio;
int huffmanNodeCount;
size_t unpredictCount;
float unpredictPercent;
float zstdCompressionRatio; //not available yet
} sz_stats;
extern sz_stats sz_stat;
void writeBlockInfo(int use_mean, size_t blockSize, size_t regressionBlocks, size_t totalBlocks);
void writeHuffmanInfo(size_t huffmanTreeSize, size_t huffmanCodingSize, size_t totalDataSize, int huffmanNocdeCount);
void writeZstdCompressionRatio(float zstdCompressionRatio);
void writeUnpredictDataCounts(size_t unpredictCount, size_t totalNumElements);
void printSZStats();
#ifdef __cplusplus
}
#endif
#endif /* ----- #ifndef _STATS_H ----- */
deps/SZ/sz/src/ArithmeticCoding.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/CacheTable.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file CacheTable.c
* @author Xiangyu Zou, Tao Lu, Wen Xia, Xuan Wang, Weizhe Zhang, Sheng Di, Dingwen Tao
* @date Jan, 2019
* @brief Cache Table
* (C) 2016 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include "CacheTable.h"
double* g_CacheTable;
uint32_t * g_InverseTable;
uint32_t baseIndex;
uint32_t topIndex;
int bits;
inline int doubleGetExpo(double d){
long* ptr = (long*)&d;
*ptr = ((*ptr) >> 52) - 1023;
return *ptr;
}
int CacheTableGetRequiredBits(double precision, int quantization_intervals){
double min_distance = pow((1+precision), -(quantization_intervals>>1)) * precision;
return -(doubleGetExpo(min_distance));
}
inline uint32_t CacheTableGetIndex(float value, int bits){
uint32_t* ptr = (uint32_t*)&value;
int shift = 32 - 9 - bits;
if(shift>0){
return (*ptr) >> shift;
}else{
return 0;
}
}
inline uint64_t CacheTableGetIndexDouble(double value, int bits){
uint64_t* ptr = (uint64_t*)&value;
int shift = 64 - 12 - bits;
if(shift>0){
return (*ptr) >> shift;
}else{
return 0;
}
}
inline int CacheTableIsInBoundary(uint32_t index){
if(index <= topIndex && index > baseIndex){
return 1;
}else{
return 0;
}
}
void CacheTableBuild(double * table, int count, double smallest, double largest, double precision, int quantization_intervals){
bits = CacheTableGetRequiredBits(precision, quantization_intervals);
baseIndex = CacheTableGetIndex((float)smallest, bits)+1;
topIndex = CacheTableGetIndex((float)largest, bits);
uint32_t range = topIndex - baseIndex + 1;
g_InverseTable = (uint32_t *)malloc(sizeof(uint32_t) * range);
/*
uint32_t fillInPos = 0;
for(int i=0; i<count; i++){
if(i == 0){
continue;
}
uint32_t index = CacheTableGetIndex((float)table[i], bits) - baseIndex;
g_InverseTable[index] = i;
if(index > fillInPos){
for(int j=fillInPos; j<index; j++){
g_InverseTable[j] = g_InverseTable[index];
}
}
fillInPos = index + 1;
}
*/
for(int i=count-1; i>0; i--){
uint32_t upperIndex = CacheTableGetIndex((float)table[i]*(1+precision), bits);
uint32_t lowerIndex = CacheTableGetIndex((float)table[i]/(1+precision), bits);
for(uint32_t j = lowerIndex; j<=upperIndex; j++){
if(j<baseIndex || j >topIndex){
continue;
}
g_InverseTable[j-baseIndex] = i;
}
}
}
inline uint32_t CacheTableFind(uint32_t index){
return g_InverseTable[index-baseIndex];
}
void CacheTableFree(){
free(g_InverseTable);
}
deps/SZ/sz/src/DynamicDoubleArray.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file DynamicFloatArray.c
* @author Sheng Di
* @date May, 2016
* @brief Dynamic Float Array
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "DynamicDoubleArray.h"
void new_DDA(DynamicDoubleArray **dda, size_t cap) {
*dda = (DynamicDoubleArray *)malloc(sizeof(DynamicDoubleArray));
(*dda)->size = 0;
(*dda)->capacity = cap;
(*dda)->array = (double*)malloc(sizeof(double)*cap);
}
void convertDDAtoDoubles(DynamicDoubleArray *dba, double **data)
{
size_t size = dba->size;
if(size>0)
*data = (double*)malloc(size * sizeof(double));
else
*data = NULL;
memcpy(*data, dba->array, size*sizeof(double));
}
void free_DDA(DynamicDoubleArray *dda)
{
free(dda->array);
free(dda);
}
double getDDA_Data(DynamicDoubleArray *dda, size_t pos)
{
if(pos>=dda->size)
{
printf("Error: wrong position of DIA.\n");
exit(0);
}
return dda->array[pos];
}
void addDDA_Data(DynamicDoubleArray *dda, double value)
{
if(dda->size==dda->capacity)
{
dda->capacity *= 2;
dda->array = (double *)realloc(dda->array, dda->capacity*sizeof(double));
}
dda->array[dda->size] = value;
dda->size ++;
}
deps/SZ/sz/src/DynamicFloatArray.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
/**
* @file DynamicFloatArray.c
* @author Sheng Di
* @date May, 2016
* @brief Dynamic Float Array
* (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "DynamicFloatArray.h"
void new_DFA(DynamicFloatArray **dfa, size_t cap) {
*dfa = (DynamicFloatArray *)malloc(sizeof(DynamicFloatArray));
(*dfa)->size = 0;
(*dfa)->capacity = cap;
(*dfa)->array = (float*)malloc(sizeof(float)*cap);
}
void convertDFAtoFloats(DynamicFloatArray *dfa, float **data)
{
size_t size = dfa->size;
if(size>0)
*data = (float*)malloc(size * sizeof(float));
else
*data = NULL;
memcpy(*data, dfa->array, size*sizeof(float));
}
void free_DFA(DynamicFloatArray *dfa)
{
free(dfa->array);
free(dfa);
}
float getDFA_Data(DynamicFloatArray *dfa, size_t pos)
{
if(pos>=dfa->size)
{
printf("Error: wrong position of DIA.\n");
exit(0);
}
return dfa->array[pos];
}
void addDFA_Data(DynamicFloatArray *dfa, float value)
{
if(dfa->size==dfa->capacity)
{
dfa->capacity *= 2;
dfa->array = (float *)realloc(dfa->array, dfa->capacity*sizeof(float));
}
dfa->array[dfa->size] = value;
dfa->size++;
}
deps/SZ/sz/src/MultiLevelCacheTable.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/TypeManager.c
浏览文件 @ f9bf4244
此差异已折叠。
deps/SZ/sz/src/VarSet.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/conf.c
浏览文件 @ f9bf4244
......@@ -375,15 +375,7 @@ int SZ_ReadConf(const char* sz_cfgFile) {
//initialization for Huffman encoding
//SZ_Reset();
}
else if(confparams_cpr->sol_ID == PASTRI)
{//load parameters for PSTRI
pastri_par.bf[0] = (int)iniparser_getint(ini, "PARAMETER:basisFunction_0", 0);
pastri_par.bf[1] = (int)iniparser_getint(ini, "PARAMETER:basisFunction_1", 0);
pastri_par.bf[2] = (int)iniparser_getint(ini, "PARAMETER:basisFunction_2", 0);
pastri_par.bf[3] = (int)iniparser_getint(ini, "PARAMETER:basisFunction_3", 0);
pastri_par.numBlocks = (int)iniparser_getint(ini, "PARAMETER:numBlocks", 0);
confparams_cpr->absErrBound = pastri_par.originalEb = (double)iniparser_getdouble(ini, "PARAMETER:absErrBound", 1E-3);
}
iniparser_freedict(ini);
return SZ_SUCCESS;
......@@ -408,31 +400,3 @@ int SZ_LoadConf(const char* sz_cfgFile) {
}
return SZ_SUCCESS;
}
int checkVersion(unsigned char version)
{
return version <= versionNumber;
}
void initSZ_TSC()
{
sz_tsc = (sz_tsc_metadata*)malloc(sizeof(sz_tsc_metadata));
memset(sz_tsc, 0, sizeof(sz_tsc_metadata));
/*sprintf(sz_tsc->metadata_filename, "sz_tsc_metainfo.txt");
sz_tsc->metadata_file = fopen(sz_tsc->metadata_filename, "wb");
if (sz_tsc->metadata_file == NULL)
{
printf("Failed to open sz_tsc_metainfo.txt file for writing metainfo.\n");
exit(1);
}
fputs("#metadata of the time-step based compression\n", sz_tsc->metadata_file); */
}
/*double fabs(double value)
{
if(value<0)
return -value;
else
return value;
}*/
deps/SZ/sz/src/pastri.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/rw.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/rw_interface.F90 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/rwf.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/sz.c
浏览文件 @ f9bf4244
......@@ -18,7 +18,6 @@
#include "TightDataPointStorageD.h"
#include "TightDataPointStorageF.h"
#include "zlib.h"
#include "rw.h"
#include "conf.h"
#include "utility.h"
......@@ -36,16 +35,6 @@ sz_params *confparams_dec = NULL; //used for decompression
sz_exedata *exe_params = NULL;
/*following global variables are desgined for time-series based compression*/
/*sz_varset is not used in the single-snapshot data compression*/
SZ_VarSet* sz_varset = NULL;
sz_multisteps *multisteps = NULL;
sz_tsc_metadata *sz_tsc = NULL;
//only for Pastri compressor
#ifdef PASTRI
pastri_params pastri_par;
#endif
......@@ -64,10 +53,6 @@ int SZ_Init(const char *configFilePath)
return SZ_FAILED;
exe_params->SZ_SIZE_TYPE = SZ_SIZE_TYPE_DEFUALT;
if(confparams_cpr->szMode == SZ_TEMPORAL_COMPRESSION)
{
initSZ_TSC();
}
return SZ_SUCCESS;
}
......
deps/SZ/sz/src/sz_double.c
浏览文件 @ f9bf4244
......@@ -21,10 +21,7 @@
#include "sz_double.h"
#include "szd_double.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
#include "sz_stats.h"
unsigned char* SZ_skip_compress_double(double* data, size_t dataLength, size_t* outSize)
......
deps/SZ/sz/src/sz_float.c
浏览文件 @ f9bf4244
......@@ -21,22 +21,11 @@
#include "sz_float.h"
#include "szd_float.h"
#include "zlib.h"
#include "rw.h"
#include "utility.h"
#include "CacheTable.h"
#include "MultiLevelCacheTableWideInterval.h"
#include "sz_stats.h"
# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
unsigned char* SZ_skip_compress_float(float* data, size_t dataLength, size_t* outSize)
{
*outSize = dataLength*sizeof(float);
unsigned char* out = (unsigned char*)malloc(dataLength*sizeof(float));
memcpy(out, data, dataLength*sizeof(float));
return out;
}
void computeReqLength_float(double realPrecision, short rangeExpo, int* reqLength, float* medianValue)
{
......
deps/SZ/sz/src/sz_interface.F90 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
deps/SZ/sz/src/sz_stats.c 已删除 100644 → 0
浏览文件 @ 7c1ae052
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或