Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
慢慢CG
TDengine
提交
9ff80775
T
TDengine
项目概览
慢慢CG
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
9ff80775
编写于
6月 29, 2021
作者:
T
tickduan
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
test is passed by self with lossy compression
上级
c0e3c29c
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
95 addition
and
76 deletion
+95
-76
src/kit/taospack/taospack.c
src/kit/taospack/taospack.c
+81
-61
src/tsdb/src/tsdbMain.c
src/tsdb/src/tsdbMain.c
+2
-4
src/util/inc/tscompression.h
src/util/inc/tscompression.h
+3
-2
src/util/src/tcompression.c
src/util/src/tcompression.c
+9
-9
未找到文件。
src/kit/taospack/taospack.c
浏览文件 @
9ff80775
...
@@ -144,7 +144,12 @@ bool testFile(const char* inFile, char algorithm){
...
@@ -144,7 +144,12 @@ bool testFile(const char* inFile, char algorithm){
printf
(
" file %s have count=%d
\n
"
,
inFile
,
cnt
);
printf
(
" file %s have count=%d
\n
"
,
inFile
,
cnt
);
cost_start
();
cost_start
();
int
ret_len
=
tsCompressFloatLossy
(
input
,
input_len
,
cnt
,
output
,
output_len
,
algorithm
,
buff
,
buff_len
);
int
ret_len
=
0
;
if
(
algorithm
==
2
)
ret_len
=
tsCompressFloat
(
input
,
input_len
,
cnt
,
output
,
output_len
,
algorithm
,
buff
,
buff_len
);
else
ret_len
=
tsCompressFloatLossy
(
input
,
input_len
,
cnt
,
output
,
output_len
,
algorithm
,
buff
,
buff_len
);
if
(
ret_len
==
-
1
)
{
if
(
ret_len
==
-
1
)
{
printf
(
" compress error.
\n
"
);
printf
(
" compress error.
\n
"
);
return
0
;
return
0
;
...
@@ -160,14 +165,21 @@ bool testFile(const char* inFile, char algorithm){
...
@@ -160,14 +165,21 @@ bool testFile(const char* inFile, char algorithm){
//
//
float
*
ft2
=
(
float
*
)
malloc
(
input_len
);
float
*
ft2
=
(
float
*
)
malloc
(
input_len
);
cost_start
();
cost_start
();
int
code
=
tsDecompressFloatLossy
(
output
,
ret_len
,
cnt
,
(
char
*
)
ft2
,
input_len
,
algorithm
,
buff
,
buff_len
);
int
code
=
0
;
if
(
algorithm
==
2
)
code
=
tsDecompressFloat
(
output
,
ret_len
,
cnt
,
(
char
*
)
ft2
,
input_len
,
algorithm
,
buff
,
buff_len
);
else
code
=
tsDecompressFloatLossy
(
output
,
ret_len
,
cnt
,
(
char
*
)
ft2
,
input_len
,
algorithm
,
buff
,
buff_len
);
double
use_ms2
=
cost_end
(
"Decompress"
);
double
use_ms2
=
cost_end
(
"Decompress"
);
printf
(
" Decompress return length=%d
\n
"
,
code
);
printf
(
" Decompress return length=%d
\n
"
,
code
);
// compare same
// compare same
float
same_rate
=
check_same
(
floats
,
ft2
,
cnt
);
float
same_rate
=
check_same
(
floats
,
ft2
,
cnt
);
printf
(
"
\n
------------------ count:%d TD <%s> ----------------
\n
"
,
cnt
,
algorithm
==
ONE_STAGE_COMP
?
"ONE"
:
"TWO
"
);
printf
(
"
\n
------------------ count:%d TD <%s> ----------------
\n
"
,
cnt
,
algorithm
==
2
?
"TD"
:
"SZ
"
);
printf
(
" Compress Rate ......... [%.0f%%]
\n
"
,
rate
);
printf
(
" Compress Rate ......... [%.0f%%]
\n
"
,
rate
);
double
speed1
=
(
cnt
*
sizeof
(
float
)
*
1000
/
1024
/
1024
)
/
use_ms1
;
double
speed1
=
(
cnt
*
sizeof
(
float
)
*
1000
/
1024
/
1024
)
/
use_ms1
;
printf
(
" Compress Time ......... [%.4fms] speed=%.1f MB/s
\n
"
,
use_ms1
,
speed1
);
printf
(
" Compress Time ......... [%.4fms] speed=%.1f MB/s
\n
"
,
use_ms1
,
speed1
);
...
@@ -184,61 +196,6 @@ bool testFile(const char* inFile, char algorithm){
...
@@ -184,61 +196,6 @@ bool testFile(const char* inFile, char algorithm){
return
true
;
return
true
;
}
}
int
memTest
();
int
memTestDouble
();
void
test_threadsafe
(
int
thread_count
);
void
test_threadsafe_double
(
int
thread_count
);
//
// main
//
int
main
(
int
argc
,
char
*
argv
[])
{
printf
(
"welcome to use taospack tools v1.1. sizeof(STColumn) = %lu
\n
"
,
sizeof
(
STColumn
));
tsCompressInit
();
//
//tsCompressExit();
//return 1;
if
(
argc
==
3
){
char
algo
=
0
;
// t
if
(
strcmp
(
argv
[
1
],
"-tone"
)
==
0
||
strcmp
(
argv
[
1
],
"-t"
)
==
0
)
{
algo
=
ONE_STAGE_COMP
;
}
if
(
strcmp
(
argv
[
1
],
"-ttwo"
)
==
0
)
{
algo
=
TWO_STAGE_COMP
;
}
if
(
strcmp
(
argv
[
1
],
"-sf"
)
==
0
)
{
test_threadsafe
(
atoi
(
argv
[
2
]));
return
0
;
}
if
(
strcmp
(
argv
[
1
],
"-sd"
)
==
0
)
{
test_threadsafe_double
(
atoi
(
argv
[
2
]));
return
0
;
}
if
(
algo
==
0
){
printf
(
" no param -tone -ttwo
\n
"
);
return
0
;
}
bool
ret
=
testFile
(
argv
[
2
],
algo
);
printf
(
" test file %s.
\n
"
,
ret
?
"ok"
:
"err"
);
return
1
;
}
else
if
(
argc
==
2
)
{
if
(
strcmp
(
argv
[
1
],
"-mem"
)
==
0
)
{
memTest
();
}
}
//memTest();
return
0
;
}
//
//
// txt to binary file
// txt to binary file
//
//
...
@@ -490,7 +447,11 @@ int memTest() {
...
@@ -490,7 +447,11 @@ int memTest() {
void
*
memTestThread
(
void
*
lparam
)
{
void
*
memTestThread
(
void
*
lparam
)
{
//memTest();
//memTest();
printf
(
" enter thread ....
\n
"
);
printf
(
" enter thread ....
\n
"
);
for
(
int
i
=
0
;
i
<
1000000
;
i
++
)
{
memTest
();
memTest
();
printf
(
" start i=%d ....
\n
"
,
i
);
}
return
NULL
;
return
NULL
;
}
}
...
@@ -519,7 +480,11 @@ void test_threadsafe(int thread_count){
...
@@ -519,7 +480,11 @@ void test_threadsafe(int thread_count){
void
*
memTestThreadDouble
(
void
*
lparam
)
{
void
*
memTestThreadDouble
(
void
*
lparam
)
{
//memTest();
//memTest();
printf
(
" enter thread ....
\n
"
);
printf
(
" enter thread ....
\n
"
);
memTestDouble
();
for
(
int
i
=
0
;
i
<
50000
;
i
++
)
{
memTest
();
printf
(
" double start i=%d ....
\n
"
,
i
);
}
return
NULL
;
return
NULL
;
}
}
...
@@ -543,3 +508,58 @@ void test_threadsafe_double(int thread_count){
...
@@ -543,3 +508,58 @@ void test_threadsafe_double(int thread_count){
printf
(
"
\n
---- double test thread safe end. not same count=%d-----
\n
"
,
notsame_cnt
);
printf
(
"
\n
---- double test thread safe end. not same count=%d-----
\n
"
,
notsame_cnt
);
}
}
//
// ----------------- main ----------------------
//
int
main
(
int
argc
,
char
*
argv
[])
{
printf
(
"welcome to use taospack tools v1.1. sizeof(STColumn) = %lu
\n
"
,
sizeof
(
STColumn
));
tsLossyInit
();
//
//tsCompressExit();
//return 1;
if
(
argc
==
3
){
char
algo
=
0
;
// t
if
(
strcmp
(
argv
[
1
],
"-tone"
)
==
0
||
strcmp
(
argv
[
1
],
"-t"
)
==
0
)
{
algo
=
ONE_STAGE_COMP
;
}
if
(
strcmp
(
argv
[
1
],
"-ttwo"
)
==
0
)
{
algo
=
TWO_STAGE_COMP
;
}
if
(
strcmp
(
argv
[
1
],
"-sf"
)
==
0
)
{
test_threadsafe
(
atoi
(
argv
[
2
]));
return
0
;
}
if
(
strcmp
(
argv
[
1
],
"-sd"
)
==
0
)
{
test_threadsafe_double
(
atoi
(
argv
[
2
]));
return
0
;
}
if
(
algo
==
0
){
printf
(
" no param -tone -ttwo
\n
"
);
return
0
;
}
bool
ret
=
testFile
(
argv
[
2
],
algo
);
printf
(
" test file %s.
\n
"
,
ret
?
"ok"
:
"err"
);
return
1
;
}
else
if
(
argc
==
2
)
{
if
(
strcmp
(
argv
[
1
],
"-mem"
)
==
0
)
{
memTest
();
}
}
//memTest();
return
0
;
}
src/tsdb/src/tsdbMain.c
浏览文件 @
9ff80775
...
@@ -68,7 +68,8 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
...
@@ -68,7 +68,8 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
terrno
=
TSDB_CODE_SUCCESS
;
terrno
=
TSDB_CODE_SUCCESS
;
// Compress Init
// Compress Init
tsCompressInit
();
//if(pCfg->compressLossy)
tsLossyInit
();
// Check and set default configurations
// Check and set default configurations
if
(
tsdbCheckAndSetDefaultCfg
(
&
config
)
<
0
)
{
if
(
tsdbCheckAndSetDefaultCfg
(
&
config
)
<
0
)
{
...
@@ -143,9 +144,6 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) {
...
@@ -143,9 +144,6 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) {
tsdbFreeRepo
(
pRepo
);
tsdbFreeRepo
(
pRepo
);
tsdbDebug
(
"vgId:%d repository is closed"
,
vgId
);
tsdbDebug
(
"vgId:%d repository is closed"
,
vgId
);
// compress exit
tsCompressExit
();
if
(
terrno
!=
TSDB_CODE_SUCCESS
)
{
if
(
terrno
!=
TSDB_CODE_SUCCESS
)
{
return
-
1
;
return
-
1
;
}
else
{
}
else
{
...
...
src/util/inc/tscompression.h
浏览文件 @
9ff80775
...
@@ -54,8 +54,9 @@ int tsCompressDoubleLossyImp(const char * input, const int nelements, const char
...
@@ -54,8 +54,9 @@ int tsCompressDoubleLossyImp(const char * input, const int nelements, const char
int
tsDecompressDoubleLossyImp
(
const
char
*
input
,
int
compressedSize
,
const
int
nelements
,
const
char
*
output
);
int
tsDecompressDoubleLossyImp
(
const
char
*
input
,
int
compressedSize
,
const
int
nelements
,
const
char
*
output
);
// init
// init
bool
tsCompressInit
();
bool
tsLossyInit
();
void
tsCompressExit
();
static
FORCE_INLINE
int
tsCompressTinyint
(
const
char
*
const
input
,
int
inputSize
,
const
int
nelements
,
char
*
const
output
,
int
outputSize
,
char
algorithm
,
static
FORCE_INLINE
int
tsCompressTinyint
(
const
char
*
const
input
,
int
inputSize
,
const
int
nelements
,
char
*
const
output
,
int
outputSize
,
char
algorithm
,
char
*
const
buffer
,
int
bufferSize
)
{
char
*
const
buffer
,
int
bufferSize
)
{
...
...
src/util/src/tcompression.c
浏览文件 @
9ff80775
...
@@ -891,18 +891,18 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con
...
@@ -891,18 +891,18 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con
//
//
// ----------- global init and exit resource ------
// ----------- global init and exit resource ------
//
//
int
SZ_Init
(
const
char
*
configFilePath
);
int
SZ_Init
(
const
char
*
configFilePath
);
//declare deps/sz/include/sz.h
bool
gLossyInited
=
false
;
bool
tsCompressInit
()
{
bool
tsLossyInit
()
{
// init compress init
if
(
!
gLossyInited
){
gLossyInited
=
true
;
SZ_Init
(
"./sz.config"
);
SZ_Init
(
"./sz.config"
);
}
return
true
;
return
true
;
}
}
void
tsCompressExit
(){
}
//
//
// ---------- float double lossy -----------
// ---------- float double lossy -----------
//
//
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录