Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
木子87
TDengine
提交
c04da26c
T
TDengine
项目概览
木子87
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c04da26c
编写于
1月 12, 2021
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[TD-2634]
上级
84b867e2
变更
5
展开全部
隐藏空白更改
内联
并排
Showing
5 changed file
with
447 addition
and
460 deletion
+447
-460
src/query/inc/qPercentile.h
src/query/inc/qPercentile.h
+23
-15
src/query/src/qAggMain.c
src/query/src/qAggMain.c
+1
-1
src/query/src/qPercentile.c
src/query/src/qPercentile.c
+133
-425
src/query/tests/percentileTest.cpp
src/query/tests/percentileTest.cpp
+254
-0
src/util/src/tcompare.c
src/util/src/tcompare.c
+36
-19
未找到文件。
src/query/inc/qPercentile.h
浏览文件 @
c04da26c
...
...
@@ -16,32 +16,36 @@
#ifndef TDENGINE_QPERCENTILE_H
#define TDENGINE_QPERCENTILE_H
#ifdef __cplusplus
extern
"C"
{
#endif
#include "qExtbuffer.h"
#include "qResultbuf.h"
#include "qTsbuf.h"
typedef
struct
MinMaxEntry
{
union
{
double
dMinVal
;
int
32_t
i
MinVal
;
int64_t
i
64MinVal
;
double
dMinVal
;
int
64_t
i64
MinVal
;
uint64_t
u
64MinVal
;
};
union
{
double
dMaxVal
;
int32_t
iMaxVal
;
int64_t
i64MaxVal
;
int64_t
u64MaxVal
;
};
}
MinMaxEntry
;
typedef
struct
{
int32_t
size
;
int32_t
pageId
;
int32_t
size
;
int32_t
pageId
;
tFilePage
*
data
;
}
SSlotInfo
;
typedef
struct
tMemBucketSlot
{
SSlotInfo
info
;
MinMaxEntry
range
;
SSlotInfo
info
;
MinMaxEntry
range
;
}
tMemBucketSlot
;
struct
tMemBucket
;
...
...
@@ -52,16 +56,16 @@ typedef struct tMemBucket {
int16_t
type
;
int16_t
bytes
;
int32_t
total
;
int32_t
elemPerPage
;
// number of elements for each object
int32_t
maxCapacity
;
// maximum allowed number of elements that can be sort directly to get the result
int32_t
bufPageSize
;
// disk page size
MinMaxEntry
range
;
// value range
int32_t
times
;
// count that has been checked for deciding the correct data value buckets.
int32_t
elemPerPage
;
// number of elements for each object
int32_t
maxCapacity
;
// maximum allowed number of elements that can be sort directly to get the result
int32_t
bufPageSize
;
// disk page size
MinMaxEntry
range
;
// value range
int32_t
times
;
// count that has been checked for deciding the correct data value buckets.
__compar_fn_t
comparFn
;
tMemBucketSlot
*
pSlots
;
tMemBucketSlot
*
pSlots
;
SDiskbasedResultBuf
*
pBuffer
;
__perc_hash_func_t
hashFunc
;
__perc_hash_func_t
hashFunc
;
}
tMemBucket
;
tMemBucket
*
tMemBucketCreate
(
int16_t
nElemSize
,
int16_t
dataType
,
double
minval
,
double
maxval
);
...
...
@@ -73,3 +77,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
double
getPercentile
(
tMemBucket
*
pMemBucket
,
double
percent
);
#endif // TDENGINE_QPERCENTILE_H
#ifdef __cplusplus
}
#endif
\ No newline at end of file
src/query/src/qAggMain.c
浏览文件 @
c04da26c
...
...
@@ -2545,7 +2545,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
if
(
pInfo
->
numOfElems
==
0
)
{
pResInfo
->
complete
=
true
;
}
else
{
pInfo
->
pMemBucket
=
tMemBucketCreate
(
pCtx
->
inputBytes
,
pCtx
->
inputType
,
GET_DOUBLE_VAL
(
&
pInfo
->
minval
),
GET_DOUBLE_VAL
(
&
pInfo
->
maxval
)
);
pInfo
->
pMemBucket
=
tMemBucketCreate
(
pCtx
->
inputBytes
,
pCtx
->
inputType
,
pInfo
->
minval
,
pInfo
->
maxval
);
}
pInfo
->
stage
+=
1
;
...
...
src/query/src/qPercentile.c
浏览文件 @
c04da26c
此差异已折叠。
点击以展开。
src/query/tests/percentileTest.cpp
0 → 100644
浏览文件 @
c04da26c
#include <gtest/gtest.h>
#include <iostream>
#include "qResultbuf.h"
#include "taos.h"
#include "taosdef.h"
#include "qPercentile.h"
namespace
{
tMemBucket
*
createBigIntDataBucket
(
int32_t
start
,
int32_t
end
)
{
tMemBucket
*
pBucket
=
tMemBucketCreate
(
sizeof
(
int64_t
),
TSDB_DATA_TYPE_BIGINT
,
start
,
end
);
for
(
int32_t
i
=
start
;
i
<=
end
;
++
i
)
{
int64_t
val
=
i
;
tMemBucketPut
(
pBucket
,
&
val
,
1
);
}
return
pBucket
;
}
tMemBucket
*
createIntDataBucket
(
int32_t
start
,
int32_t
end
)
{
tMemBucket
*
pBucket
=
tMemBucketCreate
(
sizeof
(
int32_t
),
TSDB_DATA_TYPE_INT
,
start
,
end
);
for
(
int32_t
i
=
start
;
i
<=
end
;
++
i
)
{
int32_t
val
=
i
;
tMemBucketPut
(
pBucket
,
&
val
,
1
);
}
return
pBucket
;
}
tMemBucket
*
createDoubleDataBucket
(
int32_t
start
,
int32_t
end
)
{
tMemBucket
*
pBucket
=
tMemBucketCreate
(
sizeof
(
double
),
TSDB_DATA_TYPE_DOUBLE
,
start
,
end
);
for
(
int32_t
i
=
start
;
i
<=
end
;
++
i
)
{
double
val
=
i
;
int32_t
ret
=
tMemBucketPut
(
pBucket
,
&
val
,
1
);
if
(
ret
!=
0
)
{
printf
(
"value out of range:%f"
,
val
);
}
}
return
pBucket
;
}
tMemBucket
*
createUnsignedDataBucket
(
int32_t
start
,
int32_t
end
,
int32_t
type
)
{
tMemBucket
*
pBucket
=
tMemBucketCreate
(
tDataTypeDesc
[
type
].
nSize
,
type
,
start
,
end
);
for
(
int32_t
i
=
start
;
i
<=
end
;
++
i
)
{
uint64_t
k
=
i
;
int32_t
ret
=
tMemBucketPut
(
pBucket
,
&
k
,
1
);
if
(
ret
!=
0
)
{
printf
(
"value out of range:%f"
,
k
);
}
}
return
pBucket
;
}
void
intDataTest
()
{
printf
(
"running %s
\n
"
,
__FUNCTION__
);
tMemBucket
*
pBucket
=
NULL
;
double
result
=
0.
;
pBucket
=
createIntDataBucket
(
0
,
0
);
result
=
getPercentile
(
pBucket
,
0
);
ASSERT_DOUBLE_EQ
(
result
,
0
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createIntDataBucket
(
0
,
1
);
result
=
getPercentile
(
pBucket
,
100
);
ASSERT_DOUBLE_EQ
(
result
,
1
);
result
=
getPercentile
(
pBucket
,
0
);
ASSERT_DOUBLE_EQ
(
result
,
0
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createIntDataBucket
(
-
1
,
1
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
0
);
result
=
getPercentile
(
pBucket
,
0
);
ASSERT_DOUBLE_EQ
(
result
,
-
1
);
result
=
getPercentile
(
pBucket
,
75
);
ASSERT_DOUBLE_EQ
(
result
,
0.5
);
result
=
getPercentile
(
pBucket
,
100
);
ASSERT_DOUBLE_EQ
(
result
,
1
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createIntDataBucket
(
0
,
99999
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
49999.5
);
tMemBucketDestroy
(
pBucket
);
}
void
bigintDataTest
()
{
printf
(
"running %s
\n
"
,
__FUNCTION__
);
tMemBucket
*
pBucket
=
NULL
;
double
result
=
0.0
;
pBucket
=
createBigIntDataBucket
(
-
1000
,
1000
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
0.
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createBigIntDataBucket
(
-
10000
,
10000
);
result
=
getPercentile
(
pBucket
,
100
);
ASSERT_DOUBLE_EQ
(
result
,
10000.0
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createBigIntDataBucket
(
-
10000
,
10000
);
result
=
getPercentile
(
pBucket
,
75
);
ASSERT_DOUBLE_EQ
(
result
,
5000.0
);
tMemBucketDestroy
(
pBucket
);
}
void
doubleDataTest
()
{
printf
(
"running %s
\n
"
,
__FUNCTION__
);
tMemBucket
*
pBucket
=
NULL
;
double
result
=
0
;
pBucket
=
createDoubleDataBucket
(
-
10
,
10
);
result
=
getPercentile
(
pBucket
,
0
);
ASSERT_DOUBLE_EQ
(
result
,
-
10.0
);
printf
(
"result is: %lf
\n
"
,
result
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createDoubleDataBucket
(
-
100000
,
100000
);
result
=
getPercentile
(
pBucket
,
25
);
ASSERT_DOUBLE_EQ
(
result
,
-
50000
);
printf
(
"result is: %lf
\n
"
,
result
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createDoubleDataBucket
(
-
100000
,
100000
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
0
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createDoubleDataBucket
(
-
100000
,
100000
);
result
=
getPercentile
(
pBucket
,
75
);
ASSERT_DOUBLE_EQ
(
result
,
50000
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createDoubleDataBucket
(
-
100000
,
100000
);
result
=
getPercentile
(
pBucket
,
100
);
ASSERT_DOUBLE_EQ
(
result
,
100000.0
);
printf
(
"result is: %lf
\n
"
,
result
);
tMemBucketDestroy
(
pBucket
);
}
/*
* large data test, we employ 0.1billion double data to calculated the percentile
* which is 800MB data
*/
void
largeDataTest
()
{
printf
(
"running : %s
\n
"
,
__FUNCTION__
);
tMemBucket
*
pBucket
=
NULL
;
double
result
=
0
;
struct
timeval
tv
;
gettimeofday
(
&
tv
,
NULL
);
int64_t
start
=
tv
.
tv_sec
;
printf
(
"start time: %"
PRId64
"
\n
"
,
tv
.
tv_sec
);
pBucket
=
createDoubleDataBucket
(
0
,
100000000
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
50000000
);
gettimeofday
(
&
tv
,
NULL
);
printf
(
"total elapsed time: %"
PRId64
" sec."
,
-
start
+
tv
.
tv_sec
);
printf
(
"the result of %d is: %lf
\n
"
,
50
,
result
);
tMemBucketDestroy
(
pBucket
);
}
void
qsortTest
()
{
printf
(
"running : %s
\n
"
,
__FUNCTION__
);
SSchema
field
[
1
]
=
{
{
TSDB_DATA_TYPE_INT
,
"k"
,
sizeof
(
int32_t
)},
};
const
int32_t
num
=
2000
;
int32_t
*
d
=
(
int32_t
*
)
malloc
(
sizeof
(
int32_t
)
*
num
);
for
(
int32_t
i
=
0
;
i
<
num
;
++
i
)
{
d
[
i
]
=
i
%
4
;
}
const
int32_t
numOfOrderCols
=
1
;
int32_t
orderColIdx
=
0
;
SColumnModel
*
pModel
=
createColumnModel
(
field
,
1
,
1000
);
tOrderDescriptor
*
pDesc
=
tOrderDesCreate
(
&
orderColIdx
,
numOfOrderCols
,
pModel
,
1
);
tColDataQSort
(
pDesc
,
num
,
0
,
num
-
1
,
(
char
*
)
d
,
1
);
for
(
int32_t
i
=
0
;
i
<
num
;
++
i
)
{
printf
(
"%d
\t
"
,
d
[
i
]);
}
printf
(
"
\n
"
);
destroyColumnModel
(
pModel
);
}
void
unsignedDataTest
()
{
printf
(
"running %s
\n
"
,
__FUNCTION__
);
tMemBucket
*
pBucket
=
NULL
;
double
result
=
0.0
;
pBucket
=
createUnsignedDataBucket
(
0
,
1000
,
TSDB_DATA_TYPE_UINT
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
500.0
);
tMemBucketDestroy
(
pBucket
);
pBucket
=
createUnsignedDataBucket
(
0
,
10000
,
TSDB_DATA_TYPE_UBIGINT
);
result
=
getPercentile
(
pBucket
,
100
);
ASSERT_DOUBLE_EQ
(
result
,
10000.0
);
result
=
getPercentile
(
pBucket
,
0
);
ASSERT_DOUBLE_EQ
(
result
,
0.0
);
result
=
getPercentile
(
pBucket
,
50
);
ASSERT_DOUBLE_EQ
(
result
,
5000
);
result
=
getPercentile
(
pBucket
,
75
);
ASSERT_DOUBLE_EQ
(
result
,
7500
);
tMemBucketDestroy
(
pBucket
);
}
}
// namespace
TEST
(
testCase
,
percentileTest
)
{
// qsortTest();
intDataTest
();
bigintDataTest
();
doubleDataTest
();
unsignedDataTest
();
largeDataTest
();
}
src/util/src/tcompare.c
浏览文件 @
c04da26c
...
...
@@ -30,24 +30,32 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) {
return
0
;
}
int32_t
compareIntDoubleVal
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
int64_t
lhs
=
GET_INT64_VAL
(
pLeft
);
double
rhs
=
GET_DOUBLE_VAL
(
pRight
);
if
(
fabs
(
lhs
-
rhs
)
<
FLT_EPSILON
)
{
return
0
;
}
else
{
return
(
lhs
>
rhs
)
?
1
:
-
1
;
}
int32_t
compareUint32Val
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
int32_t
left
=
GET_UINT32_VAL
(
pLeft
),
right
=
GET_UINT32_VAL
(
pRight
);
if
(
left
>
right
)
return
1
;
if
(
left
<
right
)
return
-
1
;
return
0
;
}
int32_t
compareDoubleIntVal
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
double
lhs
=
GET_DOUBLE_VAL
(
pLeft
);
int64_t
rhs
=
GET_INT64_VAL
(
pRight
);
if
(
fabs
(
lhs
-
rhs
)
<
FLT_EPSILON
)
{
return
0
;
}
else
{
return
(
lhs
>
rhs
)
?
1
:
-
1
;
}
int32_t
compareUint64Val
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
int64_t
left
=
GET_UINT64_VAL
(
pLeft
),
right
=
GET_UINT64_VAL
(
pRight
);
if
(
left
>
right
)
return
1
;
if
(
left
<
right
)
return
-
1
;
return
0
;
}
int32_t
compareUint16Val
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
int16_t
left
=
GET_UINT16_VAL
(
pLeft
),
right
=
GET_UINT16_VAL
(
pRight
);
if
(
left
>
right
)
return
1
;
if
(
left
<
right
)
return
-
1
;
return
0
;
}
int32_t
compareUint8Val
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
uint8_t
left
=
GET_UINT8_VAL
(
pLeft
),
right
=
GET_UINT8_VAL
(
pRight
);
if
(
left
>
right
)
return
1
;
if
(
left
<
right
)
return
-
1
;
return
0
;
}
int32_t
compareFloatVal
(
const
void
*
pLeft
,
const
void
*
pRight
)
{
...
...
@@ -369,15 +377,24 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
case
TSDB_DATA_TYPE_DOUBLE
:
comparFn
=
compareDoubleVal
;
break
;
case
TSDB_DATA_TYPE_UTINYINT
:
comparFn
=
compareUint8Val
;
break
;
case
TSDB_DATA_TYPE_USMALLINT
:
comparFn
=
compareUint16Val
;
break
;
case
TSDB_DATA_TYPE_UINT
:
comparFn
=
compareUint32Val
;
break
;
case
TSDB_DATA_TYPE_UBIGINT
:
comparFn
=
compareUint64Val
;
break
;
case
TSDB_DATA_TYPE_BINARY
:
comparFn
=
compareLenPrefixedStr
;
break
;
case
TSDB_DATA_TYPE_NCHAR
:
comparFn
=
compareLenPrefixedWStr
;
break
;
default:
comparFn
=
compareInt32Val
;
break
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录