Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
038e9558
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
038e9558
编写于
5月 13, 2022
作者:
H
Hongze Cheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
more data
上级
71c86808
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
141 addition
and
92 deletion
+141
-92
include/common/tdataformat.h
include/common/tdataformat.h
+5
-4
source/common/src/tdataformat.c
source/common/src/tdataformat.c
+136
-88
未找到文件。
include/common/tdataformat.h
浏览文件 @
038e9558
...
...
@@ -86,16 +86,17 @@ struct STSRow2 {
};
struct
STSRowBuilder
{
STColumn
*
pTColumn
;
STSchema
*
pTSchema
;
int32_t
szBitMap1
;
int32_t
szBitMap2
;
int32_t
szKVBuf
;
uint8_t
*
pKVBuf
;
int32_t
szTPBuf
;
uint8_t
*
pTPBuf
;
uint8_t
*
pBitBuf
;
int32_t
iCol
;
int32_t
nCols
;
int32_t
kvVLen
;
int32_t
tpVLen
;
int32_t
vlenKV
;
int32_t
vlenTP
;
STSRow2
row
;
};
...
...
source/common/src/tdataformat.c
浏览文件 @
038e9558
...
...
@@ -25,6 +25,8 @@ struct SKVIdx {
};
#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW)
#define SET_BIT1(p, i, v)
#define SET_BIT2(p, i, v)
// STSRow2
int32_t
tEncodeTSRow
(
SEncoder
*
pEncoder
,
const
STSRow2
*
pRow
)
{
...
...
@@ -170,126 +172,135 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
return
0
;
}
void
tTSchemaDestroy
(
STSchema
*
pTSchema
)
{
taosMemoryFree
(
pTSchema
);
}
void
tTSchemaDestroy
(
STSchema
*
pTSchema
)
{
if
(
pTSchema
)
taosMemoryFree
(
pTSchema
);
}
// STSRowBuilder
int32_t
tTSRowBuilderInit
(
STSRowBuilder
*
pBuilder
,
int32_t
sver
,
SSchema
*
pSchema
,
int32_t
nCols
)
{
int32_t
kvBufLen
;
int32_t
tpBufLen
;
uint8_t
*
p
;
static
int32_t
tTSRowBitMapLen1
(
int32_t
nCols
)
{
return
nCols
/
8
+
((
nCols
%
8
)
==
0
?
0
:
1
);
}
static
int32_t
tTSRowBitMapLen2
(
int32_t
nCols
)
{
return
nCols
/
4
+
((
nCols
%
4
)
==
0
?
0
:
1
);
}
int32_t
tTSRowBuilderInit
(
STSRowBuilder
*
pBuilder
,
int32_t
sver
,
SSchema
*
pSchema
,
int32_t
nCols
)
{
if
(
tTSchemaCreate
(
sver
,
pSchema
,
nCols
,
&
pBuilder
->
pTSchema
)
<
0
)
return
-
1
;
kvBufLen
=
sizeof
(
SKVIdx
)
*
nCols
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
pTSchema
->
vlen
;
tpBufLen
=
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
pTSchema
->
vlen
;
if
(
pBuilder
->
szKVBuf
<
kvBufLen
)
{
p
=
taosMemoryRealloc
(
pBuilder
->
pKVBuf
,
kvBufLen
);
if
(
p
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
}
pBuilder
->
pKVBuf
=
p
;
pBuilder
->
szKVBuf
=
kvBufLen
;
pBuilder
->
szBitMap1
=
tTSRowBitMapLen1
(
nCols
-
1
);
pBuilder
->
szBitMap2
=
tTSRowBitMapLen2
(
nCols
-
1
);
pBuilder
->
szKVBuf
=
sizeof
(
SKVIdx
)
*
(
nCols
-
1
)
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
pTSchema
->
vlen
;
pBuilder
->
szTPBuf
=
pBuilder
->
szBitMap2
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
pTSchema
->
vlen
;
pBuilder
->
pKVBuf
=
taosMemoryMalloc
(
pBuilder
->
szKVBuf
);
if
(
pBuilder
->
pKVBuf
==
NULL
)
{
terrno
=
TSDB_CODE_TSC_OUT_OF_MEMORY
;
return
-
1
;
}
if
(
pBuilder
->
szTPBuf
<
tpBufLen
)
{
p
=
taosMemoryRealloc
(
pBuilder
->
pTPBuf
,
tpBufLen
);
if
(
p
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
}
pBuilder
->
pTPBuf
=
p
;
pBuilder
->
szTPBuf
=
tpBufLen
;
pBuilder
->
pTPBuf
=
taosMemoryMalloc
(
pBuilder
->
szTPBuf
);
if
(
pBuilder
->
pTPBuf
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
}
tTSRowBuilderReset
(
pBuilder
);
return
0
;
}
void
tTSRowBuilderClear
(
STSRowBuilder
*
pBuilder
)
{
taosMemoryFree
(
pBuilder
->
pKVBuf
);
taosMemoryFree
(
pBuilder
->
pTPBuf
);
tTSchemaDestroy
(
pBuilder
->
pTSchema
);
pBuilder
->
pTSchema
=
NULL
;
if
(
pBuilder
->
pKVBuf
)
{
taosMemoryFree
(
pBuilder
->
pKVBuf
);
pBuilder
->
pKVBuf
=
NULL
;
}
if
(
pBuilder
->
pTPBuf
)
{
taosMemoryFree
(
pBuilder
->
pTPBuf
);
pBuilder
->
pTPBuf
=
NULL
;
}
}
void
tTSRowBuilderReset
(
STSRowBuilder
*
pBuilder
)
{
for
(
int32_t
iCol
=
pBuilder
->
pTSchema
->
numOfCols
-
1
;
iCol
>=
0
;
iCol
--
)
{
pBuilder
->
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
pBuilder
->
pTColumn
->
flags
&=
0xf
;
STColumn
*
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
COL_CLR_SET
(
pTColumn
->
flags
);
}
pBuilder
->
iCol
=
0
;
pBuilder
->
nCols
=
0
;
pBuilder
->
kvVLen
=
0
;
pBuilder
->
tpVLen
=
0
;
pBuilder
->
vlenKV
=
0
;
pBuilder
->
vlenTP
=
0
;
pBuilder
->
row
.
flags
=
0
;
}
int32_t
tTSRowBuilderPut
(
STSRowBuilder
*
pBuilder
,
int32_t
cid
,
const
uint8_t
*
pData
,
uint32_t
nData
)
{
// search column (TODO: bsearch with interp)
if
(
pBuilder
->
pTColumn
->
colId
<
cid
)
{
int32_t
iCol
=
(
pBuilder
->
pTColumn
-
pBuilder
->
pTSchema
->
columns
)
/
sizeof
(
STColumn
)
+
1
;
for
(;
iCol
<
pBuilder
->
pTSchema
->
numOfCols
;
iCol
++
)
{
pBuilder
->
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
pBuilder
->
pTColumn
->
colId
==
cid
)
break
;
STColumn
*
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
pBuilder
->
iCol
];
uint8_t
*
p
;
int32_t
iCol
;
// use interp search (todo)
if
(
pTColumn
->
colId
>
cid
)
{
for
(
iCol
=
pBuilder
->
iCol
+
1
;
iCol
<
pBuilder
->
pTSchema
->
numOfCols
;
iCol
++
)
{
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
pTColumn
->
colId
==
cid
)
break
;
}
}
else
if
(
pBuilder
->
pTColumn
->
colId
>
cid
)
{
int32_t
iCol
=
(
pBuilder
->
pTColumn
-
pBuilder
->
pTSchema
->
columns
)
/
sizeof
(
STColumn
)
-
1
;
for
(;
iCol
>=
0
;
iCol
--
)
{
pBuilder
->
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
pBuilder
->
pTColumn
->
colId
==
cid
)
break
;
}
else
if
(
pTColumn
->
colId
<
cid
)
{
for
(
iCol
=
pBuilder
->
iCol
-
1
;
iCol
>=
0
;
iCol
--
)
{
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
pTColumn
->
colId
==
cid
)
break
;
}
}
// check
if
(
pBuilder
->
pTColumn
->
colId
!=
cid
||
COL_IS_SET
(
pBuilder
->
pTColumn
->
flags
))
{
if
(
pTColumn
->
colId
!=
cid
||
COL_IS_SET
(
pTColumn
->
flags
))
{
return
-
1
;
}
pBuilder
->
iCol
=
iCol
;
// set value
uint8_t
*
p
;
if
(
cid
==
0
)
{
ASSERT
(
pData
&&
nData
==
sizeof
(
TSKEY
));
ASSERT
(
pData
&&
nData
==
sizeof
(
TSKEY
)
&&
iCol
==
0
);
pBuilder
->
row
.
ts
=
*
(
TSKEY
*
)
pData
;
pBuilder
->
pTColumn
->
flags
|=
COL_SET_VAL
;
pTColumn
->
flags
|=
COL_SET_VAL
;
}
else
{
if
(
pData
)
{
// set val
pBuilder
->
row
.
flags
|=
TSROW_HAS_VAL
;
pBuilder
->
pTColumn
->
flags
|=
COL_SET_VAL
;
// set tuple data
p
=
pBuilder
->
pTPBuf
+
pBuilder
->
pTColumn
->
offset
;
if
(
IS_VAR_DATA_TYPE
(
pBuilder
->
pTColumn
->
type
))
{
*
(
int32_t
*
)
p
=
pBuilder
->
tpVLen
;
if
(
pData
)
{
// set VAL
// encode the variant-length data
p
=
pBuilder
->
pTPBuf
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
tpVLen
;
pBuilder
->
tpVLen
+=
tPutBinary
(
p
,
pData
,
nData
);
}
else
{
memcpy
(
p
,
pData
,
nData
);
pBuilder
->
row
.
flags
|=
TSROW_HAS_VAL
;
pTColumn
->
flags
|=
COL_SET_VAL
;
/* KV */
if
(
1
)
{
// avoid KV at some threshold (todo)
p
=
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
;
((
SKVIdx
*
)
p
)
->
cid
=
cid
;
((
SKVIdx
*
)
p
)
->
offset
=
pBuilder
->
vlenKV
;
p
=
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
(
pBuilder
->
pTSchema
->
numOfCols
-
1
)
+
pBuilder
->
vlenKV
;
if
(
IS_VAR_DATA_TYPE
(
pTColumn
->
type
))
{
ASSERT
(
nData
<=
pTColumn
->
bytes
);
pBuilder
->
vlenKV
+=
tPutBinary
(
p
,
pData
,
nData
);
}
else
{
ASSERT
(
nData
==
pTColumn
->
bytes
);
memcpy
(
p
,
pData
,
nData
);
pBuilder
->
vlenKV
+=
nData
;
}
}
// set kv data
p
=
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
;
((
SKVIdx
*
)
p
)
->
cid
=
cid
;
((
SKVIdx
*
)
p
)
->
offset
=
pBuilder
->
kvVLen
;
/* TUPLE */
p
=
pBuilder
->
pTPBuf
+
pBuilder
->
szBitMap2
+
pTColumn
->
offset
;
if
(
IS_VAR_DATA_TYPE
(
pTColumn
->
type
))
{
ASSERT
(
nData
<=
pTColumn
->
bytes
);
*
(
int32_t
*
)
p
=
pBuilder
->
vlenTP
;
p
=
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
(
pBuilder
->
pTSchema
->
numOfCols
-
1
)
+
pBuilder
->
kvVLen
;
if
(
IS_VAR_DATA_TYPE
(
pBuilder
->
pTColumn
->
type
))
{
pBuilder
->
kvVLen
+=
tPutBinary
(
p
,
pData
,
nData
);
p
=
pBuilder
->
pTPBuf
+
pBuilder
->
szBitMap2
+
pBuilder
->
pTSchema
->
flen
;
pBuilder
->
vlenTP
+=
tPutBinary
(
p
,
pData
,
nData
);
}
else
{
ASSERT
(
nData
==
pTColumn
->
bytes
);
memcpy
(
p
,
pData
,
nData
);
pBuilder
->
kvVLen
+=
nData
;
}
}
else
{
// set NULL
}
else
{
// set NULL
pBuilder
->
row
.
flags
|=
TSROW_HAS_NULL
;
p
Builder
->
p
TColumn
->
flags
|=
COL_SET_NULL
;
pTColumn
->
flags
|=
COL_SET_NULL
;
p
=
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
;
((
SKVIdx
*
)
p
)
->
cid
=
cid
;
((
SKVIdx
*
)
p
)
->
offset
=
-
1
;
// for TSROW_KV_ROW, offset -1 means NULL
((
SKVIdx
*
)
p
)
->
offset
=
-
1
;
}
pBuilder
->
nCols
++
;
...
...
@@ -309,7 +320,7 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) {
return
0
;
}
int32_t
tTSRowBuilderGetRow
(
STSRowBuilder
*
pBuilder
,
const
STSRow2
**
ppRow
)
{
int32_t
tpDataLen
,
kvDataLen
;
int32_t
nDataTP
,
nDataKV
;
uint32_t
flags
;
// error not set ts
...
...
@@ -332,39 +343,76 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
pBuilder
->
row
.
pData
=
NULL
;
return
0
;
case
TSROW_HAS_NULL
|
TSROW_HAS_NONE
:
tpDataLen
=
(
pBuilder
->
pTSchema
->
numOfCols
-
1
)
/
8
;
nDataTP
=
pBuilder
->
szBitMap1
;
break
;
case
TSROW_HAS_VAL
:
tpDataLen
=
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
tpVLen
;
nDataTP
=
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
vlenTP
;
break
;
case
TSROW_HAS_VAL
|
TSROW_HAS_NONE
:
case
TSROW_HAS_VAL
|
TSROW_HAS_NULL
:
tpDataLen
=
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
tpVLen
+
(
pBuilder
->
pTSchema
->
numOfCols
-
1
)
/
8
;
nDataTP
=
pBuilder
->
szBitMap1
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
vlenTP
;
break
;
case
TSROW_HAS_VAL
|
TSROW_HAS_NULL
|
TSROW_HAS_NONE
:
tpDataLen
=
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
tpVLen
+
(
pBuilder
->
pTSchema
->
numOfCols
-
1
)
/
4
;
nDataTP
=
pBuilder
->
szBitMap2
+
pBuilder
->
pTSchema
->
flen
+
pBuilder
->
vlenTP
;
break
;
default:
// decide chose tuple or kv
kvDataLen
=
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
+
pBuilder
->
kvVLen
;
break
;
ASSERT
(
0
);
}
if
(
kvDataLen
<
tpDataLen
)
{
nDataKV
=
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
+
pBuilder
->
vlenKV
;
ASSERT
(
pBuilder
->
row
.
flags
&
TSROW_KV_ROW
==
0
);
if
(
nDataKV
<
nDataTP
)
{
// generate KV row
pBuilder
->
row
.
flags
|=
TSROW_KV_ROW
;
pBuilder
->
row
.
ncols
=
pBuilder
->
nCols
;
pBuilder
->
row
.
nData
=
kvDataLen
;
pBuilder
->
row
.
nData
=
nDataKV
;
pBuilder
->
row
.
pData
=
pBuilder
->
pKVBuf
;
qsort
(
pBuilder
->
pKVBuf
,
pBuilder
->
nCols
,
sizeof
(
SKVIdx
),
tSKVIdxCmprFn
);
if
(
pBuilder
->
nCols
<
pBuilder
->
pTSchema
->
numOfCols
-
1
)
{
memmove
(
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
pBuilder
->
nCols
,
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
(
pBuilder
->
pTSchema
->
numOfCols
-
1
),
pBuilder
->
kvVLen
);
pBuilder
->
pKVBuf
+
sizeof
(
SKVIdx
)
*
(
pBuilder
->
pTSchema
->
numOfCols
-
1
),
pBuilder
->
vlenKV
);
}
}
else
{
// generate TUPLE row
uint8_t
*
p
;
STColumn
*
pTColumn
;
pBuilder
->
row
.
sver
=
pBuilder
->
pTSchema
->
version
;
pBuilder
->
row
.
nData
=
nDataTP
;
if
(
pBuilder
->
row
.
flags
&
0xf
==
TSROW_HAS_VAL
)
{
// no bitmap
pBuilder
->
row
.
pData
=
pBuilder
->
pTPBuf
+
pBuilder
->
szBitMap2
;
}
else
if
(
pBuilder
->
row
.
flags
&
0xf
==
TSROW_HAS_VAL
|
TSROW_HAS_NULL
|
TSROW_HAS_NONE
)
{
// bitmap2
p
=
pBuilder
->
pTPBuf
;
for
(
int32_t
iCol
=
1
;
iCol
<
pBuilder
->
pTSchema
->
numOfCols
;
iCol
++
)
{
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
pTColumn
->
flags
&
COL_SET_VAL
)
{
SET_BIT2
(
p
,
iCol
-
1
,
0x2
);
}
else
if
(
pTColumn
->
flags
&
COL_SET_VAL
)
{
SET_BIT2
(
p
,
iCol
-
1
,
0x1
);
}
}
pBuilder
->
row
.
pData
=
p
;
}
else
{
// bitmap1
p
=
pBuilder
->
pTPBuf
+
pBuilder
->
szBitMap2
-
pBuilder
->
szBitMap1
;
pBuilder
->
row
.
nData
=
tpDataLen
;
for
(
int32_t
iCol
=
1
;
iCol
<
pBuilder
->
pTSchema
->
numOfCols
;
iCol
++
)
{
pTColumn
=
&
pBuilder
->
pTSchema
->
columns
[
iCol
];
if
(
1
)
{
SET_BIT1
(
p
,
iCol
-
1
,
0x1
);
}
}
pBuilder
->
row
.
pData
=
p
;
}
}
return
0
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录