Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
慢慢CG
TDengine
提交
9a9ea692
T
TDengine
项目概览
慢慢CG
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
9a9ea692
编写于
7月 22, 2020
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[td-225] update resbuf
上级
f72183fe
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
273 addition
and
118 deletion
+273
-118
src/query/inc/qResultbuf.h
src/query/inc/qResultbuf.h
+29
-18
src/query/src/qResultbuf.c
src/query/src/qResultbuf.c
+218
-97
src/query/tests/resultBufferTest.cpp
src/query/tests/resultBufferTest.cpp
+25
-3
src/util/src/tlist.c
src/util/src/tlist.c
+1
-0
未找到文件。
src/query/inc/qResultbuf.h
浏览文件 @
9a9ea692
...
@@ -20,42 +20,56 @@
...
@@ -20,42 +20,56 @@
extern
"C"
{
extern
"C"
{
#endif
#endif
#include <tlist.h>
#include "hash.h"
#include "hash.h"
#include "os.h"
#include "os.h"
#include "qExtbuffer.h"
#include "qExtbuffer.h"
#include "tlockfree.h"
typedef
struct
SArray
*
SIDList
;
typedef
struct
SArray
*
SIDList
;
typedef
struct
SPageInfo
{
typedef
struct
SPageDiskInfo
{
int32_t
pageId
;
int32_t
offset
;
int32_t
offset
;
int32_t
lengthOnDisk
;
int32_t
length
;
}
SPageDiskInfo
;
typedef
struct
SPageInfo
{
int32_t
pageId
;
SPageDiskInfo
info
;
void
*
pData
;
T_REF_DECLARE
();
}
SPageInfo
;
}
SPageInfo
;
typedef
struct
SFreeListItem
{
int32_t
offset
;
int32_t
len
;
}
SFreeListItem
;
typedef
struct
SDiskbasedResultBuf
{
typedef
struct
SDiskbasedResultBuf
{
int32_t
numOfRowsPerPage
;
int32_t
numOfRowsPerPage
;
int32_t
numOfPages
;
int32_t
numOfPages
;
int64_t
totalBufSize
;
int64_t
totalBufSize
;
int32_t
fd
;
//
int32_t fd;
//
FILE* file;
FILE
*
file
;
int32_t
allocateId
;
// allocated page id
int32_t
allocateId
;
// allocated page id
int32_t
incStep
;
// minimum allocated pages
//
int32_t incStep; // minimum allocated pages
void
*
pBuf
;
// mmap buffer pointer
void
*
pBuf
;
// mmap buffer pointer
char
*
path
;
// file path
char
*
path
;
// file path
int32_t
pageSize
;
// current used page size
int32_t
pageSize
;
// current used page size
int32_t
inMemPages
;
// numOfPages that are allocated in memory
int32_t
inMemPages
;
// numOfPages that are allocated in memory
SHashObj
*
idsTable
;
// id hash table
SHashObj
*
idsTable
;
// id hash table
SIDList
list
;
// for each id, there is a page id list
SHashObj
*
all
;
SList
*
pPageList
;
void
*
iBuf
;
// inmemory buf
void
*
handle
;
// for debug purpose
void
*
handle
;
// for debug purpose
void
*
emptyDummyIdList
;
// dummy id list
void
*
emptyDummyIdList
;
// dummy id list
bool
comp
;
bool
comp
;
SArray
*
pFree
;
// free area in file
int32_t
nextPos
;
// next page flush position
}
SDiskbasedResultBuf
;
}
SDiskbasedResultBuf
;
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
#define DEFAULT_INMEM_BUF_PAGES 10
#define DEFAULT_INMEM_BUF_PAGES 10
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
/**
/**
* create disk-based result buffer
* create disk-based result buffer
...
@@ -65,7 +79,7 @@ typedef struct SDiskbasedResultBuf {
...
@@ -65,7 +79,7 @@ typedef struct SDiskbasedResultBuf {
* @return
* @return
*/
*/
int32_t
createDiskbasedResultBuffer
(
SDiskbasedResultBuf
**
pResultBuf
,
int32_t
numOfPages
,
int32_t
rowSize
,
int32_t
pagesize
,
int32_t
createDiskbasedResultBuffer
(
SDiskbasedResultBuf
**
pResultBuf
,
int32_t
numOfPages
,
int32_t
rowSize
,
int32_t
pagesize
,
int32_t
inMemPages
,
const
void
*
handle
);
int32_t
inMemPages
,
const
void
*
handle
);
/**
/**
*
*
...
@@ -97,13 +111,10 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
...
@@ -97,13 +111,10 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
* @param id
* @param id
* @return
* @return
*/
*/
static
FORCE_INLINE
tFilePage
*
getResBufPage
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
id
)
{
tFilePage
*
getResBufPage
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
id
);
if
(
id
<
pResultBuf
->
inMemPages
)
{
return
(
tFilePage
*
)
((
char
*
)
pResultBuf
->
iBuf
+
id
*
pResultBuf
->
pageSize
);
void
releaseResBufPage
(
SDiskbasedResultBuf
*
pResultBuf
,
void
*
page
);
}
else
{
return
(
tFilePage
*
)
((
char
*
)
pResultBuf
->
pBuf
+
(
id
-
pResultBuf
->
inMemPages
)
*
pResultBuf
->
pageSize
);
}
}
/**
/**
* get the total buffer size in the format of disk file
* get the total buffer size in the format of disk file
* @param pResultBuf
* @param pResultBuf
...
...
src/query/src/qResultbuf.c
浏览文件 @
9a9ea692
#include "qResultbuf.h"
#include "qResultbuf.h"
#include <stddef.h>
#include "hash.h"
#include "hash.h"
#include "qExtbuffer.h"
#include "qExtbuffer.h"
#include "queryLog.h"
#include "queryLog.h"
...
@@ -14,29 +15,26 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t nu
...
@@ -14,29 +15,26 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t nu
}
}
pResBuf
->
pageSize
=
pagesize
;
pResBuf
->
pageSize
=
pagesize
;
pResBuf
->
numOfPages
=
inMemPages
;
// all pages are in buffer in the first place
pResBuf
->
numOfPages
=
0
;
// all pages are in buffer in the first place
pResBuf
->
inMemPages
=
inMemPages
;
pResBuf
->
inMemPages
=
inMemPages
;
assert
(
inMemPages
<=
numOfPages
);
assert
(
inMemPages
<=
numOfPages
);
pResBuf
->
numOfRowsPerPage
=
(
pagesize
-
sizeof
(
tFilePage
))
/
rowSize
;
pResBuf
->
numOfRowsPerPage
=
(
pagesize
-
sizeof
(
tFilePage
))
/
rowSize
;
pResBuf
->
totalBufSize
=
pResBuf
->
numOfPages
*
pagesize
;
pResBuf
->
totalBufSize
=
pResBuf
->
numOfPages
*
pagesize
;
pResBuf
->
incStep
=
4
;
pResBuf
->
allocateId
=
-
1
;
pResBuf
->
allocateId
=
-
1
;
// todo opt perf by on demand create in memory buffer
pResBuf
->
pPageList
=
tdListNew
(
POINTER_BYTES
);
pResBuf
->
iBuf
=
calloc
(
pResBuf
->
inMemPages
,
pResBuf
->
pageSize
);
// init id hash table
// init id hash table
pResBuf
->
idsTable
=
taosHashInit
(
numOfPages
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_INT
),
false
);
pResBuf
->
idsTable
=
taosHashInit
(
10
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_INT
),
false
);
pResBuf
->
list
=
taosArrayInit
(
numOfPages
,
POINTER_BYTES
);
pResBuf
->
all
=
taosHashInit
(
10
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_INT
),
false
);
char
path
[
PATH_MAX
]
=
{
0
};
char
path
[
PATH_MAX
]
=
{
0
};
getTmpfilePath
(
"qbuf"
,
path
);
getTmpfilePath
(
"qbuf"
,
path
);
pResBuf
->
path
=
strdup
(
path
);
pResBuf
->
path
=
strdup
(
path
);
pResBuf
->
fd
=
FD_INITIALIZER
;
pResBuf
->
file
=
NULL
;
pResBuf
->
pBuf
=
NULL
;
pResBuf
->
emptyDummyIdList
=
taosArrayInit
(
1
,
sizeof
(
int32_t
));
pResBuf
->
emptyDummyIdList
=
taosArrayInit
(
1
,
sizeof
(
int32_t
));
qDebug
(
"QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %"
PRId64
"bytes"
,
handle
,
qDebug
(
"QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %"
PRId64
"bytes"
,
handle
,
...
@@ -53,133 +51,258 @@ int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->tota
...
@@ -53,133 +51,258 @@ int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->tota
#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
static
int32_t
createDiskResidesBuf
(
SDiskbasedResultBuf
*
pResultBuf
)
{
static
int32_t
createDiskResidesBuf
(
SDiskbasedResultBuf
*
pResultBuf
)
{
pResultBuf
->
fd
=
open
(
pResultBuf
->
path
,
O_CREAT
|
O_RDWR
,
0666
);
//
pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666);
//
pResultBuf->file = fopen(pResultBuf->path, "w");
pResultBuf
->
file
=
fopen
(
pResultBuf
->
path
,
"w"
);
if
(
!
FD_VALID
(
pResultBuf
->
fd
)
)
{
if
(
pResultBuf
->
file
==
NULL
)
{
qError
(
"failed to create tmp file: %s on disk. %s"
,
pResultBuf
->
path
,
strerror
(
errno
));
qError
(
"failed to create tmp file: %s on disk. %s"
,
pResultBuf
->
path
,
strerror
(
errno
));
return
TAOS_SYSTEM_ERROR
(
errno
);
return
TAOS_SYSTEM_ERROR
(
errno
);
}
}
return
TSDB_CODE_SUCCESS
;
}
assert
(
pResultBuf
->
numOfPages
==
pResultBuf
->
inMemPages
);
static
char
*
doCompressData
(
void
*
data
,
int32_t
srcSize
,
int32_t
*
dst
)
{
// do nothing
pResultBuf
->
numOfPages
+=
pResultBuf
->
incStep
;
*
dst
=
srcSize
;
return
data
;
}
int32_t
ret
=
ftruncate
(
pResultBuf
->
fd
,
NUM_OF_PAGES_ON_DISK
(
pResultBuf
)
*
pResultBuf
->
pageSize
);
static
int32_t
allocatePositionInFile
(
SDiskbasedResultBuf
*
pResultBuf
,
size_t
size
)
{
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
if
(
pResultBuf
->
pFree
==
NULL
)
{
qError
(
"failed to create tmp file: %s on disk. %s"
,
pResultBuf
->
path
,
strerror
(
errno
))
;
return
pResultBuf
->
nextPos
;
return
TAOS_SYSTEM_ERROR
(
errno
);
}
else
{
//todo speed up the search procedure
}
size_t
num
=
taosArrayGetSize
(
pResultBuf
->
pFree
);
pResultBuf
->
pBuf
=
mmap
(
NULL
,
FILE_SIZE_ON_DISK
(
pResultBuf
),
PROT_READ
|
PROT_WRITE
,
MAP_SHARED
,
pResultBuf
->
fd
,
0
)
;
int32_t
offset
=
-
1
;
if
(
pResultBuf
->
pBuf
==
MAP_FAILED
)
{
for
(
int32_t
i
=
0
;
i
<
num
;
++
i
)
{
qError
(
"QInfo:%p failed to map temp file: %s. %s"
,
pResultBuf
->
handle
,
pResultBuf
->
path
,
strerror
(
errno
));
SFreeListItem
*
pi
=
taosArrayGet
(
pResultBuf
->
pFree
,
i
);
return
TAOS_SYSTEM_ERROR
(
errno
);
if
(
pi
->
len
>=
size
)
{
}
offset
=
pi
->
offset
;
pi
->
offset
+=
size
;
pi
->
len
-=
size
;
pResultBuf
->
totalBufSize
=
pResultBuf
->
numOfPages
*
pResultBuf
->
pageSize
;
return
offset
;
return
TSDB_CODE_SUCCESS
;
}
}
// no available recycle space, allocate new area in file
return
pResultBuf
->
nextPos
;
}
}
}
static
int32_t
extendDiskFileSize
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
incNumOfPages
)
{
static
void
doFlushPageToDisk
(
SDiskbasedResultBuf
*
pResultBuf
,
SPageInfo
*
pg
)
{
assert
(
pResultBuf
->
numOfPages
*
pResultBuf
->
pageSize
==
pResultBuf
->
totalBufSize
);
assert
(
T_REF_VAL_GET
(
pg
)
==
0
);
int32_t
ret
=
TSDB_CODE_SUCCESS
;
i
f
(
pResultBuf
->
pBuf
==
NULL
)
{
i
nt32_t
size
=
-
1
;
assert
(
!
FD_VALID
(
pResultBuf
->
fd
)
);
char
*
t
=
doCompressData
(
pg
->
pData
+
POINTER_BYTES
,
pResultBuf
->
pageSize
,
&
size
);
if
((
ret
=
createDiskResidesBuf
(
pResultBuf
))
!=
TSDB_CODE_SUCCESS
)
{
// this page is flushed to disk for the first time
return
ret
;
if
(
pg
->
info
.
offset
==
-
1
)
{
}
int32_t
offset
=
allocatePositionInFile
(
pResultBuf
,
size
);
pResultBuf
->
nextPos
+=
size
;
fseek
(
pResultBuf
->
file
,
offset
,
SEEK_SET
);
fwrite
(
t
,
size
,
1
,
pResultBuf
->
file
);
}
else
{
}
else
{
ret
=
munmap
(
pResultBuf
->
pBuf
,
FILE_SIZE_ON_DISK
(
pResultBuf
));
if
(
pg
->
info
.
length
<
size
)
{
// length becomes greater, current space is not enough, allocate new place.
pResultBuf
->
numOfPages
+=
incNumOfPages
;
//1. add current space to free list
taosArrayPush
(
pResultBuf
->
pFree
,
&
pg
->
info
);
/*
* disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
//2. allocate new position, and update the info
* be insufficient
int32_t
offset
=
allocatePositionInFile
(
pResultBuf
,
size
);
*/
pResultBuf
->
nextPos
+=
size
;
ret
=
ftruncate
(
pResultBuf
->
fd
,
NUM_OF_PAGES_ON_DISK
(
pResultBuf
)
*
pResultBuf
->
pageSize
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
//3. write to disk.
// dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
fseek
(
pResultBuf
->
file
,
offset
,
SEEK_SET
);
// strerror(errno));
fwrite
(
t
,
size
,
1
,
pResultBuf
->
file
);
return
TSDB_CODE_QRY_NO_DISKSPACE
;
}
}
}
}
pResultBuf
->
totalBufSize
=
pResultBuf
->
numOfPages
*
pResultBuf
->
pageSize
;
static
int32_t
flushPageToDisk
(
SDiskbasedResultBuf
*
pResultBuf
,
SPageInfo
*
pg
)
{
pResultBuf
->
pBuf
=
mmap
(
NULL
,
FILE_SIZE_ON_DISK
(
pResultBuf
),
PROT_READ
|
PROT_WRITE
,
MAP_SHARED
,
pResultBuf
->
fd
,
0
);
int32_t
ret
=
TSDB_CODE_SUCCESS
;
assert
(
pResultBuf
->
numOfPages
*
pResultBuf
->
pageSize
==
pResultBuf
->
totalBufSize
&&
pResultBuf
->
numOfPages
>=
pResultBuf
->
inMemPages
);
if
(
pResultBuf
->
pBuf
==
MAP_FAILED
)
{
if
(
pResultBuf
->
pBuf
==
NULL
)
{
// dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
assert
(
pResultBuf
->
file
==
NULL
);
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
if
((
ret
=
createDiskResidesBuf
(
pResultBuf
))
!=
TSDB_CODE_SUCCESS
)
{
return
ret
;
}
}
}
}
doFlushPageToDisk
(
pResultBuf
,
pg
);
return
TSDB_CODE_SUCCESS
;
return
TSDB_CODE_SUCCESS
;
}
}
#define NO_AVAILABLE_PAGES(_b) ((_b)->
allocateId == (_b)->numOfPages - 1
)
#define NO_AVAILABLE_PAGES(_b) ((_b)->
numOfPages >= (_b)->inMemPages
)
static
FORCE_INLINE
int32_t
getGroupIndex
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
static
SIDList
addNewGroup
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
assert
(
pResultBuf
!
=
NULL
);
assert
(
taosHashGet
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
))
=
=
NULL
);
char
*
p
=
taosHashGet
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
));
SArray
*
pa
=
taosArrayInit
(
1
,
sizeof
(
SPageInfo
));
int32_t
ret
=
taosHashPut
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
),
&
pa
,
POINTER_BYTES
);
assert
(
ret
==
0
);
return
pa
;
}
static
SPageInfo
*
registerPage
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
pageId
)
{
SIDList
list
=
NULL
;
char
**
p
=
taosHashGet
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
));
if
(
p
==
NULL
)
{
// it is a new group id
if
(
p
==
NULL
)
{
// it is a new group id
return
-
1
;
list
=
addNewGroup
(
pResultBuf
,
groupId
);
}
else
{
list
=
(
SIDList
)
(
*
p
);
}
}
int32_t
slot
=
GET_INT32_VAL
(
p
);
pResultBuf
->
numOfPages
+=
1
;
assert
(
slot
>=
0
&&
slot
<
taosHashGetSize
(
pResultBuf
->
idsTable
));
return
slot
;
SPageInfo
ppi
=
{
.
info
=
PAGE_INFO_INITIALIZER
,
.
pageId
=
pageId
,
};
return
taosArrayPush
(
list
,
&
ppi
);
}
}
static
int32_t
addNewGroupId
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
tFilePage
*
getNewDataBuf
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
*
pageId
)
{
int32_t
num
=
getNumOfResultBufGroupId
(
pResultBuf
);
// the num is the newest allocated group id slot
if
(
NO_AVAILABLE_PAGES
(
pResultBuf
))
{
taosHashPut
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
),
&
num
,
sizeof
(
int32_t
));
// get the last page in linked list
SListIter
iter
=
{
0
};
tdListInitIter
(
pResultBuf
->
pPageList
,
&
iter
,
TD_LIST_BACKWARD
);
SListNode
*
pn
=
NULL
;
while
((
pn
=
tdListNext
(
&
iter
))
!=
NULL
)
{
assert
(
pn
!=
NULL
);
if
(
T_REF_VAL_GET
(
*
(
SPageInfo
**
)
pn
->
data
)
==
0
)
{
break
;
}
}
// all pages are referenced by user, try to allocate new space
if
(
pn
==
NULL
)
{
int32_t
prev
=
pResultBuf
->
inMemPages
;
pResultBuf
->
inMemPages
=
pResultBuf
->
inMemPages
*
1
.
5
;
qWarn
(
"%p in memory buf page not sufficient, expand from %d to %d, page size:%d"
,
pResultBuf
,
prev
,
pResultBuf
->
inMemPages
,
pResultBuf
->
pageSize
);
}
else
{
tdListPopNode
(
pResultBuf
->
pPageList
,
pn
);
if
(
flushPageToDisk
(
pResultBuf
,
*
(
SPageInfo
**
)
pn
->
data
)
!=
TSDB_CODE_SUCCESS
)
{
return
NULL
;
}
}
}
SArray
*
pa
=
taosArrayInit
(
1
,
sizeof
(
int32_t
));
// register new id in this group
taosArrayPush
(
pResultBuf
->
list
,
&
pa
);
*
pageId
=
(
++
pResultBuf
->
allocateId
);
assert
(
taosArrayGetSize
(
pResultBuf
->
list
)
==
taosHashGetSize
(
pResultBuf
->
idsTable
));
// register page id info
return
num
;
SPageInfo
*
pi
=
registerPage
(
pResultBuf
,
groupId
,
*
pageId
);
}
static
void
registerPageId
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
pageId
)
{
// add to LRU list
int32_t
slot
=
getGroupIndex
(
pResultBuf
,
groupId
);
assert
(
listNEles
(
pResultBuf
->
pPageList
)
<
pResultBuf
->
inMemPages
);
if
(
slot
<
0
)
{
tdListPrepend
(
pResultBuf
->
pPageList
,
&
pi
);
slot
=
addNewGroupId
(
pResultBuf
,
groupId
);
}
// add to hash map
taosHashPut
(
pResultBuf
->
all
,
pageId
,
sizeof
(
int32_t
),
&
pi
,
POINTER_BYTES
);
// allocate buf
pi
->
pData
=
calloc
(
1
,
pResultBuf
->
pageSize
+
POINTER_BYTES
);
pResultBuf
->
totalBufSize
+=
pResultBuf
->
pageSize
;
T_REF_INC
(
pi
);
// add ref count
((
void
**
)
pi
->
pData
)[
0
]
=
pi
;
SIDList
pList
=
taosArrayGetP
(
pResultBuf
->
list
,
slot
);
return
pi
->
pData
+
POINTER_BYTES
;
taosArrayPush
(
pList
,
&
pageId
);
}
}
tFilePage
*
getNewDataBuf
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
*
pageId
)
{
tFilePage
*
getResBufPage
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
id
)
{
if
(
NO_AVAILABLE_PAGES
(
pResultBuf
))
{
assert
(
pResultBuf
!=
NULL
&&
id
>=
0
);
if
(
extendDiskFileSize
(
pResultBuf
,
pResultBuf
->
incStep
)
!=
TSDB_CODE_SUCCESS
)
{
SPageInfo
**
pi
=
taosHashGet
(
pResultBuf
->
all
,
&
id
,
sizeof
(
int32_t
));
assert
(
pi
!=
NULL
&&
*
pi
!=
NULL
);
if
((
*
pi
)
->
pData
!=
NULL
)
{
// it is in memory
// no need to update the LRU list
if
(
pResultBuf
->
numOfPages
==
1
)
{
return
(
*
pi
)
->
pData
+
POINTER_BYTES
;
}
SListNode
*
pnode
=
NULL
;
// todo speed up
SListIter
iter
=
{
0
};
tdListInitIter
(
pResultBuf
->
pPageList
,
&
iter
,
TD_LIST_FORWARD
);
while
((
pnode
=
tdListNext
(
&
iter
))
!=
NULL
)
{
SPageInfo
**
pInfo
=
(
SPageInfo
**
)
pnode
->
data
;
// remove it and add it into the front of linked-list
if
((
*
pInfo
)
->
pageId
==
id
)
{
tdListPopNode
(
pResultBuf
->
pPageList
,
pnode
);
tdListPrependNode
(
pResultBuf
->
pPageList
,
pnode
);
T_REF_INC
(
*
(
SPageInfo
**
)
pnode
->
data
);
return
((
*
(
SPageInfo
**
)
pnode
->
data
)
->
pData
+
POINTER_BYTES
);
}
}
}
else
{
// not in memory
// choose the be flushed page
// get the last page in linked list
SListIter
iter1
=
{
0
};
tdListInitIter
(
pResultBuf
->
pPageList
,
&
iter1
,
TD_LIST_BACKWARD
);
SListNode
*
pn
=
NULL
;
while
((
pn
=
tdListNext
(
&
iter1
))
!=
NULL
)
{
assert
(
pn
!=
NULL
);
if
(
T_REF_VAL_GET
(
*
(
SPageInfo
**
)
pn
->
data
)
==
0
)
{
break
;
}
}
// all pages are referenced by user, try to allocate new space
if
(
pn
==
NULL
)
{
pResultBuf
->
inMemPages
=
pResultBuf
->
inMemPages
*
1
.
5
;
assert
(
0
);
return
NULL
;
return
NULL
;
}
else
{
tdListPopNode
(
pResultBuf
->
pPageList
,
pn
);
if
(
flushPageToDisk
(
pResultBuf
,
*
(
SPageInfo
**
)
pn
->
data
)
!=
TSDB_CODE_SUCCESS
)
{
return
NULL
;
}
char
*
buf
=
(
*
(
SPageInfo
**
)
pn
->
data
)
->
pData
;
(
*
(
SPageInfo
**
)
pn
->
data
)
->
pData
=
NULL
;
// load file in disk
fseek
(
pResultBuf
->
file
,
(
*
pi
)
->
info
.
offset
,
SEEK_SET
);
fread
(
buf
,
(
*
pi
)
->
info
.
length
,
1
,
pResultBuf
->
file
);
(
*
pi
)
->
pData
=
buf
;
return
(
*
pi
)
->
pData
;
}
}
}
}
// register new id in this group
return
NULL
;
*
pageId
=
(
++
pResultBuf
->
allocateId
);
}
registerPageId
(
pResultBuf
,
groupId
,
*
pageId
);
// clear memory for the new page
void
releaseResBufPage
(
SDiskbasedResultBuf
*
pResultBuf
,
void
*
page
)
{
tFilePage
*
page
=
getResBufPage
(
pResultBuf
,
*
pageId
);
assert
(
pResultBuf
!=
NULL
&&
page
!=
NULL
);
memset
(
page
,
0
,
pResultBuf
->
pageSize
);
char
*
p
=
(
char
*
)
page
-
POINTER_BYTES
;
return
page
;
SPageInfo
*
ppi
=
((
SPageInfo
**
)
p
)[
0
];
assert
(
T_REF_VAL_GET
(
ppi
)
>
0
);
T_REF_DEC
(
ppi
);
}
}
int32_t
getNumOfRowsPerPage
(
SDiskbasedResultBuf
*
pResultBuf
)
{
return
pResultBuf
->
numOfRowsPerPage
;
}
int32_t
getNumOfRowsPerPage
(
SDiskbasedResultBuf
*
pResultBuf
)
{
return
pResultBuf
->
numOfRowsPerPage
;
}
SIDList
getDataBufPagesIdList
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
SIDList
getDataBufPagesIdList
(
SDiskbasedResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
int32_t
slot
=
getGroupIndex
(
pResultBuf
,
groupId
);
assert
(
pResultBuf
!=
NULL
);
if
(
slot
<
0
)
{
char
**
p
=
taosHashGet
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
));
if
(
p
==
NULL
)
{
// it is a new group id
return
pResultBuf
->
emptyDummyIdList
;
return
pResultBuf
->
emptyDummyIdList
;
}
else
{
}
else
{
return
taosArrayGetP
(
pResultBuf
->
list
,
slot
);
return
(
SArray
*
)
(
*
p
);
}
}
}
}
...
@@ -188,12 +311,11 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
...
@@ -188,12 +311,11 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
return
;
return
;
}
}
if
(
FD_VALID
(
pResultBuf
->
fd
)
)
{
if
(
pResultBuf
->
file
!=
NULL
)
{
qDebug
(
"QInfo:%p disk-based output buffer closed, total:%"
PRId64
" bytes, file created:%s, file size:%d"
,
handle
,
qDebug
(
"QInfo:%p disk-based output buffer closed, total:%"
PRId64
" bytes, file created:%s, file size:%d"
,
handle
,
pResultBuf
->
totalBufSize
,
pResultBuf
->
path
,
FILE_SIZE_ON_DISK
(
pResultBuf
));
pResultBuf
->
totalBufSize
,
pResultBuf
->
path
,
FILE_SIZE_ON_DISK
(
pResultBuf
));
close
(
pResultBuf
->
fd
);
fclose
(
pResultBuf
->
file
);
munmap
(
pResultBuf
->
pBuf
,
FILE_SIZE_ON_DISK
(
pResultBuf
));
pResultBuf
->
pBuf
=
NULL
;
pResultBuf
->
pBuf
=
NULL
;
}
else
{
}
else
{
qDebug
(
"QInfo:%p disk-based output buffer closed, total:%"
PRId64
" bytes, no file created"
,
handle
,
qDebug
(
"QInfo:%p disk-based output buffer closed, total:%"
PRId64
" bytes, no file created"
,
handle
,
...
@@ -203,17 +325,16 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
...
@@ -203,17 +325,16 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
unlink
(
pResultBuf
->
path
);
unlink
(
pResultBuf
->
path
);
tfree
(
pResultBuf
->
path
);
tfree
(
pResultBuf
->
path
);
size_t
size
=
taosArrayGetSize
(
pResultBuf
->
list
);
//
size_t size = taosArrayGetSize(pResultBuf->list);
for
(
int32_t
i
=
0
;
i
<
size
;
++
i
)
{
//
for (int32_t i = 0; i < size; ++i) {
SArray
*
pa
=
taosArrayGetP
(
pResultBuf
->
list
,
i
);
//
SArray* pa = taosArrayGetP(pResultBuf->list, i);
taosArrayDestroy
(
pa
);
//
taosArrayDestroy(pa);
}
//
}
t
aosArrayDestroy
(
pResultBuf
->
l
ist
);
t
dListFree
(
pResultBuf
->
pPageL
ist
);
taosArrayDestroy
(
pResultBuf
->
emptyDummyIdList
);
taosArrayDestroy
(
pResultBuf
->
emptyDummyIdList
);
taosHashCleanup
(
pResultBuf
->
idsTable
);
taosHashCleanup
(
pResultBuf
->
idsTable
);
tfree
(
pResultBuf
->
iBuf
);
tfree
(
pResultBuf
);
tfree
(
pResultBuf
);
}
}
...
...
src/query/tests/resultBufferTest.cpp
浏览文件 @
9a9ea692
...
@@ -18,13 +18,35 @@ void simpleTest() {
...
@@ -18,13 +18,35 @@ void simpleTest() {
tFilePage
*
pBufPage
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
pBufPage
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
ASSERT_TRUE
(
pBufPage
!=
NULL
);
ASSERT_TRUE
(
pBufPage
!=
NULL
);
ASSERT_EQ
(
getNumOfRowsPerPage
(
pResultBuf
),
(
16384L
-
sizeof
(
int64_t
))
/
64
);
ASSERT_EQ
(
getResBufSize
(
pResultBuf
),
1024
);
ASSERT_EQ
(
getResBufSize
(
pResultBuf
),
1000
*
16384L
);
SIDList
list
=
getDataBufPagesIdList
(
pResultBuf
,
groupId
);
SIDList
list
=
getDataBufPagesIdList
(
pResultBuf
,
groupId
);
ASSERT_EQ
(
taosArrayGetSize
(
list
),
1
);
ASSERT_EQ
(
taosArrayGetSize
(
list
),
1
);
ASSERT_EQ
(
getNumOfResultBufGroupId
(
pResultBuf
),
1
);
ASSERT_EQ
(
getNumOfResultBufGroupId
(
pResultBuf
),
1
);
releaseResBufPage
(
pResultBuf
,
pBufPage
);
tFilePage
*
pBufPage1
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
t
=
getResBufPage
(
pResultBuf
,
pageId
);
assert
(
t
==
pBufPage1
);
tFilePage
*
pBufPage2
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
t1
=
getResBufPage
(
pResultBuf
,
pageId
);
assert
(
t1
==
pBufPage2
);
tFilePage
*
pBufPage3
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
t2
=
getResBufPage
(
pResultBuf
,
pageId
);
assert
(
t2
==
pBufPage3
);
tFilePage
*
pBufPage4
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
t3
=
getResBufPage
(
pResultBuf
,
pageId
);
assert
(
t3
==
pBufPage4
);
tFilePage
*
pBufPage5
=
getNewDataBuf
(
pResultBuf
,
groupId
,
&
pageId
);
tFilePage
*
t4
=
getResBufPage
(
pResultBuf
,
pageId
);
assert
(
t4
==
pBufPage5
);
destroyResultBuf
(
pResultBuf
,
NULL
);
destroyResultBuf
(
pResultBuf
,
NULL
);
}
}
}
// namespace
}
// namespace
...
...
src/util/src/tlist.c
浏览文件 @
9a9ea692
...
@@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
...
@@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
SListNode
*
node
=
(
SListNode
*
)
malloc
(
sizeof
(
SListNode
)
+
list
->
eleSize
);
SListNode
*
node
=
(
SListNode
*
)
malloc
(
sizeof
(
SListNode
)
+
list
->
eleSize
);
if
(
node
==
NULL
)
return
-
1
;
if
(
node
==
NULL
)
return
-
1
;
node
->
next
=
node
->
prev
=
NULL
;
memcpy
((
void
*
)(
node
->
data
),
data
,
list
->
eleSize
);
memcpy
((
void
*
)(
node
->
data
),
data
,
list
->
eleSize
);
tdListPrependNode
(
list
,
node
);
tdListPrependNode
(
list
,
node
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录