Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
王长生-AnH
TDengine
提交
26e4efe5
T
TDengine
项目概览
王长生-AnH
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
5
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
26e4efe5
编写于
2月 07, 2020
作者:
H
hjxilinx
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor code for query intermeidate buffer.
上级
1de610a4
变更
14
展开全部
隐藏空白更改
内联
并排
Showing
14 changed file
with
683 addition
and
252 deletion
+683
-252
src/client/inc/tscSecondaryMerge.h
src/client/inc/tscSecondaryMerge.h
+4
-4
src/client/src/tscSecondaryMerge.c
src/client/src/tscSecondaryMerge.c
+8
-7
src/client/src/tscServer.c
src/client/src/tscServer.c
+1
-0
src/client/src/tscUtil.c
src/client/src/tscUtil.c
+1
-1
src/inc/hash.h
src/inc/hash.h
+8
-6
src/inc/tresultBuf.h
src/inc/tresultBuf.h
+112
-0
src/system/detail/inc/vnodeQueryImpl.h
src/system/detail/inc/vnodeQueryImpl.h
+6
-6
src/system/detail/inc/vnodeRead.h
src/system/detail/inc/vnodeRead.h
+13
-10
src/system/detail/src/vnodeQueryImpl.c
src/system/detail/src/vnodeQueryImpl.c
+272
-192
src/system/detail/src/vnodeQueryProcess.c
src/system/detail/src/vnodeQueryProcess.c
+4
-2
src/util/src/hash.c
src/util/src/hash.c
+9
-1
src/util/src/textbuffer.c
src/util/src/textbuffer.c
+20
-22
src/util/src/tresultBuf.c
src/util/src/tresultBuf.c
+224
-0
src/util/src/ttokenizer.c
src/util/src/ttokenizer.c
+1
-1
未找到文件。
src/client/inc/tscSecondaryMerge.h
浏览文件 @
26e4efe5
...
...
@@ -90,14 +90,14 @@ typedef struct SSubqueryState {
}
SSubqueryState
;
typedef
struct
SRetrieveSupport
{
tExtMemBuffer
**
pExtMemBuffer
;
// for build loser tree
tExtMemBuffer
**
pExtMemBuffer
;
// for build loser tree
tOrderDescriptor
*
pOrderDescriptor
;
SColumnModel
*
pFinalColModel
;
// colModel for final result
SColumnModel
*
pFinalColModel
;
// colModel for final result
SSubqueryState
*
pState
;
int32_t
subqueryIndex
;
// index of current vnode in vnode list
SSqlObj
*
pParentSqlObj
;
tFilePage
*
localBuffer
;
// temp buffer, there is a buffer for each vnode to
uint32_t
numOfRetry
;
// record the number of retry times
tFilePage
*
localBuffer
;
// temp buffer, there is a buffer for each vnode to
uint32_t
numOfRetry
;
// record the number of retry times
pthread_mutex_t
queryMutex
;
}
SRetrieveSupport
;
...
...
src/client/src/tscSecondaryMerge.c
浏览文件 @
26e4efe5
...
...
@@ -387,18 +387,19 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
int32_t
saveToBuffer
(
tExtMemBuffer
*
pMemoryBuf
,
tOrderDescriptor
*
pDesc
,
tFilePage
*
pPage
,
void
*
data
,
int32_t
numOfRows
,
int32_t
orderType
)
{
if
(
pPage
->
numOfElems
+
numOfRows
<=
pDesc
->
pColumnModel
->
capacity
)
{
tColModelAppend
(
pDesc
->
pColumnModel
,
pPage
,
data
,
0
,
numOfRows
,
numOfRows
);
SColumnModel
*
pModel
=
pDesc
->
pColumnModel
;
if
(
pPage
->
numOfElems
+
numOfRows
<=
pModel
->
capacity
)
{
tColModelAppend
(
pModel
,
pPage
,
data
,
0
,
numOfRows
,
numOfRows
);
return
0
;
}
SColumnModel
*
pModel
=
pDesc
->
pColumnModel
;
int32_t
numOfRemainEntries
=
pDesc
->
pColumnModel
->
capacity
-
pPage
->
numOfElems
;
// current buffer is overflow, flush data to extensive buffer
int32_t
numOfRemainEntries
=
pModel
->
capacity
-
pPage
->
numOfElems
;
tColModelAppend
(
pModel
,
pPage
,
data
,
0
,
numOfRemainEntries
,
numOfRows
);
/
* current buffer is full, need to flushed to disk */
assert
(
pPage
->
numOfElems
==
p
Desc
->
pColumn
Model
->
capacity
);
/
/ current buffer is full, need to flushed to disk
assert
(
pPage
->
numOfElems
==
pModel
->
capacity
);
int32_t
ret
=
tscFlushTmpBuffer
(
pMemoryBuf
,
pDesc
,
pPage
,
orderType
);
if
(
ret
!=
0
)
{
return
-
1
;
...
...
src/client/src/tscServer.c
浏览文件 @
26e4efe5
...
...
@@ -1202,6 +1202,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
tscAbortFurtherRetryRetrieval
(
trsupport
,
tres
,
TSDB_CODE_CLI_NO_DISKSPACE
);
return
;
}
int32_t
ret
=
saveToBuffer
(
trsupport
->
pExtMemBuffer
[
idx
],
pDesc
,
trsupport
->
localBuffer
,
pRes
->
data
,
pRes
->
numOfRows
,
pQueryInfo
->
groupbyExpr
.
orderType
);
if
(
ret
<
0
)
{
...
...
src/client/src/tscUtil.c
浏览文件 @
26e4efe5
...
...
@@ -676,7 +676,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
STableDataBlocks
**
dataBlocks
)
{
*
dataBlocks
=
NULL
;
STableDataBlocks
**
t1
=
(
STableDataBlocks
**
)
taosGetDataFromHash
(
pHashList
,
(
const
char
*
)
&
id
,
sizeof
(
id
));
STableDataBlocks
**
t1
=
(
STableDataBlocks
**
)
taosGetDataFromHash
Table
(
pHashList
,
(
const
char
*
)
&
id
,
sizeof
(
id
));
if
(
t1
!=
NULL
)
{
*
dataBlocks
=
*
t1
;
}
...
...
src/inc/hash.h
浏览文件 @
26e4efe5
...
...
@@ -43,10 +43,10 @@ typedef struct SHashEntry {
typedef
struct
HashObj
{
SHashEntry
**
hashList
;
uint32_t
capacity
;
int
size
;
_hash_fn_t
hashFp
;
bool
multithreadSafe
;
// enable lock
uint32_t
capacity
;
// number of slots
int
size
;
// number of elements in hash table
_hash_fn_t
hashFp
;
// hash function
bool
multithreadSafe
;
// enable lock
or not
#if defined LINUX
pthread_rwlock_t
lock
;
...
...
@@ -57,11 +57,13 @@ typedef struct HashObj {
}
HashObj
;
void
*
taosInitHashTable
(
uint32_t
capacity
,
_hash_fn_t
fn
,
bool
multithreadSafe
);
void
taosDeleteFromHashTable
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
);
int32_t
taosAddToHashTable
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
,
void
*
data
,
uint32_t
size
);
void
taosDeleteFromHashTable
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
);
int32_t
taosNumElemsInHashTable
(
HashObj
*
pObj
);
char
*
taosGetDataFromHashTable
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
);
char
*
taosGetDataFromHash
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
);
void
taosCleanUpHashTable
(
void
*
handle
);
...
...
src/inc/tresultBuf.h
0 → 100644
浏览文件 @
26e4efe5
#ifndef TDENGINE_VNODEQUERYUTIL_H
#define TDENGINE_VNODEQUERYUTIL_H
#ifdef __cplusplus
extern
"C"
{
#endif
#include "os.h"
#include "textbuffer.h"
typedef
struct
SIDList
{
uint32_t
alloc
;
int32_t
size
;
int32_t
*
pData
;
}
SIDList
;
typedef
struct
SQueryResultBuf
{
int32_t
numOfRowsPerPage
;
int32_t
numOfPages
;
int64_t
totalBufSize
;
int32_t
fd
;
// data file fd
int32_t
ifd
;
// index file fd
int32_t
allocateId
;
// allocated page id
int32_t
incStep
;
// minimum allocated pages
char
*
pBuf
;
// mmap buffer pointer
char
*
path
;
// file path
char
*
ipath
;
// index file path
int32_t
*
pIndexData
;
// index file data
char
*
internBuf
;
// intern buf
int32_t
internfd
;
// intern fd
char
*
internpath
;
uint32_t
numOfAllocGroupIds
;
// number of allocated id list
void
*
idsTable
;
// id hash table
SIDList
*
list
;
// for each id, there is a page id list
}
SQueryResultBuf
;
/**
* create disk-based result buffer
* @param pResultBuf
* @param size
* @param rowSize
* @return
*/
int32_t
createResultBuf
(
SQueryResultBuf
**
pResultBuf
,
int32_t
size
,
int32_t
rowSize
);
/**
*
* @param pResultBuf
* @param groupId
* @param pageId
* @return
*/
tFilePage
*
getNewDataBuf
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
*
pageId
);
/**
*
* @param pResultBuf
* @return
*/
int32_t
getNumOfRowsPerPage
(
SQueryResultBuf
*
pResultBuf
);
/**
*
* @param pResultBuf
* @param groupId
* @return
*/
SIDList
getDataBufPagesIdList
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
);
/**
* get the specified buffer page by id
* @param pResultBuf
* @param id
* @return
*/
tFilePage
*
getResultBufferPageById
(
SQueryResultBuf
*
pResultBuf
,
int32_t
id
);
/**
* get the total buffer size in the format of disk file
* @param pResultBuf
* @return
*/
int32_t
getResBufSize
(
SQueryResultBuf
*
pResultBuf
);
/**
* get the number of groups in the result buffer
* @param pResultBuf
* @return
*/
int32_t
getNumOfResultBufGroupId
(
SQueryResultBuf
*
pResultBuf
);
/**
* destroy result buffer
* @param pResultBuf
*/
void
destroyResultBuf
(
SQueryResultBuf
*
pResultBuf
);
/**
*
* @param pList
* @return
*/
int32_t
getLastPageId
(
SIDList
*
pList
);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_VNODEQUERYUTIL_H
src/system/detail/inc/vnodeQueryImpl.h
浏览文件 @
26e4efe5
...
...
@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_VNODEQUERY
UTI
L_H
#define TDENGINE_VNODEQUERY
UTI
L_H
#ifndef TDENGINE_VNODEQUERY
IMP
L_H
#define TDENGINE_VNODEQUERY
IMP
L_H
#ifdef __cplusplus
extern
"C"
{
...
...
@@ -120,7 +120,7 @@ typedef enum {
typedef
int
(
*
__block_search_fn_t
)(
char
*
data
,
int
num
,
int64_t
key
,
int
order
);
static
FORCE_INLINE
SMeterObj
*
getMeterObj
(
void
*
hashHandle
,
int32_t
sid
)
{
return
*
(
SMeterObj
**
)
taosGetDataFromHash
(
hashHandle
,
(
const
char
*
)
&
sid
,
sizeof
(
sid
));
return
*
(
SMeterObj
**
)
taosGetDataFromHash
Table
(
hashHandle
,
(
const
char
*
)
&
sid
,
sizeof
(
sid
));
}
bool
isQueryKilled
(
SQuery
*
pQuery
);
...
...
@@ -209,7 +209,7 @@ int32_t vnodeGetHeaderFile(SQueryRuntimeEnv *pRuntimeEnv, int32_t fileIndex);
* @param ekey
* @return
*/
SMeterQueryInfo
*
createMeterQueryInfo
(
SQuery
*
pQuery
,
TSKEY
skey
,
TSKEY
ekey
);
SMeterQueryInfo
*
createMeterQueryInfo
(
SQuery
*
pQuery
,
int32_t
sid
,
TSKEY
skey
,
TSKEY
ekey
);
/**
* Destroy meter query info
...
...
@@ -224,7 +224,7 @@ void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, int32_t numOfCols);
* @param skey
* @param ekey
*/
void
changeMeterQueryInfoForSuppleQuery
(
SMeterQueryInfo
*
pMeterQueryInfo
,
TSKEY
skey
,
TSKEY
ekey
);
void
changeMeterQueryInfoForSuppleQuery
(
S
QueryResultBuf
*
pResultBuf
,
S
MeterQueryInfo
*
pMeterQueryInfo
,
TSKEY
skey
,
TSKEY
ekey
);
/**
* add the new allocated disk page to meter query info
...
...
@@ -289,4 +289,4 @@ void closeAllSlidingWindow(SSlidingWindowInfo* pSlidingWindowInfo);
}
#endif
#endif // TDENGINE_VNODEQUERY
UTI
L_H
#endif // TDENGINE_VNODEQUERY
IMP
L_H
src/system/detail/inc/vnodeRead.h
浏览文件 @
26e4efe5
...
...
@@ -21,6 +21,7 @@ extern "C" {
#endif
#include "os.h"
#include "tresultBuf.h"
#include "tinterpolation.h"
#include "vnodeTagMgmt.h"
...
...
@@ -182,8 +183,8 @@ typedef struct SMeterQueryInfo {
int64_t
skey
;
int64_t
ekey
;
int32_t
numOfRes
;
uint32_t
numOfPages
;
uint32_t
numOfAlloc
;
//
uint32_t numOfPages;
//
uint32_t numOfAlloc;
int32_t
reverseIndex
;
// reversed output indicator, start from (numOfRes-1)
int16_t
reverseFillRes
;
// denote if reverse fill the results in supplementary scan required or not
int16_t
queryRangeSet
;
// denote if the query range is set, only available for interval query
...
...
@@ -191,7 +192,9 @@ typedef struct SMeterQueryInfo {
int64_t
tag
;
STSCursor
cur
;
SResultInfo
*
resultInfo
;
uint32_t
*
pageList
;
// uint32_t* pageList;
// SIDList pageIdList;
int32_t
sid
;
// for retrieve the page id list
}
SMeterQueryInfo
;
typedef
struct
SMeterDataInfo
{
...
...
@@ -235,15 +238,15 @@ typedef struct SMeterQuerySupportObj {
*/
int32_t
meterIdx
;
int32_t
meterOutputFd
;
int32_t
lastPageId
;
int32_t
numOfPages
;
//
int32_t meterOutputFd;
//
int32_t lastPageId;
//
int32_t numOfPages;
int32_t
numOfGroupResultPages
;
int32_t
groupResultSize
;
char
*
meterOutputMMapBuf
;
int64_t
bufSize
;
char
extBufFile
[
256
];
// external file name
SQueryResultBuf
*
pResultBuf
;
//
char* meterOutputMMapBuf;
//
int64_t bufSize;
//
char extBufFile[256]; // external file name
SMeterDataInfo
*
pMeterDataInfo
;
...
...
src/system/detail/src/vnodeQueryImpl.c
浏览文件 @
26e4efe5
此差异已折叠。
点击以展开。
src/system/detail/src/vnodeQueryProcess.c
浏览文件 @
26e4efe5
...
...
@@ -132,7 +132,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
pRuntimeEnv
->
pMeterObj
=
pMeterObj
;
if
(
pMeterInfo
[
k
].
pMeterQInfo
==
NULL
)
{
pMeterInfo
[
k
].
pMeterQInfo
=
createMeterQueryInfo
(
pQuery
,
pSupporter
->
rawSKey
,
pSupporter
->
rawEKey
);
pMeterInfo
[
k
].
pMeterQInfo
=
createMeterQueryInfo
(
pQuery
,
p
MeterObj
->
sid
,
p
Supporter
->
rawSKey
,
pSupporter
->
rawEKey
);
}
if
(
pMeterInfo
[
k
].
pMeterObj
==
NULL
)
{
// no data in disk for this meter, set its pointer
...
...
@@ -858,7 +858,9 @@ static void doOrderedScan(SQInfo *pQInfo) {
static
void
setupMeterQueryInfoForSupplementQuery
(
SMeterQuerySupportObj
*
pSupporter
)
{
for
(
int32_t
i
=
0
;
i
<
pSupporter
->
numOfMeters
;
++
i
)
{
SMeterQueryInfo
*
pMeterQueryInfo
=
pSupporter
->
pMeterDataInfo
[
i
].
pMeterQInfo
;
changeMeterQueryInfoForSuppleQuery
(
pMeterQueryInfo
,
pSupporter
->
rawSKey
,
pSupporter
->
rawEKey
);
SQueryResultBuf
*
pResultBuf
=
pSupporter
->
pResultBuf
;
changeMeterQueryInfoForSuppleQuery
(
pResultBuf
,
pMeterQueryInfo
,
pSupporter
->
rawSKey
,
pSupporter
->
rawEKey
);
}
}
...
...
src/util/src/hash.c
浏览文件 @
26e4efe5
...
...
@@ -346,6 +346,14 @@ static void doAddToHashTable(HashObj *pObj, SHashNode *pNode) {
// pTrace("key:%s %p add to hash table", key, pNode);
}
int32_t
taosNumElemsInHashTable
(
HashObj
*
pObj
)
{
if
(
pObj
==
NULL
)
{
return
0
;
}
return
pObj
->
size
;
}
/**
* add data node into hash table
* @param pObj hash object
...
...
@@ -392,7 +400,7 @@ int32_t taosAddToHashTable(HashObj *pObj, const char *key, uint32_t keyLen, void
return
0
;
}
char
*
taosGetDataFromHash
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
)
{
char
*
taosGetDataFromHash
Table
(
HashObj
*
pObj
,
const
char
*
key
,
uint32_t
keyLen
)
{
if
(
pObj
->
multithreadSafe
)
{
__rd_lock
(
&
pObj
->
lock
);
}
...
...
src/util/src/textbuffer.c
浏览文件 @
26e4efe5
...
...
@@ -135,11 +135,11 @@ static bool allocFlushoutInfoEntries(SFileInfo *pFileMeta) {
}
static
bool
tExtMemBufferAlloc
(
tExtMemBuffer
*
pMemBuffer
)
{
/*
* the in-mem buffer is full.
* To flush data to disk to accommodate more data
*/
if
(
pMemBuffer
->
numOfInMemPages
>
0
&&
pMemBuffer
->
numOfInMemPages
==
pMemBuffer
->
inMemCapacity
)
{
/*
* the in-mem buffer is full.
* To flush data to disk to accommodate more data
*/
if
(
!
tExtMemBufferFlush
(
pMemBuffer
))
{
return
false
;
}
...
...
@@ -147,7 +147,7 @@ static bool tExtMemBufferAlloc(tExtMemBuffer *pMemBuffer) {
/*
* We do not recycle the file page structure. And in flush data operations, all
* filepage that are full of data are destroyed after data being flushed to disk.
* file
page that are full of data are destroyed after data being flushed to disk.
*
* The memory buffer pages may be recycle in order to avoid unnecessary memory
* allocation later.
...
...
@@ -189,9 +189,9 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
pLast
=
pMemBuffer
->
pTail
;
}
if
(
pLast
->
item
.
numOfElems
+
numOfRows
<=
pMemBuffer
->
numOfElemsPerPage
)
{
// enough space for records
if
(
pLast
->
item
.
numOfElems
+
numOfRows
<=
pMemBuffer
->
numOfElemsPerPage
)
{
// enough space for records
tColModelAppend
(
pMemBuffer
->
pColumnModel
,
&
pLast
->
item
,
data
,
0
,
numOfRows
,
numOfRows
);
pMemBuffer
->
numOfElemsInBuffer
+=
numOfRows
;
pMemBuffer
->
numOfTotalElems
+=
numOfRows
;
}
else
{
...
...
@@ -205,8 +205,7 @@ int16_t tExtMemBufferPut(tExtMemBuffer *pMemBuffer, void *data, int32_t numOfRow
int32_t
remain
=
numOfRows
-
numOfRemainEntries
;
while
(
remain
>
0
)
{
if
(
!
tExtMemBufferAlloc
(
pMemBuffer
))
{
// failed to allocate memory buffer
if
(
!
tExtMemBufferAlloc
(
pMemBuffer
))
{
// failed to allocate memory buffer
return
-
1
;
}
...
...
@@ -252,7 +251,7 @@ static bool tExtMemBufferUpdateFlushoutInfo(tExtMemBuffer *pMemBuffer) {
pFlushoutInfo
->
numOfPages
=
pMemBuffer
->
numOfInMemPages
;
pFileMeta
->
flushoutData
.
nLength
+=
1
;
}
else
{
// always update the first flushout array in single_flush_model
// always update the first flush
out array in single_flush_model
pFileMeta
->
flushoutData
.
nLength
=
1
;
tFlushoutInfo
*
pFlushoutInfo
=
&
pFileMeta
->
flushoutData
.
pFlushoutInfo
[
0
];
pFlushoutInfo
->
numOfPages
+=
pMemBuffer
->
numOfInMemPages
;
...
...
@@ -320,9 +319,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) {
return
;
}
/*
* release all data in memory buffer
*/
//release all data in memory buffer
tFilePagesItem
*
first
=
pMemBuffer
->
pHead
;
while
(
first
!=
NULL
)
{
tFilePagesItem
*
ptmp
=
first
;
...
...
@@ -335,6 +332,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) {
pMemBuffer
->
numOfElemsInBuffer
=
0
;
pMemBuffer
->
numOfInMemPages
=
0
;
pMemBuffer
->
pHead
=
NULL
;
pMemBuffer
->
pTail
=
NULL
;
...
...
@@ -586,7 +584,7 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
char
*
endx
=
COLMODEL_GET_VAL
(
data
,
pDescriptor
->
pColumnModel
,
numOfRows
,
end
,
f
);
int32_t
colIdx
=
pDescriptor
->
orderIdx
.
pData
[
0
];
tSortDataPrint
(
pDescriptor
->
pColumnModel
->
pFields
[
colIdx
].
type
,
"before"
,
startx
,
midx
,
endx
);
tSortDataPrint
(
pDescriptor
->
pColumnModel
->
pFields
[
colIdx
].
field
.
type
,
"before"
,
startx
,
midx
,
endx
);
#endif
if
(
compareFn
(
pDescriptor
,
numOfRows
,
midIdx
,
start
,
data
)
==
1
)
{
...
...
@@ -607,7 +605,7 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
midx
=
COLMODEL_GET_VAL
(
data
,
pDescriptor
->
pColumnModel
,
numOfRows
,
midIdx
,
f
);
startx
=
COLMODEL_GET_VAL
(
data
,
pDescriptor
->
pColumnModel
,
numOfRows
,
start
,
f
);
endx
=
COLMODEL_GET_VAL
(
data
,
pDescriptor
->
pColumnModel
,
numOfRows
,
end
,
f
);
tSortDataPrint
(
pDescriptor
->
pColumnModel
->
pFields
[
colIdx
].
type
,
"after"
,
startx
,
midx
,
endx
);
tSortDataPrint
(
pDescriptor
->
pColumnModel
->
pFields
[
colIdx
].
field
.
type
,
"after"
,
startx
,
midx
,
endx
);
#endif
}
...
...
@@ -661,15 +659,15 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
}
#ifdef _DEBUG_VIEW
printf
(
"before sort:
\n
"
);
tRowModelDisplay
(
pDescriptor
,
numOfRows
,
data
,
end
-
start
+
1
);
//
printf("before sort:\n");
//
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
int32_t
s
=
start
,
e
=
end
;
median
(
pDescriptor
,
numOfRows
,
start
,
end
,
data
,
compareFn
);
#ifdef _DEBUG_VIEW
printf
(
"%s called: %d
\n
"
,
__FUNCTION__
,
qsort_call
++
);
//
printf("%s called: %d\n", __FUNCTION__, qsort_call++);
#endif
UNUSED
(
qsort_call
);
...
...
@@ -695,7 +693,7 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
}
#ifdef _DEBUG_VIEW
tRowModelDisplay
(
pDescriptor
,
numOfRows
,
data
,
end
-
start
+
1
);
//
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
while
(
s
<
e
)
{
...
...
@@ -714,7 +712,7 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
swap
(
pDescriptor
->
pColumnModel
,
numOfRows
,
s
,
data
,
e
);
}
#ifdef _DEBUG_VIEW
tRowModelDisplay
(
pDescriptor
,
numOfRows
,
data
,
end
-
start
+
1
);
//
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
...
...
@@ -731,7 +729,7 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
rightx
+=
(
end
-
end_same
);
#ifdef _DEBUG_VIEW
tRowModelDisplay
(
pDescriptor
,
numOfRows
,
data
,
end
-
start
+
1
);
//
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
...
...
@@ -748,7 +746,7 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
leftx
-=
(
start_same
-
start
);
#ifdef _DEBUG_VIEW
tRowModelDisplay
(
pDescriptor
,
numOfRows
,
data
,
end
-
start
+
1
);
//
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
...
...
src/util/src/tresultBuf.c
0 → 100644
浏览文件 @
26e4efe5
#include "hash.h"
#include "taoserror.h"
#include "textbuffer.h"
#include "tlog.h"
#include "tsqlfunction.h"
#include "tresultBuf.h"
#define DEFAULT_INTERN_BUF_SIZE 16384L
int32_t
createResultBuf
(
SQueryResultBuf
**
pResultBuf
,
int32_t
size
,
int32_t
rowSize
)
{
SQueryResultBuf
*
pResBuf
=
calloc
(
1
,
sizeof
(
SQueryResultBuf
));
pResBuf
->
numOfRowsPerPage
=
(
DEFAULT_INTERN_BUF_SIZE
-
sizeof
(
tFilePage
))
/
rowSize
;
pResBuf
->
numOfPages
=
size
;
pResBuf
->
totalBufSize
=
pResBuf
->
numOfPages
*
DEFAULT_INTERN_BUF_SIZE
;
pResBuf
->
incStep
=
4
;
// init id hash table
pResBuf
->
idsTable
=
taosInitHashTable
(
size
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_INT
),
false
);
pResBuf
->
list
=
calloc
(
size
,
sizeof
(
SIDList
));
pResBuf
->
numOfAllocGroupIds
=
size
;
char
path
[
4096
]
=
{
0
};
getTmpfilePath
(
"tsdb_q_buf"
,
path
);
pResBuf
->
path
=
strdup
(
path
);
pResBuf
->
fd
=
open
(
pResBuf
->
path
,
O_CREAT
|
O_RDWR
,
0666
);
memset
(
path
,
0
,
tListLen
(
path
));
getTmpfilePath
(
"tsdb_q_i"
,
path
);
pResBuf
->
internpath
=
strdup
(
path
);
pResBuf
->
internfd
=
open
(
pResBuf
->
internpath
,
O_CREAT
|
O_RDWR
,
0666
);
if
(
!
FD_VALID
(
pResBuf
->
fd
))
{
pError
(
"failed to create tmp file: %s on disk. %s"
,
pResBuf
->
path
,
strerror
(
errno
));
return
TSDB_CODE_CLI_NO_DISKSPACE
;
}
int32_t
ret
=
ftruncate
(
pResBuf
->
fd
,
pResBuf
->
numOfPages
*
DEFAULT_INTERN_BUF_SIZE
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
pError
(
"failed to create tmp file: %s on disk. %s"
,
pResBuf
->
path
,
strerror
(
errno
));
return
TSDB_CODE_CLI_NO_DISKSPACE
;
}
pResBuf
->
pBuf
=
mmap
(
NULL
,
pResBuf
->
totalBufSize
,
PROT_READ
|
PROT_WRITE
,
MAP_SHARED
,
pResBuf
->
fd
,
0
);
if
(
pResBuf
->
pBuf
==
MAP_FAILED
)
{
pError
(
"QInfo:%p failed to map temp file: %s. %s"
,
pResBuf
->
path
,
strerror
(
errno
));
return
TSDB_CODE_CLI_OUT_OF_MEMORY
;
// todo change error code
}
pTrace
(
"create tmp file for output result, %s, "
PRId64
"bytes"
,
pResBuf
->
path
,
pResBuf
->
totalBufSize
);
*
pResultBuf
=
pResBuf
;
return
TSDB_CODE_SUCCESS
;
}
tFilePage
*
getResultBufferPageById
(
SQueryResultBuf
*
pResultBuf
,
int32_t
id
)
{
assert
(
id
<
pResultBuf
->
numOfPages
&&
id
>=
0
);
return
(
tFilePage
*
)(
pResultBuf
->
pBuf
+
DEFAULT_INTERN_BUF_SIZE
*
id
);
}
int32_t
getNumOfResultBufGroupId
(
SQueryResultBuf
*
pResultBuf
)
{
return
taosNumElemsInHashTable
(
pResultBuf
->
idsTable
);
}
int32_t
getResBufSize
(
SQueryResultBuf
*
pResultBuf
)
{
return
pResultBuf
->
totalBufSize
;
}
static
int32_t
extendDiskFileSize
(
SQueryResultBuf
*
pResultBuf
,
int32_t
numOfPages
)
{
assert
(
pResultBuf
->
numOfPages
*
DEFAULT_INTERN_BUF_SIZE
==
pResultBuf
->
totalBufSize
);
int32_t
ret
=
munmap
(
pResultBuf
->
pBuf
,
pResultBuf
->
totalBufSize
);
pResultBuf
->
numOfPages
+=
numOfPages
;
/*
* disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
* be insufficient
*/
ret
=
ftruncate
(
pResultBuf
->
fd
,
pResultBuf
->
numOfPages
*
DEFAULT_INTERN_BUF_SIZE
);
if
(
ret
!=
0
)
{
// dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
// strerror(errno));
return
-
TSDB_CODE_SERV_NO_DISKSPACE
;
}
pResultBuf
->
totalBufSize
=
pResultBuf
->
numOfPages
*
DEFAULT_INTERN_BUF_SIZE
;
pResultBuf
->
pBuf
=
mmap
(
NULL
,
pResultBuf
->
totalBufSize
,
PROT_READ
|
PROT_WRITE
,
MAP_SHARED
,
pResultBuf
->
fd
,
0
);
if
(
pResultBuf
->
pBuf
==
MAP_FAILED
)
{
// dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
return
-
TSDB_CODE_SERV_OUT_OF_MEMORY
;
}
return
TSDB_CODE_SUCCESS
;
}
static
bool
noMoreAvailablePages
(
SQueryResultBuf
*
pResultBuf
)
{
return
(
pResultBuf
->
allocateId
==
pResultBuf
->
numOfPages
-
1
);
}
static
int32_t
getGroupIndex
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
assert
(
pResultBuf
!=
NULL
);
char
*
p
=
taosGetDataFromHashTable
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
));
if
(
p
==
NULL
)
{
// it is a new group id
return
-
1
;
}
int32_t
slot
=
GET_INT32_VAL
(
p
);
assert
(
slot
>=
0
&&
slot
<
pResultBuf
->
numOfAllocGroupIds
);
return
slot
;
}
static
int32_t
addNewGroupId
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
int32_t
num
=
getNumOfResultBufGroupId
(
pResultBuf
);
// the num is the newest allocated group id slot
if
(
pResultBuf
->
numOfAllocGroupIds
<=
num
)
{
size_t
n
=
pResultBuf
->
numOfAllocGroupIds
<<
1u
;
SIDList
*
p
=
(
SIDList
*
)
realloc
(
pResultBuf
->
list
,
sizeof
(
SIDList
)
*
n
);
assert
(
p
!=
NULL
);
memset
(
&
p
[
pResultBuf
->
numOfAllocGroupIds
],
0
,
sizeof
(
SIDList
)
*
pResultBuf
->
numOfAllocGroupIds
);
pResultBuf
->
list
=
p
;
pResultBuf
->
numOfAllocGroupIds
=
n
;
}
taosAddToHashTable
(
pResultBuf
->
idsTable
,
(
const
char
*
)
&
groupId
,
sizeof
(
int32_t
),
&
num
,
sizeof
(
int32_t
));
return
num
;
}
static
int32_t
doRegisterId
(
SIDList
*
pList
,
int32_t
id
)
{
if
(
pList
->
size
>=
pList
->
alloc
)
{
int32_t
s
=
0
;
if
(
pList
->
alloc
==
0
)
{
s
=
4
;
assert
(
pList
->
pData
==
NULL
);
}
else
{
s
=
pList
->
alloc
<<
1u
;
}
int32_t
*
c
=
realloc
(
pList
->
pData
,
s
*
sizeof
(
int32_t
));
assert
(
c
);
memset
(
&
c
[
pList
->
alloc
],
0
,
sizeof
(
int32_t
)
*
pList
->
alloc
);
pList
->
pData
=
c
;
pList
->
alloc
=
s
;
}
pList
->
pData
[
pList
->
size
++
]
=
id
;
return
0
;
}
static
void
registerPageId
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
pageId
)
{
int32_t
slot
=
getGroupIndex
(
pResultBuf
,
groupId
);
if
(
slot
<
0
)
{
slot
=
addNewGroupId
(
pResultBuf
,
groupId
);
}
SIDList
*
pList
=
&
pResultBuf
->
list
[
slot
];
doRegisterId
(
pList
,
pageId
);
}
tFilePage
*
getNewDataBuf
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
,
int32_t
*
pageId
)
{
if
(
noMoreAvailablePages
(
pResultBuf
))
{
if
(
extendDiskFileSize
(
pResultBuf
,
pResultBuf
->
incStep
)
!=
TSDB_CODE_SUCCESS
)
{
return
NULL
;
}
}
// register new id in this group
*
pageId
=
(
pResultBuf
->
allocateId
++
);
registerPageId
(
pResultBuf
,
groupId
,
*
pageId
);
return
getResultBufferPageById
(
pResultBuf
,
*
pageId
);
}
int32_t
getNumOfRowsPerPage
(
SQueryResultBuf
*
pResultBuf
)
{
return
pResultBuf
->
numOfRowsPerPage
;
}
SIDList
getDataBufPagesIdList
(
SQueryResultBuf
*
pResultBuf
,
int32_t
groupId
)
{
SIDList
list
=
{
0
};
int32_t
slot
=
getGroupIndex
(
pResultBuf
,
groupId
);
if
(
slot
<
0
)
{
return
list
;
}
else
{
return
pResultBuf
->
list
[
slot
];
}
}
void
destroyResultBuf
(
SQueryResultBuf
*
pResultBuf
)
{
if
(
pResultBuf
==
NULL
)
{
return
;
}
if
(
FD_VALID
(
pResultBuf
->
fd
))
{
close
(
pResultBuf
->
fd
);
}
pTrace
(
"disk-based output buffer closed, %"
PRId64
" bytes, file:%s"
,
pResultBuf
->
totalBufSize
,
pResultBuf
->
path
);
munmap
(
pResultBuf
->
pBuf
,
pResultBuf
->
totalBufSize
);
unlink
(
pResultBuf
->
path
);
tfree
(
pResultBuf
->
path
);
for
(
int32_t
i
=
0
;
i
<
pResultBuf
->
numOfAllocGroupIds
;
++
i
)
{
SIDList
*
pList
=
&
pResultBuf
->
list
[
i
];
tfree
(
pList
->
pData
);
}
tfree
(
pResultBuf
->
list
);
taosCleanUpHashTable
(
pResultBuf
->
idsTable
);
tfree
(
pResultBuf
);
}
int32_t
getLastPageId
(
SIDList
*
pList
)
{
if
(
pList
==
NULL
&&
pList
->
size
<=
0
)
{
return
-
1
;
}
return
pList
->
pData
[
pList
->
size
-
1
];
}
src/util/src/ttokenizer.c
浏览文件 @
26e4efe5
...
...
@@ -274,7 +274,7 @@ int tSQLKeywordCode(const char* z, int n) {
}
}
SKeyword
**
pKey
=
(
SKeyword
**
)
taosGetDataFromHash
(
KeywordHashTable
,
key
,
n
);
SKeyword
**
pKey
=
(
SKeyword
**
)
taosGetDataFromHash
Table
(
KeywordHashTable
,
key
,
n
);
if
(
pKey
!=
NULL
)
{
return
(
*
pKey
)
->
type
;
}
else
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录