Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
哥们要飞
TDengine
提交
fe6672d7
T
TDengine
项目概览
哥们要飞
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
fe6672d7
编写于
8月 02, 2019
作者:
S
slguan
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix the issue #264 #253 #251
上级
04504ce4
变更
27
展开全部
隐藏空白更改
内联
并排
Showing
27 changed file
with
1149 addition
and
718 deletion
+1149
-718
src/client/inc/tscSecondaryMerge.h
src/client/inc/tscSecondaryMerge.h
+3
-3
src/client/inc/tscUtil.h
src/client/inc/tscUtil.h
+12
-10
src/client/inc/tsclient.h
src/client/inc/tsclient.h
+13
-7
src/client/src/tscAsync.c
src/client/src/tscAsync.c
+1
-1
src/client/src/tscParseInsert.c
src/client/src/tscParseInsert.c
+324
-272
src/client/src/tscSQLParserImpl.c
src/client/src/tscSQLParserImpl.c
+3
-5
src/client/src/tscSecondaryMerge.c
src/client/src/tscSecondaryMerge.c
+285
-198
src/client/src/tscServer.c
src/client/src/tscServer.c
+10
-18
src/client/src/tscUtil.c
src/client/src/tscUtil.c
+126
-43
src/inc/textbuffer.h
src/inc/textbuffer.h
+1
-1
src/inc/tinterpolation.h
src/inc/tinterpolation.h
+1
-1
src/system/inc/vnode.h
src/system/inc/vnode.h
+9
-8
src/system/inc/vnodeUtil.h
src/system/inc/vnodeUtil.h
+6
-0
src/system/src/dnodeMgmt.c
src/system/src/dnodeMgmt.c
+3
-2
src/system/src/mgmtMeter.c
src/system/src/mgmtMeter.c
+1
-42
src/system/src/vnodeCache.c
src/system/src/vnodeCache.c
+21
-7
src/system/src/vnodeCommit.c
src/system/src/vnodeCommit.c
+8
-3
src/system/src/vnodeFile.c
src/system/src/vnodeFile.c
+17
-4
src/system/src/vnodeImport.c
src/system/src/vnodeImport.c
+47
-11
src/system/src/vnodeMeter.c
src/system/src/vnodeMeter.c
+54
-34
src/system/src/vnodeQueryImpl.c
src/system/src/vnodeQueryImpl.c
+11
-0
src/system/src/vnodeShell.c
src/system/src/vnodeShell.c
+52
-21
src/system/src/vnodeStore.c
src/system/src/vnodeStore.c
+43
-15
src/system/src/vnodeStream.c
src/system/src/vnodeStream.c
+11
-2
src/system/src/vnodeUtil.c
src/system/src/vnodeUtil.c
+85
-8
src/util/src/textbuffer.c
src/util/src/textbuffer.c
+1
-1
src/util/src/tinterpolation.c
src/util/src/tinterpolation.c
+1
-1
未找到文件。
src/client/inc/tscSecondaryMerge.h
浏览文件 @
fe6672d7
...
...
@@ -37,13 +37,13 @@ extern "C" {
struct
SQLFunctionCtx
;
typedef
struct
SLocalDataS
rc
{
typedef
struct
SLocalDataS
ource
{
tExtMemBuffer
*
pMemBuffer
;
int32_t
flushoutIdx
;
int32_t
pageId
;
int32_t
rowIdx
;
tFilePage
filePage
;
}
SLocalDataS
rc
;
}
SLocalDataS
ource
;
enum
{
TSC_LOCALREDUCE_READY
=
0x0
,
...
...
@@ -52,7 +52,7 @@ enum {
};
typedef
struct
SLocalReducer
{
SLocalDataS
rc
**
pLocalDataSrc
;
SLocalDataS
ource
**
pLocalDataSrc
;
int32_t
numOfBuffer
;
int32_t
numOfCompleted
;
...
...
src/client/inc/tscUtil.h
浏览文件 @
fe6672d7
...
...
@@ -41,21 +41,24 @@ typedef struct SParsedColElem {
}
SParsedColElem
;
typedef
struct
SParsedDataColInfo
{
bool
ordered
;
// denote if the timestamp in one data block ordered or not
int16_t
numOfCols
;
int16_t
numOfAssignedCols
;
SParsedColElem
elems
[
TSDB_MAX_COLUMNS
];
bool
hasVal
[
TSDB_MAX_COLUMNS
];
int64_t
prevTimestamp
;
}
SParsedDataColInfo
;
SInsertedDataBlocks
*
tscCreateDataBlock
(
int32_t
size
);
void
tscDestroyDataBlock
(
SInsertedDataBlocks
**
pDataBlock
);
STableDataBlocks
*
tscCreateDataBlock
(
int32_t
size
);
void
tscDestroyDataBlock
(
STableDataBlocks
*
pDataBlock
);
void
tscAppendDataBlock
(
SDataBlockList
*
pList
,
STableDataBlocks
*
pBlocks
);
SDataBlockList
*
tscCreateBlockArrayList
();
void
tscDestroyBlockArrayList
(
SDataBlockList
*
*
pList
);
int32_t
tscCopyDataBlockToPayload
(
SSqlObj
*
pSql
,
S
Inserted
DataBlocks
*
pDataBlock
);
void
*
tscDestroyBlockArrayList
(
SDataBlockList
*
pList
);
int32_t
tscCopyDataBlockToPayload
(
SSqlObj
*
pSql
,
S
Table
DataBlocks
*
pDataBlock
);
void
tscFreeUnusedDataBlocks
(
SDataBlockList
*
pList
);
void
tscMergeTableDataBlocks
(
SSqlCmd
*
pCmd
,
SDataBlockList
*
pDataList
);
STableDataBlocks
*
tscGetDataBlockFromList
(
void
*
pHashList
,
SDataBlockList
*
pDataBlockList
,
int64_t
id
,
int32_t
size
,
int32_t
startOffset
,
int32_t
rowSize
,
char
*
tableId
);
STableDataBlocks
*
tscCreateDataBlockEx
(
size_t
size
,
int32_t
rowSize
,
int32_t
startOffset
,
char
*
name
);
SVnodeSidList
*
tscGetVnodeSidList
(
SMetricMeta
*
pMetricmeta
,
int32_t
vnodeIdx
);
SMeterSidExtInfo
*
tscGetMeterSidInfo
(
SVnodeSidList
*
pSidList
,
int32_t
idx
);
...
...
@@ -66,8 +69,7 @@ bool tscIsTwoStageMergeMetricQuery(SSqlObj* pSql);
/**
*
* for the projection query on metric or point interpolation query on metric,
* we iterate all the meters, instead of invoke query on all qualified meters
* simultaneously.
* we iterate all the meters, instead of invoke query on all qualified meters simultaneously.
*
* @param pSql sql object
* @return
...
...
@@ -124,8 +126,7 @@ void tscIncStreamExecutionCount(void* pStream);
bool
tscValidateColumnId
(
SSqlCmd
*
pCmd
,
int32_t
colId
);
// get starter position of metric query condition (query on tags) in
// SSqlCmd.payload
// get starter position of metric query condition (query on tags) in SSqlCmd.payload
char
*
tsGetMetricQueryCondPos
(
STagCond
*
pCond
);
void
tscTagCondAssign
(
STagCond
*
pDst
,
STagCond
*
pSrc
);
void
tscTagCondRelease
(
STagCond
*
pCond
);
...
...
@@ -139,6 +140,7 @@ void tscCleanSqlCmd(SSqlCmd* pCmd);
bool
tscShouldFreeAsyncSqlObj
(
SSqlObj
*
pSql
);
void
tscDoQuery
(
SSqlObj
*
pSql
);
int32_t
sortRemoveDuplicates
(
STableDataBlocks
*
dataBuf
,
int32_t
numOfRows
);
#ifdef __cplusplus
}
#endif
...
...
src/client/inc/tsclient.h
浏览文件 @
fe6672d7
...
...
@@ -169,16 +169,22 @@ typedef struct STagCond {
char
*
pData
;
}
STagCond
;
typedef
struct
SInsertedDataBlocks
{
char
meterId
[
TSDB_METER_ID_LEN
];
int64_t
size
;
uint32_t
nAllocSize
;
uint32_t
numOfMeters
;
typedef
struct
STableDataBlocks
{
char
meterId
[
TSDB_METER_ID_LEN
];
int64_t
vgid
;
int64_t
size
;
int64_t
prevTS
;
bool
ordered
;
int32_t
numOfMeters
;
int32_t
rowSize
;
uint32_t
nAllocSize
;
union
{
char
*
filename
;
char
*
pData
;
};
}
S
Inserted
DataBlocks
;
}
S
Table
DataBlocks
;
typedef
struct
SDataBlockList
{
int32_t
idx
;
...
...
@@ -186,7 +192,7 @@ typedef struct SDataBlockList {
int32_t
nAlloc
;
char
*
userParam
;
/* user assigned parameters for async query */
void
*
udfp
;
/* user defined function pointer, used in async model */
S
Inserted
DataBlocks
**
pData
;
S
Table
DataBlocks
**
pData
;
}
SDataBlockList
;
typedef
struct
{
...
...
src/client/src/tscAsync.c
浏览文件 @
fe6672d7
...
...
@@ -410,7 +410,7 @@ void tscAsyncInsertMultiVnodesProxy(void *param, TAOS_RES *tres, int numOfRows)
tscTrace
(
"%p Async insertion completed, destroy data block list"
,
pSql
);
// release data block data
tscDestroyBlockArrayList
(
&
pCmd
->
pDataBlocks
);
pCmd
->
pDataBlocks
=
tscDestroyBlockArrayList
(
pCmd
->
pDataBlocks
);
// all data has been sent to vnode, call user function
(
*
pSql
->
fp
)(
pSql
->
param
,
tres
,
numOfRows
);
...
...
src/client/src/tscParseInsert.c
浏览文件 @
fe6672d7
此差异已折叠。
点击以展开。
src/client/src/tscSQLParserImpl.c
浏览文件 @
fe6672d7
...
...
@@ -140,12 +140,10 @@ tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optrType) {
nodePtr
->
val
.
nType
=
TSDB_DATA_TYPE_BIGINT
;
nodePtr
->
nSQLOptr
=
TK_TIMESTAMP
;
}
else
{
// must be field id if not numbers
if
(
pToken
!=
NULL
)
{
assert
(
optrType
==
TK_ID
);
/* it must be the column name (tk_id) */
assert
(
optrType
==
TK_ALL
||
optrType
==
TK_ID
);
if
(
pToken
!=
NULL
)
{
// it must be the column name (tk_id)
nodePtr
->
colInfo
=
*
pToken
;
}
else
{
assert
(
optrType
==
TK_ALL
);
}
nodePtr
->
nSQLOptr
=
optrType
;
...
...
src/client/src/tscSecondaryMerge.c
浏览文件 @
fe6672d7
此差异已折叠。
点击以展开。
src/client/src/tscServer.c
浏览文件 @
fe6672d7
...
...
@@ -358,14 +358,17 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) {
pRes
->
code
=
TSDB_CODE_SUCCESS
;
}
tscTrace
(
"%p cmd:%d code:%d rsp len:%d"
,
pSql
,
pCmd
->
command
,
pRes
->
code
,
pRes
->
rspLen
);
/*
* There is not response callback function for submit response.
* The actual inserted number of points is the first number.
*/
if
(
pMsg
->
msgType
==
TSDB_MSG_TYPE_SUBMIT_RSP
)
{
pRes
->
numOfRows
+=
*
(
int32_t
*
)
pRes
->
pRsp
;
tscTrace
(
"%p cmd:%d code:%d, inserted rows:%d, rsp len:%d"
,
pSql
,
pCmd
->
command
,
pRes
->
code
,
*
(
int32_t
*
)
pRes
->
pRsp
,
pRes
->
rspLen
);
}
else
{
tscTrace
(
"%p cmd:%d code:%d rsp len:%d"
,
pSql
,
pCmd
->
command
,
pRes
->
code
,
pRes
->
rspLen
);
}
}
...
...
@@ -421,7 +424,7 @@ void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle) {
return
ahandle
;
}
static
SSqlObj
*
tscCreateSqlObjForSubquery
(
SSqlObj
*
pSql
,
SRetrieveSupport
*
trsupport
,
SSqlObj
*
p
Old
);
static
SSqlObj
*
tscCreateSqlObjForSubquery
(
SSqlObj
*
pSql
,
SRetrieveSupport
*
trsupport
,
SSqlObj
*
p
revSqlObj
);
static
int
tscLaunchMetricSubQueries
(
SSqlObj
*
pSql
);
int
tscProcessSql
(
SSqlObj
*
pSql
)
{
...
...
@@ -430,12 +433,6 @@ int tscProcessSql(SSqlObj *pSql) {
tscTrace
(
"%p SQL cmd:%d will be processed, name:%s"
,
pSql
,
pSql
->
cmd
.
command
,
pSql
->
cmd
.
name
);
// whether don't judge 'isInsertFromFile' ?
if
(
pSql
->
cmd
.
command
==
TSDB_SQL_INSERT
&&
pCmd
->
isInsertFromFile
==
1
)
{
// pCmd->isInsertFromFile = 0; // lihui: can not clear the flag
return
0
;
}
pSql
->
retry
=
0
;
if
(
pSql
->
cmd
.
command
<
TSDB_SQL_MGMT
)
{
pSql
->
maxRetry
=
2
;
...
...
@@ -595,7 +592,6 @@ int tscLaunchMetricSubQueries(SSqlObj *pSql) {
SSqlObj
*
pNew
=
tscCreateSqlObjForSubquery
(
pSql
,
trs
,
NULL
);
tscTrace
(
"%p sub:%p launch subquery.orderOfSub:%d"
,
pSql
,
pNew
,
pNew
->
cmd
.
vnodeIdx
);
tscProcessSql
(
pNew
);
}
...
...
@@ -665,7 +661,6 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
tscError
(
"%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d"
,
pPObj
,
pSql
,
idx
,
*
trsupport
->
code
);
}
else
{
if
(
trsupport
->
numOfRetry
++
<
MAX_NUM_OF_SUBQUERY_RETRY
&&
*
(
trsupport
->
code
)
==
TSDB_CODE_SUCCESS
)
{
/*
* current query failed, and the retry count is less than the available count,
...
...
@@ -675,11 +670,12 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
// clear local saved number of results
trsupport
->
localBuffer
->
numOfElems
=
0
;
pthread_mutex_unlock
(
&
trsupport
->
queryMutex
);
SSqlObj
*
pNew
=
tscCreateSqlObjForSubquery
(
trsupport
->
pParentSqlObj
,
trsupport
,
pSql
);
tscTrace
(
"%p sub:%p retrieve failed, code:%d, orderOfSub:%d, retry:%d, new SqlObj:%p"
,
trsupport
->
pParentSqlObj
,
pSql
,
numOfRows
,
idx
,
trsupport
->
numOfRetry
,
pNew
);
trsupport
->
pParentSqlObj
,
pSql
,
numOfRows
,
idx
,
trsupport
->
numOfRetry
,
pNew
);
tscProcessSql
(
pNew
);
return
;
...
...
@@ -689,7 +685,6 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
tscError
(
"%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d"
,
pPObj
,
pSql
,
numOfRows
,
idx
,
*
trsupport
->
code
);
}
}
if
(
__sync_add_and_fetch_32
(
trsupport
->
numOfFinished
,
1
)
<
trsupport
->
numOfVnodes
)
{
...
...
@@ -778,7 +773,7 @@ void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
tscTrace
(
"%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d"
,
pPObj
,
pSql
,
pSvd
->
ip
,
pSvd
->
vnode
,
numOfRowsFromVnode
,
idx
);
tColModelComp
ress
(
pDesc
->
pSchema
,
trsupport
->
localBuffer
,
pDesc
->
pSchema
->
maxCapacity
);
tColModelComp
act
(
pDesc
->
pSchema
,
trsupport
->
localBuffer
,
pDesc
->
pSchema
->
maxCapacity
);
#ifdef _DEBUG_VIEW
printf
(
"%ld rows data flushed to disk:
\n
"
,
trsupport
->
localBuffer
->
numOfElems
);
...
...
@@ -877,7 +872,7 @@ void tscKillMetricQuery(SSqlObj *pSql) {
tscTrace
(
"%p metric query is cancelled"
,
pSql
);
}
static
SSqlObj
*
tscCreateSqlObjForSubquery
(
SSqlObj
*
pSql
,
SRetrieveSupport
*
trsupport
,
SSqlObj
*
prevSqlObj
)
{
SSqlObj
*
tscCreateSqlObjForSubquery
(
SSqlObj
*
pSql
,
SRetrieveSupport
*
trsupport
,
SSqlObj
*
prevSqlObj
)
{
SSqlCmd
*
pCmd
=
&
pSql
->
cmd
;
SSqlObj
*
pNew
=
(
SSqlObj
*
)
calloc
(
1
,
sizeof
(
SSqlObj
));
...
...
@@ -2264,8 +2259,6 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql) {
SSqlGroupbyExpr
*
pGroupby
=
&
pCmd
->
groupbyExpr
;
pMetaMsg
->
limit
=
htobe64
(
pCmd
->
glimit
.
limit
);
pMetaMsg
->
offset
=
htobe64
(
pCmd
->
glimit
.
offset
);
pMetaMsg
->
numOfTags
=
htons
(
pCmd
->
numOfReqTags
);
pMetaMsg
->
numOfGroupbyCols
=
htons
(
pGroupby
->
numOfGroupbyCols
);
...
...
@@ -2750,7 +2743,6 @@ static int32_t tscDoGetMeterMeta(SSqlObj *pSql, char *meterId) {
}
else
{
pNew
->
fp
=
tscMeterMetaCallBack
;
pNew
->
param
=
pSql
;
pNew
->
sqlstr
=
strdup
(
pSql
->
sqlstr
);
code
=
tscProcessSql
(
pNew
);
...
...
src/client/src/tscUtil.c
浏览文件 @
fe6672d7
...
...
@@ -17,6 +17,7 @@
#include <math.h>
#include <time.h>
#include "ihash.h"
#include "taosmsg.h"
#include "tcache.h"
#include "tkey.h"
...
...
@@ -31,9 +32,10 @@
/*
* the detailed information regarding metric meta key is:
* fullmetername + '.' + querycond + '.' + [tagId1, tagId2,...] + '.' + group_orderType + '.' + limit + '.' + offset
* fullmetername + '.' + querycond + '.' + [tagId1, tagId2,...] + '.' + group_orderType
*
* if querycond is null, its format is:
* fullmetername + '.' + '(nil)' + '.' + [tagId1, tagId2,...] + '.' + group_orderType
+ '.' + limit + '.' + offset
* fullmetername + '.' + '(nil)' + '.' + [tagId1, tagId2,...] + '.' + group_orderType
*/
void
tscGetMetricMetaCacheKey
(
SSqlCmd
*
pCmd
,
char
*
keyStr
)
{
char
*
pTagCondStr
=
NULL
;
...
...
@@ -60,8 +62,7 @@ void tscGetMetricMetaCacheKey(SSqlCmd* pCmd, char* keyStr) {
pTagCondStr
=
strdup
(
tsGetMetricQueryCondPos
(
&
pCmd
->
tagCond
));
}
int32_t
keyLen
=
sprintf
(
keyStr
,
"%s.%s.[%s].%d.%lld.%lld"
,
pCmd
->
name
,
pTagCondStr
,
tagIdBuf
,
pCmd
->
groupbyExpr
.
orderType
,
pCmd
->
glimit
.
limit
,
pCmd
->
glimit
.
offset
);
int32_t
keyLen
=
sprintf
(
keyStr
,
"%s.%s.[%s].%d"
,
pCmd
->
name
,
pTagCondStr
,
tagIdBuf
,
pCmd
->
groupbyExpr
.
orderType
);
free
(
pTagCondStr
);
assert
(
keyLen
<=
TSDB_MAX_TAGS_LEN
);
...
...
@@ -142,8 +143,7 @@ bool tscProjectionQueryOnMetric(SSqlObj* pSql) {
/*
* In following cases, return false for project query on metric
* 1. failed to get metermeta from server; 2. not a metric; 3. limit 0; 4.
* show query, instead of a select query
* 1. failed to get metermeta from server; 2. not a metric; 3. limit 0; 4. show query, instead of a select query
*/
if
(
pCmd
->
pMeterMeta
==
NULL
||
!
UTIL_METER_IS_METRIC
(
pCmd
)
||
pCmd
->
command
==
TSDB_SQL_RETRIEVE_EMPTY_RESULT
||
pCmd
->
exprsInfo
.
numOfExprs
==
0
)
{
...
...
@@ -252,7 +252,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) {
}
void
tscfreeSqlCmdData
(
SSqlCmd
*
pCmd
)
{
tscDestroyBlockArrayList
(
&
pCmd
->
pDataBlocks
);
pCmd
->
pDataBlocks
=
tscDestroyBlockArrayList
(
pCmd
->
pDataBlocks
);
tscTagCondRelease
(
&
pCmd
->
tagCond
);
tscClearFieldInfo
(
pCmd
);
...
...
@@ -334,20 +334,22 @@ void tscFreeSqlObj(SSqlObj* pSql) {
free
(
pSql
);
}
S
Inserted
DataBlocks
*
tscCreateDataBlock
(
int32_t
size
)
{
S
InsertedDataBlocks
*
dataBuf
=
(
SInsertedDataBlocks
*
)
calloc
(
1
,
sizeof
(
SInserted
DataBlocks
));
dataBuf
->
nAllocSize
=
(
uint32_t
)
size
;
S
Table
DataBlocks
*
tscCreateDataBlock
(
int32_t
size
)
{
S
TableDataBlocks
*
dataBuf
=
(
STableDataBlocks
*
)
calloc
(
1
,
sizeof
(
STable
DataBlocks
));
dataBuf
->
nAllocSize
=
(
uint32_t
)
size
;
dataBuf
->
pData
=
calloc
(
1
,
dataBuf
->
nAllocSize
);
dataBuf
->
ordered
=
true
;
dataBuf
->
prevTS
=
INT64_MIN
;
return
dataBuf
;
}
void
tscDestroyDataBlock
(
S
InsertedDataBlocks
*
*
pDataBlock
)
{
if
(
*
pDataBlock
==
NULL
)
{
void
tscDestroyDataBlock
(
S
TableDataBlocks
*
pDataBlock
)
{
if
(
pDataBlock
==
NULL
)
{
return
;
}
tfree
(
(
*
pDataBlock
)
->
pData
);
tfree
(
*
pDataBlock
);
tfree
(
pDataBlock
->
pData
);
tfree
(
pDataBlock
);
}
SDataBlockList
*
tscCreateBlockArrayList
()
{
...
...
@@ -360,29 +362,31 @@ SDataBlockList* tscCreateBlockArrayList() {
return
pDataBlockArrayList
;
}
void
tscDestroyBlockArrayList
(
SDataBlockList
*
*
pList
)
{
if
(
*
pList
==
NULL
)
{
return
;
void
*
tscDestroyBlockArrayList
(
SDataBlockList
*
pList
)
{
if
(
pList
==
NULL
)
{
return
NULL
;
}
for
(
int32_t
i
=
0
;
i
<
(
*
pList
)
->
nSize
;
i
++
)
{
tscDestroyDataBlock
(
&
(
*
pList
)
->
pData
[
i
]);
for
(
int32_t
i
=
0
;
i
<
pList
->
nSize
;
i
++
)
{
tscDestroyDataBlock
(
pList
->
pData
[
i
]);
}
tfree
((
*
pList
)
->
pData
);
tfree
(
*
pList
);
tfree
(
pList
->
pData
);
tfree
(
pList
);
return
NULL
;
}
int32_t
tscCopyDataBlockToPayload
(
SSqlObj
*
pSql
,
S
Inserted
DataBlocks
*
pDataBlock
)
{
int32_t
tscCopyDataBlockToPayload
(
SSqlObj
*
pSql
,
S
Table
DataBlocks
*
pDataBlock
)
{
SSqlCmd
*
pCmd
=
&
pSql
->
cmd
;
pCmd
->
count
=
pDataBlock
->
numOfMeters
;
str
cpy
(
pCmd
->
name
,
pDataBlock
->
meterId
);
str
ncpy
(
pCmd
->
name
,
pDataBlock
->
meterId
,
TSDB_METER_ID_LEN
);
tscAllocPayloadWithSize
(
pCmd
,
pDataBlock
->
nAllocSize
);
memcpy
(
pCmd
->
payload
,
pDataBlock
->
pData
,
pDataBlock
->
nAllocSize
);
/
* set the message length */
/
/ set the message length
pCmd
->
payloadLen
=
pDataBlock
->
nAllocSize
;
return
tscGetMeterMeta
(
pSql
,
pCmd
->
name
);
}
...
...
@@ -390,12 +394,89 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, SInsertedDataBlocks* pDataBlock
void
tscFreeUnusedDataBlocks
(
SDataBlockList
*
pList
)
{
/* release additional memory consumption */
for
(
int32_t
i
=
0
;
i
<
pList
->
nSize
;
++
i
)
{
S
Inserted
DataBlocks
*
pDataBlock
=
pList
->
pData
[
i
];
pDataBlock
->
pData
=
realloc
(
pDataBlock
->
pData
,
(
size_t
)
pDataBlock
->
size
);
pDataBlock
->
nAllocSize
=
(
uint32_t
)
pDataBlock
->
size
;
S
Table
DataBlocks
*
pDataBlock
=
pList
->
pData
[
i
];
pDataBlock
->
pData
=
realloc
(
pDataBlock
->
pData
,
pDataBlock
->
size
);
pDataBlock
->
nAllocSize
=
(
uint32_t
)
pDataBlock
->
size
;
}
}
STableDataBlocks
*
tscCreateDataBlockEx
(
size_t
size
,
int32_t
rowSize
,
int32_t
startOffset
,
char
*
name
)
{
STableDataBlocks
*
dataBuf
=
tscCreateDataBlock
(
size
);
dataBuf
->
rowSize
=
rowSize
;
dataBuf
->
size
=
startOffset
;
strncpy
(
dataBuf
->
meterId
,
name
,
TSDB_METER_ID_LEN
);
return
dataBuf
;
}
STableDataBlocks
*
tscGetDataBlockFromList
(
void
*
pHashList
,
SDataBlockList
*
pDataBlockList
,
int64_t
id
,
int32_t
size
,
int32_t
startOffset
,
int32_t
rowSize
,
char
*
tableId
)
{
STableDataBlocks
*
dataBuf
=
NULL
;
STableDataBlocks
**
t1
=
(
STableDataBlocks
**
)
taosGetIntHashData
(
pHashList
,
id
);
if
(
t1
!=
NULL
)
{
dataBuf
=
*
t1
;
}
if
(
dataBuf
==
NULL
)
{
dataBuf
=
tscCreateDataBlockEx
((
size_t
)
size
,
rowSize
,
startOffset
,
tableId
);
dataBuf
=
*
(
STableDataBlocks
**
)
taosAddIntHash
(
pHashList
,
id
,
(
char
*
)
&
dataBuf
);
tscAppendDataBlock
(
pDataBlockList
,
dataBuf
);
}
return
dataBuf
;
}
void
tscMergeTableDataBlocks
(
SSqlCmd
*
pCmd
,
SDataBlockList
*
pTableDataBlockList
)
{
void
*
pVnodeDataBlockHashList
=
taosInitIntHash
(
8
,
sizeof
(
void
*
),
taosHashInt
);
SDataBlockList
*
pVnodeDataBlockList
=
tscCreateBlockArrayList
();
for
(
int32_t
i
=
0
;
i
<
pTableDataBlockList
->
nSize
;
++
i
)
{
STableDataBlocks
*
pOneTableBlock
=
pTableDataBlockList
->
pData
[
i
];
STableDataBlocks
*
dataBuf
=
tscGetDataBlockFromList
(
pVnodeDataBlockHashList
,
pVnodeDataBlockList
,
pOneTableBlock
->
vgid
,
TSDB_PAYLOAD_SIZE
,
tsInsertHeadSize
,
0
,
pOneTableBlock
->
meterId
);
int64_t
destSize
=
dataBuf
->
size
+
pOneTableBlock
->
size
;
if
(
dataBuf
->
nAllocSize
<
destSize
)
{
while
(
dataBuf
->
nAllocSize
<
destSize
)
{
dataBuf
->
nAllocSize
=
dataBuf
->
nAllocSize
*
1
.
5
;
}
char
*
tmp
=
realloc
(
dataBuf
->
pData
,
dataBuf
->
nAllocSize
);
if
(
tmp
!=
NULL
)
{
dataBuf
->
pData
=
tmp
;
memset
(
dataBuf
->
pData
+
dataBuf
->
size
,
0
,
dataBuf
->
nAllocSize
-
dataBuf
->
size
);
}
else
{
// to do handle error
}
}
SShellSubmitBlock
*
pBlocks
=
(
SShellSubmitBlock
*
)
pOneTableBlock
->
pData
;
assert
(
pBlocks
->
numOfRows
*
pOneTableBlock
->
rowSize
+
sizeof
(
SShellSubmitBlock
)
==
pOneTableBlock
->
size
);
pBlocks
->
numOfRows
=
(
int16_t
)
sortRemoveDuplicates
(
pOneTableBlock
,
pBlocks
->
numOfRows
);
pBlocks
->
sid
=
htonl
(
pBlocks
->
sid
);
pBlocks
->
uid
=
htobe64
(
pBlocks
->
uid
);
pBlocks
->
sversion
=
htonl
(
pBlocks
->
sversion
);
pBlocks
->
numOfRows
=
htons
(
pBlocks
->
numOfRows
);
memcpy
(
dataBuf
->
pData
+
dataBuf
->
size
,
pOneTableBlock
->
pData
,
pOneTableBlock
->
size
);
dataBuf
->
size
+=
pOneTableBlock
->
size
;
dataBuf
->
numOfMeters
+=
1
;
}
tscDestroyBlockArrayList
(
pTableDataBlockList
);
// free the table data blocks;
pCmd
->
pDataBlocks
=
pVnodeDataBlockList
;
tscFreeUnusedDataBlocks
(
pCmd
->
pDataBlocks
);
taosCleanUpIntHash
(
pVnodeDataBlockHashList
);
}
void
tscCloseTscObj
(
STscObj
*
pObj
)
{
pObj
->
signature
=
NULL
;
SSqlObj
*
pSql
=
pObj
->
pSql
;
...
...
@@ -821,15 +902,18 @@ int32_t tscValidateName(SSQLToken* pToken) {
pToken
->
n
=
strdequote
(
pToken
->
z
);
strtrim
(
pToken
->
z
);
pToken
->
n
=
(
uint32_t
)
strlen
(
pToken
->
z
);
int
len
=
tSQLGetToken
(
pToken
->
z
,
&
pToken
->
type
);
int
len
=
tSQLGetToken
(
pToken
->
z
,
&
pToken
->
type
);
// single token, validate it
if
(
len
==
pToken
->
n
){
return
validateQuoteToken
(
pToken
);
}
else
{
}
else
{
sep
=
strnchrNoquote
(
pToken
->
z
,
TS_PATH_DELIMITER
[
0
],
pToken
->
n
);
if
(
sep
==
NULL
)
{
return
TSDB_CODE_INVALID_SQL
;
}
return
tscValidateName
(
pToken
);
}
}
else
{
...
...
@@ -965,8 +1049,7 @@ void tscSetFreeHeatBeat(STscObj* pObj) {
SSqlObj
*
pHeatBeat
=
pObj
->
pHb
;
assert
(
pHeatBeat
==
pHeatBeat
->
signature
);
pHeatBeat
->
cmd
.
type
=
1
;
// to denote the heart-beat timer close connection
// and free all allocated resources
pHeatBeat
->
cmd
.
type
=
1
;
// to denote the heart-beat timer close connection and free all allocated resources
}
bool
tscShouldFreeHeatBeat
(
SSqlObj
*
pHb
)
{
...
...
@@ -1052,7 +1135,6 @@ void tscDoQuery(SSqlObj* pSql) {
if
(
pCmd
->
command
>
TSDB_SQL_LOCAL
)
{
tscProcessLocalCmd
(
pSql
);
}
else
{
// add to sql list, so that the show queries could get the query info
if
(
pCmd
->
command
==
TSDB_SQL_SELECT
)
{
tscAddIntoSqlList
(
pSql
);
}
...
...
@@ -1061,18 +1143,19 @@ void tscDoQuery(SSqlObj* pSql) {
pSql
->
cmd
.
vnodeIdx
+=
1
;
}
if
(
pSql
->
fp
==
NULL
)
{
if
(
0
==
pCmd
->
isInsertFromFile
)
{
tscProcessSql
(
pSql
);
tscProcessMultiVnodesInsert
(
pSql
);
// handle the multi-vnode insertion
}
else
if
(
1
==
pCmd
->
isInsertFromFile
)
{
tscProcessMultiVnodesInsertForFile
(
pSql
);
}
else
{
assert
(
false
);
}
void
*
fp
=
pSql
->
fp
;
if
(
pCmd
->
isInsertFromFile
==
1
)
{
tscProcessMultiVnodesInsertForFile
(
pSql
);
}
else
{
// pSql may be released in this function if it is a async insertion.
tscProcessSql
(
pSql
);
}
// handle the multi-vnode insertion for sync model
if
(
fp
==
NULL
)
{
assert
(
pSql
->
signature
==
pSql
);
tscProcessMultiVnodesInsert
(
pSql
);
}
}
}
}
src/inc/textbuffer.h
浏览文件 @
fe6672d7
...
...
@@ -184,7 +184,7 @@ void tColModelDisplayEx(tColModel *pModel, void *pData, int32_t numOfRows, int32
/*
* compress data into consecutive block without hole in data
*/
void
tColModelComp
ress
(
tColModel
*
pModel
,
tFilePage
*
inputBuffer
,
int32_t
maxElemsCapacity
);
void
tColModelComp
act
(
tColModel
*
pModel
,
tFilePage
*
inputBuffer
,
int32_t
maxElemsCapacity
);
void
tColModelErase
(
tColModel
*
pModel
,
tFilePage
*
inputBuffer
,
int32_t
maxCapacity
,
int32_t
s
,
int32_t
e
);
...
...
src/inc/tinterpolation.h
浏览文件 @
fe6672d7
...
...
@@ -69,7 +69,7 @@ int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo *pInterpoInfo, int64_t *p
* @param pInterpoInfo
* @return
*/
bool
taosHas
NoneInterpoPoints
(
SInterpolationInfo
*
pInterpoInfo
);
bool
taosHas
RemainsDataForInterpolation
(
SInterpolationInfo
*
pInterpoInfo
);
int32_t
taosNumOfRemainPoints
(
SInterpolationInfo
*
pInterpoInfo
);
...
...
src/system/inc/vnode.h
浏览文件 @
fe6672d7
...
...
@@ -69,11 +69,12 @@ enum _sync_cmd {
};
enum
_meter_state
{
TSDB_METER_STATE_READY
,
TSDB_METER_STATE_IMPORTING
,
TSDB_METER_STATE_UPDATING
,
TSDB_METER_STATE_DELETING
,
TSDB_METER_STATE_DELETED
,
TSDB_METER_STATE_READY
=
0x00
,
TSDB_METER_STATE_INSERT
=
0x01
,
TSDB_METER_STATE_IMPORTING
=
0x02
,
TSDB_METER_STATE_UPDATING
=
0x04
,
TSDB_METER_STATE_DELETING
=
0x10
,
TSDB_METER_STATE_DELETED
=
0x18
,
};
typedef
struct
{
...
...
@@ -184,10 +185,10 @@ typedef struct _meter_obj {
short
sqlLen
;
char
searchAlgorithm
:
4
;
char
compAlgorithm
:
4
;
char
state
:
5
;
// deleted or added, 1: added
char
status
:
3
;
// 0: ok, 1: stop stream computing
char
status
;
// 0: ok, 1: stop stream computing
char
reserved
[
16
];
int
state
;
int
numOfQueries
;
char
*
pSql
;
void
*
pStream
;
...
...
@@ -499,7 +500,7 @@ int vnodeInitStore();
void
vnodeCleanUpVnodes
();
void
vnodeRemoveVnode
(
int
vnode
);
int
vnodeRemoveVnode
(
int
vnode
);
int
vnodeCreateVnode
(
int
vnode
,
SVnodeCfg
*
pCfg
,
SVPeerDesc
*
pDesc
);
...
...
src/system/inc/vnodeUtil.h
浏览文件 @
fe6672d7
...
...
@@ -75,6 +75,12 @@ int32_t vnodeIncQueryRefCount(SQueryMeterMsg *pQueryMsg, SMeterSidExtInfo **pSid
void
vnodeDecQueryRefCount
(
SQueryMeterMsg
*
pQueryMsg
,
SMeterObj
**
pMeterObjList
,
int32_t
numOfInc
);
int32_t
vnodeTransferMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
);
void
vnodeClearMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
);
bool
vnodeIsMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
);
void
vnodeSetMeterDeleting
(
SMeterObj
*
pMeterObj
);
bool
vnodeIsSafeToDeleteMeter
(
SVnodeObj
*
pVnode
,
int32_t
sid
);
#ifdef __cplusplus
}
#endif
...
...
src/system/src/dnodeMgmt.c
浏览文件 @
fe6672d7
...
...
@@ -445,7 +445,8 @@ int vnodeProcessFreeVnodeRequest(char *pMsg) {
}
dTrace
(
"vid:%d receive free vnode message"
,
pFree
->
vnode
);
vnodeRemoveVnode
(
pFree
->
vnode
);
int32_t
code
=
vnodeRemoveVnode
(
pFree
->
vnode
);
assert
(
code
==
TSDB_CODE_SUCCESS
||
code
==
TSDB_CODE_ACTION_IN_PROGRESS
);
pStart
=
(
char
*
)
malloc
(
128
);
if
(
pStart
==
NULL
)
return
0
;
...
...
@@ -453,7 +454,7 @@ int vnodeProcessFreeVnodeRequest(char *pMsg) {
*
pStart
=
TSDB_MSG_TYPE_FREE_VNODE_RSP
;
pMsg
=
pStart
+
1
;
*
pMsg
=
0
;
*
pMsg
=
code
;
vnodeSendMsgToMgmt
(
pStart
);
return
0
;
...
...
src/system/src/mgmtMeter.c
浏览文件 @
fe6672d7
...
...
@@ -1140,54 +1140,13 @@ static void mgmtReorganizeMetersInMetricMeta(STabObj *pMetric, SMetricMetaMsg *p
startPos
[
1
]
=
(
int32_t
)
pRes
->
num
;
}
/* if pInfo->limit == 0, the query will be intercepted by sdk, and wont be
* sent to mnode */
assert
(
pInfo
->
limit
==
-
1
||
pInfo
->
limit
>
0
);
int32_t
numOfTotal
=
0
;
if
(
pInfo
->
offset
>=
numOfSubset
)
{
numOfTotal
=
0
;
}
else
if
(
numOfSubset
==
1
)
{
// no 'groupBy' clause, all tables returned
numOfTotal
=
pRes
->
num
;
}
else
{
/* there is a offset value of group */
int32_t
start
=
0
;
int32_t
end
=
0
;
if
(
pInfo
->
orderType
==
TSQL_SO_ASC
)
{
start
=
startPos
[
pInfo
->
offset
];
if
(
pInfo
->
limit
+
pInfo
->
offset
>=
numOfSubset
||
pInfo
->
limit
==
-
1
)
{
/* all results are required */
end
=
startPos
[
numOfSubset
];
}
else
{
end
=
startPos
[
pInfo
->
limit
+
pInfo
->
offset
];
}
}
else
{
end
=
startPos
[
numOfSubset
-
pInfo
->
offset
];
if
(
pInfo
->
limit
+
pInfo
->
offset
>=
numOfSubset
||
pInfo
->
limit
==
-
1
)
{
start
=
startPos
[
0
];
}
else
{
start
=
startPos
[
numOfSubset
-
pInfo
->
limit
-
pInfo
->
offset
];
}
}
numOfTotal
=
end
-
start
;
assert
(
numOfTotal
>
0
);
memmove
(
pRes
->
pRes
,
pRes
->
pRes
+
start
,
numOfTotal
*
POINTER_BYTES
);
}
/*
* sort the result according to vgid to ensure meters with the same vgid is
* continuous in the result list
*/
__compar_fn_t
functor
=
(
pRes
->
nodeType
==
TAST_NODE_TYPE_METER_PTR
)
?
tabObjVGIDComparator
:
nodeVGIDComparator
;
qsort
(
pRes
->
pRes
,
numOfTotal
,
POINTER_BYTES
,
functor
);
qsort
(
pRes
->
pRes
,
(
size_t
)
pRes
->
num
,
POINTER_BYTES
,
functor
);
pRes
->
num
=
numOfTotal
;
free
(
descriptor
->
pTagSchema
);
free
(
descriptor
);
free
(
startPos
);
...
...
src/system/src/vnodeCache.c
浏览文件 @
fe6672d7
...
...
@@ -340,19 +340,33 @@ void vnodeCommitOver(SVnodeObj *pVnode) {
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
}
void
vnodeCancelCommit
(
SVnodeObj
*
pVnode
)
{
static
void
vnodeWaitForCommitComplete
(
SVnodeObj
*
pVnode
)
{
SCachePool
*
pPool
=
(
SCachePool
*
)(
pVnode
->
pCachePool
);
if
(
pPool
==
NULL
)
return
;
pthread_mutex_lock
(
&
pPool
->
vmutex
);
// wait for 100s at most
const
int32_t
totalCount
=
1000
;
int32_t
count
=
0
;
if
(
pPool
->
commitInProcess
)
{
pPool
->
commitInProcess
=
0
;
pthread_cancel
(
pVnode
->
commitThread
);
// all meter is marked as dropped, so the commit will abort very quickly
while
(
count
++
<
totalCount
)
{
int32_t
commitInProcess
=
0
;
pthread_mutex_lock
(
&
pPool
->
vmutex
);
commitInProcess
=
pPool
->
commitInProcess
;
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
if
(
commitInProcess
)
{
dWarn
(
"vid:%d still in commit, wait for completed"
,
pVnode
->
vnode
);
taosMsleep
(
10
);
}
}
}
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
void
vnodeCancelCommit
(
SVnodeObj
*
pVnode
)
{
SCachePool
*
pPool
=
(
SCachePool
*
)(
pVnode
->
pCachePool
);
if
(
pPool
==
NULL
)
return
;
vnodeWaitForCommitComplete
(
pVnode
);
taosTmrReset
(
vnodeProcessCommitTimer
,
pVnode
->
cfg
.
commitTime
*
1000
,
pVnode
,
vnodeTmrCtrl
,
&
pVnode
->
commitTimer
);
}
...
...
src/system/src/vnodeCommit.c
浏览文件 @
fe6672d7
...
...
@@ -26,6 +26,7 @@
#include "tsdb.h"
#include "vnode.h"
#include "vnodeUtil.h"
typedef
struct
{
int
sversion
;
...
...
@@ -160,13 +161,17 @@ size_t vnodeRestoreDataFromLog(int vnode, char *fileName, uint64_t *firstV) {
if
(
*
(
int
*
)(
cont
+
head
.
contLen
)
!=
simpleCheck
)
break
;
SMeterObj
*
pObj
=
pVnode
->
meterList
[
head
.
sid
];
if
(
pObj
==
NULL
)
{
dError
(
"vid:%d, sid:%d not exists, ignore data in commit log, "
"contLen:%d action:%d"
,
dError
(
"vid:%d, sid:%d not exists, ignore data in commit log, contLen:%d action:%d"
,
vnode
,
head
.
sid
,
head
.
contLen
,
head
.
action
);
continue
;
}
if
(
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_DELETING
))
{
dWarn
(
"vid:%d sid:%d id:%s, meter is dropped, ignore data in commit log, contLen:%d action:%d"
,
vnode
,
head
.
sid
,
head
.
contLen
,
head
.
action
);
continue
;
}
int32_t
numOfPoints
=
0
;
(
*
vnodeProcessAction
[
head
.
action
])(
pObj
,
cont
,
head
.
contLen
,
TSDB_DATA_SOURCE_LOG
,
NULL
,
head
.
sversion
,
&
numOfPoints
);
...
...
src/system/src/vnodeFile.c
浏览文件 @
fe6672d7
...
...
@@ -577,8 +577,20 @@ _again:
// read compInfo
for
(
sid
=
0
;
sid
<
pCfg
->
maxSessions
;
++
sid
)
{
if
(
pVnode
->
meterList
==
NULL
)
{
// vnode is being freed, abort
goto
_over
;
}
pObj
=
(
SMeterObj
*
)(
pVnode
->
meterList
[
sid
]);
if
(
pObj
==
NULL
)
continue
;
if
(
pObj
==
NULL
)
{
continue
;
}
// meter is going to be deleted, abort
if
(
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_DELETING
))
{
dWarn
(
"vid:%d sid:%d is dropped, ignore this meter"
,
vnode
,
sid
);
continue
;
}
pMeter
=
meterInfo
+
sid
;
pHeader
=
((
SCompHeader
*
)
tmem
)
+
sid
;
...
...
@@ -672,8 +684,9 @@ _again:
pointsReadLast
=
pMeter
->
lastBlock
.
numOfPoints
;
query
.
over
=
0
;
headInfo
.
totalStorage
-=
(
pointsReadLast
*
pObj
->
bytesPerPoint
);
dTrace
(
"vid:%d sid:%d id:%s, points:%d in last block will be merged to new block"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pointsReadLast
);
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pointsReadLast
);
}
pMeter
->
changed
=
1
;
...
...
@@ -717,8 +730,8 @@ _again:
}
dTrace
(
"vid:%d sid:%d id:%s, %d points are committed, lastKey:%lld slot:%d pos:%d newNumOfBlocks:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pMeter
->
committedPoints
,
pObj
->
lastKeyOnFile
,
query
.
slot
,
query
.
pos
,
pMeter
->
newNumOfBlocks
);
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pMeter
->
committedPoints
,
pObj
->
lastKeyOnFile
,
query
.
slot
,
query
.
pos
,
pMeter
->
newNumOfBlocks
);
if
(
pMeter
->
committedPoints
>
0
)
{
pMeter
->
commitSlot
=
query
.
slot
;
...
...
src/system/src/vnodeImport.c
浏览文件 @
fe6672d7
...
...
@@ -24,6 +24,7 @@
#include "vnode.h"
#include "vnodeMgmt.h"
#include "vnodeShell.h"
#include "vnodeShell.h"
#include "vnodeUtil.h"
#pragma GCC diagnostic ignored "-Wpointer-sign"
#pragma GCC diagnostic ignored "-Wint-conversion"
...
...
@@ -281,14 +282,32 @@ void vnodeProcessImportTimer(void *param, void *tmrId) {
SShellObj
*
pShell
=
pImport
->
pShell
;
pImport
->
retry
++
;
pObj
->
state
=
TSDB_METER_STATE_IMPORTING
;
//slow query will block the import operation
int32_t
state
=
vnodeTransferMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
if
(
state
>=
TSDB_METER_STATE_DELETING
)
{
dError
(
"vid:%d sid:%d id:%s, meter is deleted, failed to import, state:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
state
);
return
;
}
int32_t
num
=
0
;
pthread_mutex_lock
(
&
pVnode
->
vmutex
);
num
=
pObj
->
numOfQueries
;
pthread_mutex_unlock
(
&
pVnode
->
vmutex
);
//if the num == 0, it will never be increased before state is set to TSDB_METER_STATE_READY
int32_t
commitInProcess
=
0
;
pthread_mutex_lock
(
&
pPool
->
vmutex
);
if
(
pPool
->
commitInProcess
||
pObj
->
numOfQueries
>
0
)
{
if
(
((
commitInProcess
=
pPool
->
commitInProcess
)
==
1
)
||
num
>
0
||
state
!=
TSDB_METER_STATE_READY
)
{
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
if
(
pImport
->
retry
<
1000
)
{
dTrace
(
"vid:%d sid:%d id:%s, commit in process, try to import later"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
);
dTrace
(
"vid:%d sid:%d id:%s, import failed, retry later. commit in process or queries on it, or not ready."
"commitInProcess:%d, numOfQueries:%d, state:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
commitInProcess
,
num
,
state
);
taosTmrStart
(
vnodeProcessImportTimer
,
10
,
pImport
,
vnodeTmrCtrl
);
return
;
}
else
{
...
...
@@ -304,7 +323,8 @@ void vnodeProcessImportTimer(void *param, void *tmrId) {
}
}
pObj
->
state
=
TSDB_METER_STATE_READY
;
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
pVnode
->
version
++
;
// send response back to shell
...
...
@@ -862,15 +882,19 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
}
if
(
*
((
TSKEY
*
)(
pSubmit
->
payLoad
+
(
rows
-
1
)
*
pObj
->
bytesPerPoint
))
>
pObj
->
lastKey
)
{
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
vnodeTransferMeterState
(
pObj
,
TSDB_METER_STATE_INSERT
);
code
=
vnodeInsertPoints
(
pObj
,
cont
,
contLen
,
TSDB_DATA_SOURCE_LOG
,
NULL
,
pObj
->
sversion
,
&
pointsImported
);
if
(
pShell
)
{
pShell
->
code
=
code
;
pShell
->
numOfTotalPoints
+=
pointsImported
;
}
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_INSERT
);
}
else
{
SImportInfo
*
pNew
,
import
;
pObj
->
state
=
TSDB_METER_STATE_IMPORTING
;
dTrace
(
"vid:%d sid:%d id:%s, import %d rows data"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
rows
);
memset
(
&
import
,
0
,
sizeof
(
import
));
import
.
firstKey
=
*
((
TSKEY
*
)(
payload
));
...
...
@@ -880,10 +904,19 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
import
.
payload
=
payload
;
import
.
rows
=
rows
;
int32_t
num
=
0
;
pthread_mutex_lock
(
&
pVnode
->
vmutex
);
num
=
pObj
->
numOfQueries
;
pthread_mutex_unlock
(
&
pVnode
->
vmutex
);
int32_t
commitInProcess
=
0
;
pthread_mutex_lock
(
&
pPool
->
vmutex
);
if
(
pPool
->
commitInProcess
||
pObj
->
numOfQueries
>
0
)
{
if
(
((
commitInProcess
=
pPool
->
commitInProcess
)
==
1
)
||
num
>
0
)
{
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
//restore meter state
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
pNew
=
(
SImportInfo
*
)
malloc
(
sizeof
(
SImportInfo
));
memcpy
(
pNew
,
&
import
,
sizeof
(
SImportInfo
));
...
...
@@ -892,8 +925,9 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
pNew
->
payload
=
malloc
(
payloadLen
);
memcpy
(
pNew
->
payload
,
payload
,
payloadLen
);
dTrace
(
"vid:%d sid:%d id:%s, commit/query:%d in process, import later, "
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pObj
->
numOfQueries
);
dTrace
(
"vid:%d sid:%d id:%s, import later, commit in process:%d, numOfQueries:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
commitInProcess
,
pObj
->
numOfQueries
);
taosTmrStart
(
vnodeProcessImportTimer
,
10
,
pNew
,
vnodeTmrCtrl
);
return
0
;
}
else
{
...
...
@@ -905,9 +939,10 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
pShell
->
numOfTotalPoints
+=
import
.
importedRows
;
}
}
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_IMPORTING
);
}
pObj
->
state
=
TSDB_METER_STATE_READY
;
pVnode
->
version
++
;
if
(
pShell
)
{
...
...
@@ -918,6 +953,7 @@ int vnodeImportPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
return
0
;
}
//todo abort from the procedure if the meter is going to be dropped
int
vnodeImportData
(
SMeterObj
*
pObj
,
SImportInfo
*
pImport
)
{
int
code
=
0
;
...
...
src/system/src/vnodeMeter.c
浏览文件 @
fe6672d7
...
...
@@ -47,6 +47,8 @@ void vnodeFreeMeterObj(SMeterObj *pObj) {
if
(
vnodeList
[
pObj
->
vnode
].
meterList
!=
NULL
)
{
vnodeList
[
pObj
->
vnode
].
meterList
[
pObj
->
sid
]
=
NULL
;
}
memset
(
pObj
->
meterId
,
0
,
tListLen
(
pObj
->
meterId
));
tfree
(
pObj
);
}
...
...
@@ -143,7 +145,7 @@ int vnodeSaveMeterObjToFile(SMeterObj *pObj) {
memcpy
(
buffer
,
pObj
,
offsetof
(
SMeterObj
,
reserved
));
memcpy
(
buffer
+
offsetof
(
SMeterObj
,
reserved
),
pObj
->
schema
,
pObj
->
numOfColumns
*
sizeof
(
SColumn
));
memcpy
(
buffer
+
offsetof
(
SMeterObj
,
reserved
)
+
pObj
->
numOfColumns
*
sizeof
(
SColumn
),
pObj
->
pSql
,
pObj
->
sqlLen
);
taosCalcChecksumAppend
(
0
,
buffer
,
new_length
);
taosCalcChecksumAppend
(
0
,
(
uint8_t
*
)
buffer
,
new_length
);
if
(
offset
==
0
||
length
<
new_length
)
{
// New, append to file end
fseek
(
fp
,
0
,
SEEK_END
);
...
...
@@ -208,7 +210,7 @@ int vnodeSaveAllMeterObjToFile(int vnode) {
memcpy
(
buffer
,
pObj
,
offsetof
(
SMeterObj
,
reserved
));
memcpy
(
buffer
+
offsetof
(
SMeterObj
,
reserved
),
pObj
->
schema
,
pObj
->
numOfColumns
*
sizeof
(
SColumn
));
memcpy
(
buffer
+
offsetof
(
SMeterObj
,
reserved
)
+
pObj
->
numOfColumns
*
sizeof
(
SColumn
),
pObj
->
pSql
,
pObj
->
sqlLen
);
taosCalcChecksumAppend
(
0
,
buffer
,
new_length
);
taosCalcChecksumAppend
(
0
,
(
uint8_t
*
)
buffer
,
new_length
);
if
(
offset
==
0
||
length
>
new_length
)
{
// New, append to file end
new_offset
=
fseek
(
fp
,
0
,
SEEK_END
);
...
...
@@ -391,7 +393,7 @@ int vnodeOpenMetersVnode(int vnode) {
fseek
(
fp
,
offset
,
SEEK_SET
);
if
(
fread
(
buffer
,
length
,
1
,
fp
)
<=
0
)
break
;
if
(
taosCheckChecksumWhole
(
buffer
,
length
))
{
if
(
taosCheckChecksumWhole
(
(
uint8_t
*
)
buffer
,
length
))
{
vnodeRestoreMeterObj
(
buffer
,
length
-
sizeof
(
TSCKSUM
));
}
else
{
dError
(
"meter object file is broken since checksum mismatch, vnode: %d sid: %d, try to recover"
,
vnode
,
sid
);
...
...
@@ -440,7 +442,7 @@ int vnodeCreateMeterObj(SMeterObj *pNew, SConnSec *pSec) {
}
dTrace
(
"vid:%d sid:%d id:%s, update schema"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
if
(
pObj
->
state
!=
TSDB_METER_STATE_UPDATING
)
vnodeUpdateMeter
(
pNew
,
NULL
);
if
(
!
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
)
)
vnodeUpdateMeter
(
pNew
,
NULL
);
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -483,27 +485,20 @@ int vnodeRemoveMeterObj(int vnode, int sid) {
if
(
vnodeList
[
vnode
].
meterList
==
NULL
)
return
0
;
pObj
=
vnodeList
[
vnode
].
meterList
[
sid
];
if
((
pObj
==
NULL
)
||
(
pObj
->
state
==
TSDB_METER_STATE_DELETED
))
return
0
;
if
(
pObj
->
state
==
TSDB_METER_STATE_IMPORTING
)
return
TSDB_CODE_ACTION_IN_PROGRESS
;
if
(
pObj
==
NULL
)
{
return
TSDB_CODE_SUCCESS
;
}
int32_t
retFlag
=
0
;
pthread_mutex_lock
(
&
vnodeList
[
vnode
].
vmutex
);
pObj
->
state
=
TSDB_METER_STATE_DELETING
;
if
(
pObj
->
numOfQueries
>
0
)
{
retFlag
=
TSDB_CODE_ACTION_IN_PROGRESS
;
dWarn
(
"vid:%d sid:%d id:%s %d queries executing on it, wait query to be finished"
,
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pObj
->
numOfQueries
);
if
(
!
vnodeIsSafeToDeleteMeter
(
&
vnodeList
[
vnode
],
sid
))
{
return
TSDB_CODE_ACTION_IN_PROGRESS
;
}
pthread_mutex_unlock
(
&
vnodeList
[
vnode
].
vmutex
);
if
(
retFlag
!=
0
)
return
retFlag
;
// after remove this meter, change its stat to DELETED
// after remove this meter, change its stat
e
to DELETED
pObj
->
state
=
TSDB_METER_STATE_DELETED
;
pObj
->
timeStamp
=
taosGetTimestampMs
();
vnodeList
[
vnode
].
lastRemove
=
pObj
->
timeStamp
;
vnodeRemoveStream
(
pObj
);
pObj
->
meterId
[
0
]
=
0
;
vnodeSaveMeterObjToFile
(
pObj
);
vnodeFreeMeterObj
(
pObj
);
...
...
@@ -578,10 +573,19 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
if
(
pVnode
->
lastKeyOnFile
>
pVnode
->
cfg
.
daysToKeep
*
tsMsPerDay
[
pVnode
->
cfg
.
precision
]
+
firstKey
)
{
dError
(
"vid:%d sid:%d id:%s, vnode lastKeyOnFile:%lld, data is too old to insert, key:%lld"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pVnode
->
lastKeyOnFile
,
firstKey
);
return
TSDB_CODE_
OTHERS
;
return
TSDB_CODE_
TIMESTAMP_OUT_OF_RANGE
;
}
for
(
i
=
0
;
i
<
numOfPoints
;
++
i
)
{
// meter will be dropped, abort current insertion
if
(
pObj
->
state
>=
TSDB_METER_STATE_DELETING
)
{
dWarn
(
"vid:%d sid:%d id:%s, meter is dropped, abort insert, state:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pObj
->
state
);
code
=
TSDB_CODE_INVALID_SESSION_ID
;
break
;
}
if
(
*
((
TSKEY
*
)
pData
)
<=
pObj
->
lastKey
)
{
dWarn
(
"vid:%d sid:%d id:%s, received key:%ld not larger than lastKey:%ld"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
*
((
TSKEY
*
)
pData
),
pObj
->
lastKey
);
...
...
@@ -632,9 +636,11 @@ void vnodeProcessUpdateSchemaTimer(void *param, void *tmrId) {
pthread_mutex_lock
(
&
pPool
->
vmutex
);
if
(
pPool
->
commitInProcess
)
{
dTrace
(
"vid:%d sid:%d mid:%s, commiting in process, commit later"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
);
if
(
taosTmrStart
(
vnodeProcessUpdateSchemaTimer
,
10
,
pObj
,
vnodeTmrCtrl
)
==
NULL
)
pObj
->
state
=
TSDB_METER_STATE_READY
;
dTrace
(
"vid:%d sid:%d mid:%s, committing in process, commit later"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
);
if
(
taosTmrStart
(
vnodeProcessUpdateSchemaTimer
,
10
,
pObj
,
vnodeTmrCtrl
)
==
NULL
)
{
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
);
}
pthread_mutex_unlock
(
&
pPool
->
vmutex
);
return
;
}
...
...
@@ -649,41 +655,54 @@ void vnodeUpdateMeter(void *param, void *tmrId) {
SMeterObj
*
pNew
=
(
SMeterObj
*
)
param
;
if
(
pNew
==
NULL
||
pNew
->
vnode
<
0
||
pNew
->
sid
<
0
)
return
;
if
(
vnodeList
[
pNew
->
vnode
].
meterList
==
NULL
)
{
SVnodeObj
*
pVnode
=
&
vnodeList
[
pNew
->
vnode
];
if
(
pVnode
->
meterList
==
NULL
)
{
dTrace
(
"vid:%d sid:%d id:%s, vnode is deleted, abort update schema"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
free
(
pNew
->
schema
);
free
(
pNew
);
return
;
}
SMeterObj
*
pObj
=
vnodeList
[
pNew
->
vnode
].
meterList
[
pNew
->
sid
];
if
(
pObj
==
NULL
)
{
SMeterObj
*
pObj
=
pVnode
->
meterList
[
pNew
->
sid
];
if
(
pObj
==
NULL
||
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_DELETING
)
)
{
dTrace
(
"vid:%d sid:%d id:%s, meter is deleted, abort update schema"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
free
(
pNew
->
schema
);
free
(
pNew
);
return
;
}
pObj
->
state
=
TSDB_METER_STATE_UPDATING
;
int32_t
state
=
vnodeTransferMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
);
if
(
state
>=
TSDB_METER_STATE_DELETING
)
{
dError
(
"vid:%d sid:%d id:%s, meter is deleted, failed to update, state:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
state
);
return
;
}
int32_t
num
=
0
;
pthread_mutex_lock
(
&
pVnode
->
vmutex
);
num
=
pObj
->
numOfQueries
;
pthread_mutex_unlock
(
&
pVnode
->
vmutex
);
if
(
num
>
0
||
state
!=
TSDB_METER_STATE_READY
)
{
dTrace
(
"vid:%d sid:%d id:%s, update failed, retry later, numOfQueries:%d, state:%d"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
,
num
,
state
);
if
(
pObj
->
numOfQueries
>
0
)
{
// retry update meter in 50ms
if
(
taosTmrStart
(
vnodeUpdateMeter
,
50
,
pNew
,
vnodeTmrCtrl
)
==
NULL
)
{
dError
(
"vid:%d sid:%d id:%s, failed to start update timer"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
dError
(
"vid:%d sid:%d id:%s, failed to start update timer, no retry"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
free
(
pNew
->
schema
);
free
(
pNew
);
}
dTrace
(
"vid:%d sid:%d id:%s, there are ongoing queries, update later"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
return
;
}
// commit first
if
(
!
vnodeIsCacheCommitted
(
pObj
))
{
// commit
// commit
data first
if
(
taosTmrStart
(
vnodeProcessUpdateSchemaTimer
,
0
,
pObj
,
vnodeTmrCtrl
)
==
NULL
)
{
dError
(
"vid:%d sid:%d id:%s, failed to start commit timer"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
)
;
free
(
pNew
->
schema
);
free
(
pNew
);
return
;
...
...
@@ -691,13 +710,14 @@ void vnodeUpdateMeter(void *param, void *tmrId) {
if
(
taosTmrStart
(
vnodeUpdateMeter
,
50
,
pNew
,
vnodeTmrCtrl
)
==
NULL
)
{
dError
(
"vid:%d sid:%d id:%s, failed to start update timer"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
)
;
free
(
pNew
->
schema
);
free
(
pNew
);
}
dTrace
(
"vid:%d sid:%d meterId:%s, there are data in cache, commit first, update later"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
);
return
;
}
...
...
@@ -716,7 +736,7 @@ void vnodeUpdateMeter(void *param, void *tmrId) {
pObj
->
sversion
=
pNew
->
sversion
;
vnodeSaveMeterObjToFile
(
pObj
);
pObj
->
state
=
TSDB_METER_STATE_READY
;
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_UPDATING
)
;
dTrace
(
"vid:%d sid:%d id:%s, schema is updated"
,
pNew
->
vnode
,
pNew
->
sid
,
pNew
->
meterId
);
free
(
pNew
);
...
...
src/system/src/vnodeQueryImpl.c
浏览文件 @
fe6672d7
...
...
@@ -1730,6 +1730,17 @@ static int64_t getOldestKey(int32_t numOfFiles, int64_t fileId, SVnodeCfg *pCfg)
bool
isQueryKilled
(
SQuery
*
pQuery
)
{
SQInfo
*
pQInfo
=
(
SQInfo
*
)
GET_QINFO_ADDR
(
pQuery
);
/*
* check if the queried meter is going to be deleted.
* if it will be deleted soon, stop current query ASAP.
*/
SMeterObj
*
pMeterObj
=
pQInfo
->
pObj
;
if
(
vnodeIsMeterState
(
pMeterObj
,
TSDB_METER_STATE_DELETING
))
{
pQInfo
->
killed
=
1
;
return
true
;
}
return
(
pQInfo
->
killed
==
1
);
}
...
...
src/system/src/vnodeShell.c
浏览文件 @
fe6672d7
...
...
@@ -15,12 +15,13 @@
#define _DEFAULT_SOURCE
#include "vnodeShell.h"
#include <arpa/inet.h>
#include <assert.h>
#include <endian.h>
#include <stdint.h>
#include "taosmsg.h"
#include "vnode.h"
#include "vnodeShell.h"
#include "tschemautil.h"
#include "textbuffer.h"
...
...
@@ -28,6 +29,7 @@
#include "vnode.h"
#include "vnodeRead.h"
#include "vnodeUtil.h"
#pragma GCC diagnostic ignored "-Wint-conversion"
void
*
pShellServer
=
NULL
;
...
...
@@ -87,6 +89,7 @@ void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) {
dTrace
(
"vid:%d sid:%d, msg:%s is received pConn:%p"
,
vnode
,
sid
,
taosMsg
[
pMsg
->
msgType
],
thandle
);
// set in query processing flag
if
(
pMsg
->
msgType
==
TSDB_MSG_TYPE_QUERY
)
{
vnodeProcessQueryRequest
((
char
*
)
pMsg
->
content
,
pMsg
->
msgLen
-
sizeof
(
SIntMsg
),
pObj
);
}
else
if
(
pMsg
->
msgType
==
TSDB_MSG_TYPE_RETRIEVE
)
{
...
...
@@ -96,7 +99,7 @@ void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) {
}
else
{
dError
(
"%s is not processed"
,
taosMsg
[
pMsg
->
msgType
]);
}
return
pObj
;
}
...
...
@@ -157,16 +160,30 @@ int vnodeOpenShellVnode(int vnode) {
return
0
;
}
void
vnodeCloseShellVnode
(
int
vnode
)
{
taosCloseRpcChann
(
pShellServer
,
vnode
);
void
vnodeDelayedFreeResource
(
void
*
param
,
void
*
tmrId
)
{
int32_t
vnode
=
*
(
int32_t
*
)
param
;
taosCloseRpcChann
(
pShellServer
,
vnode
);
// close connection
tfree
(
shellList
[
vnode
]);
//free SShellObj
tfree
(
param
);
}
void
vnodeCloseShellVnode
(
int
vnode
)
{
if
(
shellList
[
vnode
]
==
NULL
)
return
;
for
(
int
i
=
0
;
i
<
vnodeList
[
vnode
].
cfg
.
maxSessions
;
++
i
)
{
vnodeFreeQInfo
(
shellList
[
vnode
][
i
].
qhandle
,
true
);
}
tfree
(
shellList
[
vnode
]);
int32_t
*
v
=
malloc
(
sizeof
(
int32_t
));
*
v
=
vnode
;
/*
* free the connection related resource after 5sec, since the msg may be in
* the task queue, free it immediate will cause crash
*/
dTrace
(
"vid:%d, delay 5sec to free resources"
,
vnode
);
taosTmrStart
(
vnodeDelayedFreeResource
,
5000
,
v
,
vnodeTmrCtrl
);
}
void
vnodeCleanUpShell
()
{
...
...
@@ -488,24 +505,38 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) {
int
subMsgLen
=
sizeof
(
pBlocks
->
numOfRows
)
+
htons
(
pBlocks
->
numOfRows
)
*
pMeterObj
->
bytesPerPoint
;
int
sversion
=
htonl
(
pBlocks
->
sversion
);
if
(
pMeterObj
->
state
==
TSDB_METER_STATE_READY
)
{
if
(
pSubmit
->
import
)
code
=
vnodeImportPoints
(
pMeterObj
,
(
char
*
)
&
(
pBlocks
->
numOfRows
),
subMsgLen
,
TSDB_DATA_SOURCE_SHELL
,
pObj
,
int32_t
state
=
TSDB_METER_STATE_READY
;
if
(
pSubmit
->
import
)
{
state
=
vnodeTransferMeterState
(
pMeterObj
,
TSDB_METER_STATE_IMPORTING
);
}
else
{
state
=
vnodeTransferMeterState
(
pMeterObj
,
TSDB_METER_STATE_INSERT
);
}
if
(
state
==
TSDB_METER_STATE_READY
)
{
// meter status is ready for insert/import
if
(
pSubmit
->
import
)
{
code
=
vnodeImportPoints
(
pMeterObj
,
(
char
*
)
&
(
pBlocks
->
numOfRows
),
subMsgLen
,
TSDB_DATA_SOURCE_SHELL
,
pObj
,
sversion
,
&
numOfPoints
);
else
code
=
vnodeInsertPoints
(
pMeterObj
,
(
char
*
)
&
(
pBlocks
->
numOfRows
),
subMsgLen
,
TSDB_DATA_SOURCE_SHELL
,
NULL
,
}
else
{
code
=
vnodeInsertPoints
(
pMeterObj
,
(
char
*
)
&
(
pBlocks
->
numOfRows
),
subMsgLen
,
TSDB_DATA_SOURCE_SHELL
,
NULL
,
sversion
,
&
numOfPoints
);
if
(
code
!=
0
)
break
;
}
else
if
(
pMeterObj
->
state
>=
TSDB_METER_STATE_DELETING
)
{
dTrace
(
"vid:%d sid:%d id:%s, is is removed, state:"
,
pMeterObj
->
vnode
,
pMeterObj
->
sid
,
pMeterObj
->
meterId
,
pMeterObj
->
state
);
code
=
TSDB_CODE_NOT_ACTIVE_SESSION
;
break
;
}
else
{
// importing state or others
dTrace
(
"vid:%d sid:%d id:%s, try again since in state:%d"
,
pMeterObj
->
vnode
,
pMeterObj
->
sid
,
pMeterObj
->
meterId
,
pMeterObj
->
state
);
code
=
TSDB_CODE_ACTION_IN_PROGRESS
;
break
;
vnodeClearMeterState
(
pMeterObj
,
TSDB_METER_STATE_INSERT
);
}
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
break
;}
}
else
{
if
(
vnodeIsMeterState
(
pMeterObj
,
TSDB_METER_STATE_DELETING
))
{
dTrace
(
"vid:%d sid:%d id:%s, it is removed, state:%d"
,
pMeterObj
->
vnode
,
pMeterObj
->
sid
,
pMeterObj
->
meterId
,
pMeterObj
->
state
);
code
=
TSDB_CODE_NOT_ACTIVE_SESSION
;
break
;
}
else
{
// waiting for 300ms by default and try again
dTrace
(
"vid:%d sid:%d id:%s, try submit again since in state:%d"
,
pMeterObj
->
vnode
,
pMeterObj
->
sid
,
pMeterObj
->
meterId
,
pMeterObj
->
state
);
code
=
TSDB_CODE_ACTION_IN_PROGRESS
;
break
;
}
}
numOfTotalPoints
+=
numOfPoints
;
...
...
src/system/src/vnodeStore.c
浏览文件 @
fe6672d7
...
...
@@ -85,13 +85,42 @@ int vnodeOpenVnode(int vnode) {
return
0
;
}
void
vnodeCloseVnode
(
int
vnode
)
{
if
(
vnodeList
==
NULL
)
return
;
static
int32_t
vnodeMarkAllMetersDropped
(
SVnodeObj
*
pVnode
)
{
if
(
pVnode
->
meterList
==
NULL
)
{
assert
(
pVnode
->
cfg
.
maxSessions
==
0
);
return
TSDB_CODE_SUCCESS
;
}
bool
ready
=
true
;
for
(
int
sid
=
0
;
sid
<
pVnode
->
cfg
.
maxSessions
;
++
sid
)
{
if
(
!
vnodeIsSafeToDeleteMeter
(
pVnode
,
sid
))
{
ready
=
false
;
}
else
{
// set the meter is to be deleted
SMeterObj
*
pObj
=
pVnode
->
meterList
[
sid
];
if
(
pObj
!=
NULL
)
{
pObj
->
state
=
TSDB_METER_STATE_DELETED
;
}
}
}
return
ready
?
TSDB_CODE_SUCCESS
:
TSDB_CODE_ACTION_IN_PROGRESS
;
}
int
vnodeCloseVnode
(
int
vnode
)
{
if
(
vnodeList
==
NULL
)
return
TSDB_CODE_SUCCESS
;
SVnodeObj
*
pVnode
=
&
vnodeList
[
vnode
];
pthread_mutex_lock
(
&
dmutex
);
if
(
vnodeList
[
vnode
].
cfg
.
maxSessions
==
0
)
{
if
(
pVnode
->
cfg
.
maxSessions
==
0
)
{
pthread_mutex_unlock
(
&
dmutex
);
return
;
return
TSDB_CODE_SUCCESS
;
}
// set the meter is dropped flag
if
(
vnodeMarkAllMetersDropped
(
pVnode
)
!=
TSDB_CODE_SUCCESS
)
{
pthread_mutex_unlock
(
&
dmutex
);
return
TSDB_CODE_ACTION_IN_PROGRESS
;
}
vnodeCloseStream
(
vnodeList
+
vnode
);
...
...
@@ -111,6 +140,7 @@ void vnodeCloseVnode(int vnode) {
vnodeCalcOpenVnodes
();
pthread_mutex_unlock
(
&
dmutex
);
return
TSDB_CODE_SUCCESS
;
}
int
vnodeCreateVnode
(
int
vnode
,
SVnodeCfg
*
pCfg
,
SVPeerDesc
*
pDesc
)
{
...
...
@@ -182,25 +212,23 @@ void vnodeRemoveDataFiles(int vnode) {
dTrace
(
"vnode %d is removed!"
,
vnode
);
}
void
vnodeRemoveVnode
(
int
vnode
)
{
if
(
vnodeList
==
NULL
)
return
;
int
vnodeRemoveVnode
(
int
vnode
)
{
if
(
vnodeList
==
NULL
)
return
TSDB_CODE_SUCCESS
;
if
(
vnodeList
[
vnode
].
cfg
.
maxSessions
>
0
)
{
vnodeCloseVnode
(
vnode
);
int32_t
ret
=
vnodeCloseVnode
(
vnode
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
return
ret
;
}
vnodeRemoveDataFiles
(
vnode
);
// sprintf(cmd, "rm -rf %s/vnode%d", tsDirectory, vnode);
// if ( system(cmd) < 0 ) {
// dError("vid:%d, failed to run command %s vnode, reason:%s", vnode, cmd, strerror(errno));
// } else {
// dTrace("vid:%d, this vnode is deleted!!!", vnode);
// }
}
else
{
dTrace
(
"vid:%d, max sessions:%d, this vnode already dropped!!!"
,
vnode
,
vnodeList
[
vnode
].
cfg
.
maxSessions
);
vnodeList
[
vnode
].
cfg
.
maxSessions
=
0
;
vnodeList
[
vnode
].
cfg
.
maxSessions
=
0
;
//reset value
vnodeCalcOpenVnodes
();
}
return
TSDB_CODE_SUCCESS
;
}
int
vnodeInitStore
()
{
...
...
src/system/src/vnodeStream.c
浏览文件 @
fe6672d7
...
...
@@ -51,8 +51,17 @@ void vnodeProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
}
contLen
+=
sizeof
(
SSubmitMsg
);
int32_t
numOfPoints
=
0
;
vnodeInsertPoints
(
pObj
,
(
char
*
)
pMsg
,
contLen
,
TSDB_DATA_SOURCE_SHELL
,
NULL
,
pObj
->
sversion
,
&
numOfPoints
);
int32_t
state
=
vnodeTransferMeterState
(
pObj
,
TSDB_METER_STATE_INSERT
);
if
(
state
==
TSDB_METER_STATE_READY
)
{
vnodeInsertPoints
(
pObj
,
(
char
*
)
pMsg
,
contLen
,
TSDB_DATA_SOURCE_SHELL
,
NULL
,
pObj
->
sversion
,
&
numOfPoints
);
vnodeClearMeterState
(
pObj
,
TSDB_METER_STATE_INSERT
);
}
else
{
dError
(
"vid:%d sid:%d id:%s, failed to insert continuous query results, state:%d"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
state
);
}
assert
(
numOfPoints
>=
0
&&
numOfPoints
<=
1
);
tfree
(
pTemp
);
...
...
@@ -76,7 +85,7 @@ void vnodeOpenStreams(void *param, void *tmrId) {
for
(
int
sid
=
0
;
sid
<
pVnode
->
cfg
.
maxSessions
;
++
sid
)
{
pObj
=
pVnode
->
meterList
[
sid
];
if
(
pObj
==
NULL
||
pObj
->
sqlLen
==
0
||
pObj
->
status
==
1
||
pObj
->
state
==
TSDB_METER_STATE_DELETED
)
continue
;
if
(
pObj
==
NULL
||
pObj
->
sqlLen
==
0
||
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_DELETING
)
)
continue
;
dTrace
(
"vid:%d sid:%d id:%s, open stream:%s"
,
pObj
->
vnode
,
sid
,
pObj
->
meterId
,
pObj
->
pSql
);
...
...
src/system/src/vnodeUtil.c
浏览文件 @
fe6672d7
...
...
@@ -361,6 +361,7 @@ void vnodeUpdateFilterColumnIndex(SQuery* pQuery) {
// TODO support k<12 and k<>9
int32_t
vnodeCreateFilterInfo
(
void
*
pQInfo
,
SQuery
*
pQuery
)
{
for
(
int32_t
i
=
0
;
i
<
pQuery
->
numOfCols
;
++
i
)
{
if
(
pQuery
->
colList
[
i
].
data
.
filterOn
>
0
)
{
pQuery
->
numOfFilterCols
++
;
...
...
@@ -401,8 +402,6 @@ int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) {
pFilterInfo
->
fp
=
rangeFilterArray
[
2
];
}
}
else
{
assert
(
lower
==
TSDB_RELATION_LARGE
);
if
(
upper
==
TSDB_RELATION_LESS_EQUAL
)
{
pFilterInfo
->
fp
=
rangeFilterArray
[
3
];
}
else
{
...
...
@@ -421,6 +420,7 @@ int32_t vnodeCreateFilterInfo(void* pQInfo, SQuery* pQuery) {
pFilterInfo
->
fp
=
filterArray
[
upper
];
}
}
pFilterInfo
->
elemSize
=
bytes
;
j
++
;
}
...
...
@@ -470,6 +470,18 @@ bool vnodeIsProjectionQuery(SSqlFunctionExpr* pExpr, int32_t numOfOutput) {
return
true
;
}
/*
* the pMeter->state may be changed by vnodeIsSafeToDeleteMeter and import/update processor, the check of
* the state will not always be correct.
*
* The import/update/deleting is actually blocked by current query processing if the check of meter state is
* passed, but later queries are denied.
*
* 1. vnodeIsSafeToDelete will wait for this complete, since it also use the vmutex to check the numOfQueries
* 2. import will check the numOfQueries again after setting state to be TSDB_METER_STATE_IMPORTING, while the
* vmutex is also used.
* 3. insert has nothing to do with the query processing.
*/
int32_t
vnodeIncQueryRefCount
(
SQueryMeterMsg
*
pQueryMsg
,
SMeterSidExtInfo
**
pSids
,
SMeterObj
**
pMeterObjList
,
int32_t
*
numOfInc
)
{
SVnodeObj
*
pVnode
=
&
vnodeList
[
pQueryMsg
->
vnode
];
...
...
@@ -477,21 +489,24 @@ int32_t vnodeIncQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterSidExtInfo** pSid
int32_t
num
=
0
;
int32_t
code
=
TSDB_CODE_SUCCESS
;
// check all meter metadata to ensure all metadata are identical.
for
(
int32_t
i
=
0
;
i
<
pQueryMsg
->
numOfSids
;
++
i
)
{
SMeterObj
*
pMeter
=
pVnode
->
meterList
[
pSids
[
i
]
->
sid
];
if
(
pMeter
==
NULL
||
pMeter
->
state
!=
TSDB_METER_STATE_READY
)
{
if
(
pMeter
==
NULL
)
{
if
(
pMeter
==
NULL
||
(
pMeter
->
state
>
TSDB_METER_STATE_INSERT
)
)
{
if
(
pMeter
==
NULL
||
vnodeIsMeterState
(
pMeter
,
TSDB_METER_STATE_DELETING
)
)
{
code
=
TSDB_CODE_NOT_ACTIVE_SESSION
;
dError
(
"qmsg:%p, vid:%d sid:%d, not there"
,
pQueryMsg
,
pQueryMsg
->
vnode
,
pSids
[
i
]
->
sid
);
dError
(
"qmsg:%p, vid:%d sid:%d, not there
or will be dropped
"
,
pQueryMsg
,
pQueryMsg
->
vnode
,
pSids
[
i
]
->
sid
);
vnodeSendMeterCfgMsg
(
pQueryMsg
->
vnode
,
pSids
[
i
]
->
sid
);
}
else
{
}
else
{
//update or import
code
=
TSDB_CODE_ACTION_IN_PROGRESS
;
dTrace
(
"qmsg:%p, vid:%d sid:%d id:%s, it is in state:%d, wait!"
,
pQueryMsg
,
pQueryMsg
->
vnode
,
pSids
[
i
]
->
sid
,
pMeter
->
meterId
,
pMeter
->
state
);
}
}
else
{
/*
* vnodeIsSafeToDeleteMeter will wait for this function complete, and then it can
* check if the numOfQueries is 0 or not.
*/
pMeterObjList
[(
*
numOfInc
)
++
]
=
pMeter
;
__sync_fetch_and_add
(
&
pMeter
->
numOfQueries
,
1
);
...
...
@@ -517,7 +532,6 @@ void vnodeDecQueryRefCount(SQueryMeterMsg* pQueryMsg, SMeterObj** pMeterObjList,
SMeterObj
*
pMeter
=
pMeterObjList
[
i
];
if
(
pMeter
!=
NULL
)
{
// here, do not need to lock to perform operations
assert
(
pMeter
->
state
!=
TSDB_METER_STATE_DELETING
&&
pMeter
->
state
!=
TSDB_METER_STATE_DELETED
);
__sync_fetch_and_sub
(
&
pMeter
->
numOfQueries
,
1
);
if
(
pMeter
->
numOfQueries
>
0
)
{
...
...
@@ -571,3 +585,66 @@ void vnodeUpdateQueryColumnIndex(SQuery* pQuery, SMeterObj* pMeterObj) {
}
}
}
int32_t
vnodeTransferMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
)
{
return
__sync_val_compare_and_swap
(
&
pMeterObj
->
state
,
TSDB_METER_STATE_READY
,
state
);
}
void
vnodeClearMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
)
{
pMeterObj
->
state
&=
(
~
state
);
}
bool
vnodeIsMeterState
(
SMeterObj
*
pMeterObj
,
int32_t
state
)
{
if
(
state
==
TSDB_METER_STATE_READY
)
{
return
pMeterObj
->
state
==
TSDB_METER_STATE_READY
;
}
else
if
(
state
==
TSDB_METER_STATE_DELETING
)
{
return
pMeterObj
->
state
>=
state
;
}
else
{
return
(((
pMeterObj
->
state
)
&
state
)
==
state
);
}
}
void
vnodeSetMeterDeleting
(
SMeterObj
*
pMeterObj
)
{
if
(
pMeterObj
==
NULL
)
{
return
;
}
pMeterObj
->
state
|=
TSDB_METER_STATE_DELETING
;
}
bool
vnodeIsSafeToDeleteMeter
(
SVnodeObj
*
pVnode
,
int32_t
sid
)
{
SMeterObj
*
pObj
=
pVnode
->
meterList
[
sid
];
if
(
pObj
==
NULL
||
vnodeIsMeterState
(
pObj
,
TSDB_METER_STATE_DELETED
))
{
return
true
;
}
int32_t
prev
=
vnodeTransferMeterState
(
pObj
,
TSDB_METER_STATE_DELETING
);
/*
* if the meter is not in ready/deleting state, it must be in insert/import/update,
* set the deleting state and wait the procedure to be completed
*/
if
(
prev
!=
TSDB_METER_STATE_READY
&&
prev
<
TSDB_METER_STATE_DELETING
)
{
vnodeSetMeterDeleting
(
pObj
);
dWarn
(
"vid:%d sid:%d id:%s, can not be deleted, state:%d, wait"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
prev
);
return
false
;
}
bool
ready
=
true
;
/*
* the query will be stopped ASAP, since the state of meter is set to TSDB_METER_STATE_DELETING,
* and new query will abort since the meter is deleted.
*/
pthread_mutex_lock
(
&
pVnode
->
vmutex
);
if
(
pObj
->
numOfQueries
>
0
)
{
dWarn
(
"vid:%d sid:%d id:%s %d queries executing on it, wait query to be finished"
,
pObj
->
vnode
,
pObj
->
sid
,
pObj
->
meterId
,
pObj
->
numOfQueries
);
ready
=
false
;
}
pthread_mutex_unlock
(
&
pVnode
->
vmutex
);
return
ready
;
}
src/util/src/textbuffer.c
浏览文件 @
fe6672d7
...
...
@@ -1532,7 +1532,7 @@ void tColModelDisplayEx(tColModel *pModel, void *pData, int32_t numOfRows, int32
}
////////////////////////////////////////////////////////////////////////////////////////////
void
tColModelComp
ress
(
tColModel
*
pModel
,
tFilePage
*
inputBuffer
,
int32_t
maxElemsCapacity
)
{
void
tColModelComp
act
(
tColModel
*
pModel
,
tFilePage
*
inputBuffer
,
int32_t
maxElemsCapacity
)
{
if
(
inputBuffer
->
numOfElems
==
0
||
maxElemsCapacity
==
inputBuffer
->
numOfElems
)
{
return
;
}
...
...
src/util/src/tinterpolation.c
浏览文件 @
fe6672d7
...
...
@@ -117,7 +117,7 @@ int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo* pInterpoInfo, int64_t* p
}
}
bool
taosHas
NoneInterpoPoints
(
SInterpolationInfo
*
pInterpoInfo
)
{
return
taosNumOfRemainPoints
(
pInterpoInfo
)
>
0
;
}
bool
taosHas
RemainsDataForInterpolation
(
SInterpolationInfo
*
pInterpoInfo
)
{
return
taosNumOfRemainPoints
(
pInterpoInfo
)
>
0
;
}
int32_t
taosNumOfRemainPoints
(
SInterpolationInfo
*
pInterpoInfo
)
{
if
(
pInterpoInfo
->
rowIdx
==
-
1
||
pInterpoInfo
->
numOfRawDataInRows
==
0
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录