Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
提交 9aa1e1d3 编写于 作者: C Cary Xu
浏览文件
操作

Merge branch '3.0' into feature/TD-14481-3.0

上级 8e434be2 c2477b4e
展开全部 隐藏空白更改
内联 并排
Showing with 2095 addition and 1145 deletion
docs-cn/27-train-faq/01-faq.md
浏览文件 @ 9aa1e1d3
......@@ -74,7 +74,7 @@ title: 常见问题及反馈
检查服务器侧 TCP 端口连接是否工作：`nc -l {port}`
检查客户端侧 TCP 端口连接是否工作：`nc {hostIP} {port}`
- Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
- Windows 系统请使用 PowerShell 命令 Test-NetConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问
10. 也可以使用 taos 程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括 TCP 和 UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。
......
docs-en/14-reference/03-connector/python.mdx
浏览文件 @ 9aa1e1d3
......@@ -199,8 +199,7 @@ The `connect()` function returns a `taos.TaosConnection` instance. In client-sid
All arguments to the `connect()` function are optional keyword arguments. The following are the connection parameters specified.
- - `url`： The URL of taosAdapter REST service. The default is <http://localhost:6041>.
- `host`: The host to connect to. The default is localhost.
- `url`： The URL of taosAdapter REST service. The default is <http://localhost:6041>.
- `user`: TDengine user name. The default is `root`.
- `password`: TDengine user password. The default is `taosdata`.
- `timeout`: HTTP request timeout in seconds. The default is `socket._GLOBAL_DEFAULT_TIMEOUT`. Usually, no configuration is needed.
......
docs-en/27-train-faq/01-faq.md
浏览文件 @ 9aa1e1d3
......@@ -45,7 +45,7 @@ When the client is unable to connect to the server, you can try the following wa
Check whether a TCP port on server side is open: `nc -l {port}`
Check whether a TCP port on client side is open: `nc {hostIP} {port}`
- On Windows system `Net-TestConnection -ComputerName {fqdn} -Port {port}` on PowerShell can be used to check whether the port on server side is open for access.
- On Windows system `Test-NetConnection -ComputerName {fqdn} -Port {port}` on PowerShell can be used to check whether the port on server side is open for access.
7. TDengine CLI `taos` can also be used to check network, please refer to [TDengine CLI](/reference/taos-shell).
......
include/common/tcommon.h
浏览文件 @ 9aa1e1d3
......@@ -108,6 +108,7 @@ typedef struct SColumnInfoData {
typedef struct SQueryTableDataCond {
// STimeWindow twindow;
uint64_t suid;
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo* colList;
......
include/common/tdatablock.h
浏览文件 @ 9aa1e1d3
......@@ -71,7 +71,7 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
......@@ -180,7 +180,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
*(double*)p = *(double*)v;
}
int32_t getJsonValueLen(const char *data);
int32_t getJsonValueLen(const char* data);
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
......@@ -223,7 +223,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress);
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
void blockDebugShowData(const SArray* dataBlocks, const char* flag);
......@@ -231,6 +232,8 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag);
int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
tb_uid_t suid);
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId);
......
include/common/tmsg.h
浏览文件 @ 9aa1e1d3
......@@ -1315,6 +1315,31 @@ typedef struct {
int32_t tSerializeSKillTransReq(void* buf, int32_t bufLen, SKillTransReq* pReq);
int32_t tDeserializeSKillTransReq(void* buf, int32_t bufLen, SKillTransReq* pReq);
typedef struct {
int32_t useless; // useless
} SBalanceVgroupReq;
int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq);
int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq);
typedef struct {
int32_t vgId1;
int32_t vgId2;
} SMergeVgroupReq;
int32_t tSerializeSMergeVgroupReq(void* buf, int32_t bufLen, SMergeVgroupReq* pReq);
int32_t tDeserializeSMergeVgroupReq(void* buf, int32_t bufLen, SMergeVgroupReq* pReq);
typedef struct {
int32_t vgId;
int32_t dnodeId1;
int32_t dnodeId2;
int32_t dnodeId3;
} SRedistributeVgroupReq;
int32_t tSerializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq);
int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq);
typedef struct {
char user[TSDB_USER_LEN];
char spi;
......
include/common/tmsgdef.h
浏览文件 @ 9aa1e1d3
......@@ -153,6 +153,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "grant", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "auth", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP, "balance-vgroup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MERGE_VGROUP, "merge-vgroup", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_REDISTRIBUTE_VGROUP, "redistribute-vgroup", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG)
TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
......
include/common/ttokendef.h
浏览文件 @ 9aa1e1d3
......@@ -188,10 +188,10 @@
#define TK_KILL 170
#define TK_CONNECTION 171
#define TK_TRANSACTION 172
#define TK_MERGE 173
#define TK_BALANCE 173
#define TK_VGROUP 174
#define TK_REDISTRIBUTE 175
#define TK_SPLIT 176
#define TK_MERGE 175
#define TK_REDISTRIBUTE 176
#define TK_SYNCDB 177
#define TK_DELETE 178
#define TK_NULL 179
......
include/libs/nodes/cmdnodes.h
浏览文件 @ 9aa1e1d3
......@@ -28,6 +28,14 @@ extern "C" {
#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
#define PRIVILEGE_TYPE_MASK(n) (1 << n)
#define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0)
#define PRIVILEGE_TYPE_READ PRIVILEGE_TYPE_MASK(1)
#define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2)
#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
typedef struct SDatabaseOptions {
ENodeType type;
int32_t buffer;
......@@ -316,14 +324,6 @@ typedef struct SDropFunctionStmt {
bool ignoreNotExists;
} SDropFunctionStmt;
#define PRIVILEGE_TYPE_MASK(n) (1 << n)
#define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0)
#define PRIVILEGE_TYPE_READ PRIVILEGE_TYPE_MASK(1)
#define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2)
#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
typedef struct SGrantStmt {
ENodeType type;
char userName[TSDB_USER_LEN];
......@@ -333,6 +333,30 @@ typedef struct SGrantStmt {
typedef SGrantStmt SRevokeStmt;
typedef struct SBalanceVgroupStmt {
ENodeType type;
} SBalanceVgroupStmt;
typedef struct SMergeVgroupStmt {
ENodeType type;
int32_t vgId1;
int32_t vgId2;
} SMergeVgroupStmt;
typedef struct SRedistributeVgroupStmt {
ENodeType type;
int32_t vgId;
int32_t dnodeId1;
int32_t dnodeId2;
int32_t dnodeId3;
SNodeList* pDnodes;
} SRedistributeVgroupStmt;
typedef struct SSplitVgroupStmt {
ENodeType type;
int32_t vgId;
} SSplitVgroupStmt;
#ifdef __cplusplus
}
#endif
......
include/libs/nodes/nodes.h
浏览文件 @ 9aa1e1d3
......@@ -140,6 +140,7 @@ typedef enum ENodeType {
QUERY_NODE_DROP_FUNCTION_STMT,
QUERY_NODE_CREATE_STREAM_STMT,
QUERY_NODE_DROP_STREAM_STMT,
QUERY_NODE_BALANCE_VGROUP_STMT,
QUERY_NODE_MERGE_VGROUP_STMT,
QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
QUERY_NODE_SPLIT_VGROUP_STMT,
......
include/libs/stream/tstream.h
浏览文件 @ 9aa1e1d3
......@@ -145,26 +145,6 @@ SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit);
#if 0
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
static FORCE_INLINE int32_t streamEnqueue1(SStreamQueue* queue, SStreamQueueItem* pItem) {
int8_t inputStatus = atomic_load_8(&queue->enqueueStatus);
if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem);
if (pSubmitClone == NULL) {
atomic_store_8(&queue->enqueueStatus, TASK_INPUT_STATUS__FAILED);
return -1;
}
taosWriteQitem(queue->queue, pSubmitClone);
} else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
taosWriteQitem(queue->queue, pItem);
} else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
taosWriteQitem(queue->queue, pItem);
}
return 0;
}
return 0;
}
#endif
typedef struct {
......
include/util/taoserror.h
浏览文件 @ 9aa1e1d3
......@@ -653,6 +653,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2653)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2654)
#define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655)
#define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
......
source/client/src/tmq.c
浏览文件 @ 9aa1e1d3
......@@ -195,7 +195,7 @@ typedef struct {
tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
conf->withTbName = -1;
conf->withTbName = false;
conf->autoCommit = true;
conf->autoCommitInterval = 5000;
conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
......@@ -256,13 +256,10 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcmp(key, "msg.with.table.name") == 0) {
if (strcmp(value, "true") == 0) {
conf->withTbName = 1;
conf->withTbName = true;
return TMQ_CONF_OK;
} else if (strcmp(value, "false") == 0) {
conf->withTbName = 0;
return TMQ_CONF_OK;
} else if (strcmp(value, "none") == 0) {
conf->withTbName = -1;
conf->withTbName = false;
return TMQ_CONF_OK;
} else {
return TMQ_CONF_INVALID;
......
source/common/src/tdatablock.c
浏览文件 @ 9aa1e1d3
......@@ -18,6 +18,7 @@
#include "tcompare.h"
#include "tglobal.h"
#include "tlog.h"
#include "tname.h"
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0;
......@@ -99,7 +100,7 @@ void colDataTrim(SColumnInfoData* pColumnInfoData) {
// TODO
}
int32_t getJsonValueLen(const char *data) {
int32_t getJsonValueLen(const char* data) {
int32_t dataLen = 0;
if (*data == TSDB_DATA_TYPE_NULL) {
dataLen = CHAR_BYTES;
......@@ -109,7 +110,7 @@ int32_t getJsonValueLen(const char *data) {
dataLen = DOUBLE_BYTES + CHAR_BYTES;
} else if (*data == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES + CHAR_BYTES;
} else if (*data & TD_TAG_JSON) { // json string
} else if (*data & TD_TAG_JSON) { // json string
dataLen = ((STag*)(data))->len;
} else {
ASSERT(0);
......@@ -137,7 +138,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
int32_t dataLen = 0;
if (type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
}else {
} else {
dataLen = varDataTLen(pData);
}
......@@ -1283,7 +1284,7 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
if (n % 8 == 0) {
memmove(nullBitmap, nullBitmap + n / 8, newLen);
} else {
int32_t tail = n % 8;
int32_t tail = n % 8;
int32_t i = 0;
uint8_t* p = (uint8_t*)nullBitmap;
......@@ -1301,7 +1302,7 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
}
} else if (n > 8) {
int32_t gap = len - newLen;
while(i < newLen) {
while (i < newLen) {
uint8_t v = p[i + gap];
p[i] = (v << tail);
......@@ -1316,7 +1317,6 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
}
}
static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t));
......@@ -1544,7 +1544,8 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
*
* TODO: colId should be set
*/
int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) {
int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
tb_uid_t suid) {
int32_t sz = taosArrayGetSize(pDataBlocks);
int32_t bufSize = sizeof(SSubmitReq);
for (int32_t i = 0; i < sz; ++i) {
......@@ -1585,12 +1586,12 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
int32_t dataLen = 0;
for (int32_t j = 0; j < rows; ++j) { // iterate by row
tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf
bool isStartKey = false;
bool isStartKey = false;
int32_t offset = 0;
for (int32_t k = 0; k < colNum; ++k) { // iterate by column
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
STColumn* pCol = &pTSchema->columns[k];
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
STColumn* pCol = &pTSchema->columns[k];
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP:
if (!isStartKey) {
......@@ -1599,15 +1600,18 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
offset, k);
} else {
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k);
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var,
true, offset, k);
}
break;
case TSDB_DATA_TYPE_NCHAR: {
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k);
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true,
offset, k);
break;
}
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k);
tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true,
offset, k);
break;
}
case TSDB_DATA_TYPE_VARBINARY:
......@@ -1645,7 +1649,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
}
break;
}
offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation
offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation
}
dataLen += TD_ROW_LEN(rb.pBuf);
#ifdef TD_DEBUG_PRINT_ROW
......@@ -1681,11 +1685,38 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
return TSDB_CODE_SUCCESS;
}
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) {
SArray* tags = taosArrayInit(0, sizeof(void*));
SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv));
pTag->key = "group_id";
pTag->keyLen = strlen(pTag->key);
pTag->type = TSDB_DATA_TYPE_UBIGINT;
pTag->u = groupId;
taosArrayPush(tags, &pTag);
void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1);
RandTableName rname = {
.tags = tags,
.sTableName = stbName,
.sTableNameLen = strlen(stbName),
.childTableName = cname,
};
buildChildTableName(&rname);
taosMemoryFree(pTag);
taosArrayDestroy(tags);
ASSERT(rname.childTableName && rname.childTableName[0]);
return rname.childTableName;
}
SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid,
const char* stbFullName, int32_t vgId) {
SSubmitReq* ret = NULL;
SArray* tagArray = taosArrayInit(1, sizeof(STagVal));
if(!tagArray) {
if (!tagArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
......@@ -1703,15 +1734,12 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
if (createTb) {
SVCreateTbReq createTbReq = {0};
char* cname = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
snprintf(cname, TSDB_TABLE_FNAME_LEN, "%s:%ld", stbFullName, pDataBlock->info.groupId);
char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
createTbReq.name = cname;
createTbReq.flags = 0;
createTbReq.type = TSDB_CHILD_TABLE;
createTbReq.ctb.suid = suid;
STagVal tagVal = {.cid = 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.pData = (uint8_t*)&pDataBlock->info.groupId,
......
source/common/src/tmsg.c
浏览文件 @ 9aa1e1d3
......@@ -3258,6 +3258,89 @@ int32_t tDeserializeSKillTransReq(void *buf, int32_t bufLen, SKillTransReq *pReq
return 0;
}
int32_t tSerializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId1) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId2) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId1) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId2) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSRedistributeVgroupReq(void *buf, int32_t bufLen, SRedistributeVgroupReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId1) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId2) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId3) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSRedistributeVgroupReq(void *buf, int32_t bufLen, SRedistributeVgroupReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId1) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId2) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId3) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ 9aa1e1d3
......@@ -36,12 +36,12 @@ target_sources(
# tsdb
"src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCommit2.c"
# "src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c"
"src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c"
"src/tsdb/tsdbMemTable2.c"
# "src/tsdb/tsdbMemTable2.c"
"src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbWrite.c"
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 9aa1e1d3
......@@ -39,24 +39,46 @@ typedef struct SDelOp SDelOp;
static int tsdbKeyCmprFn(const void *p1, const void *p2);
// tsdbMemTable ==============================================================================================
typedef struct STbData STbData;
typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter;
typedef struct SMergeInfo SMergeInfo;
typedef struct STable STable;
// SMemTable
int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
void tsdbMemTableDestroy(SMemTable *pMemTable);
void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
// STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void *tsdbTbDataIterDestroy(STbDataIter *pIter);
void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
bool tsdbTbDataIterNext(STbDataIter *pIter);
bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// tsdbMemTable2.c ==============================================================================================
typedef struct SMemTable SMemTable;
typedef struct SMemData SMemData;
typedef struct SMemDataIter SMemDataIter;
// typedef struct SMemTable2 SMemTable2;
// typedef struct SMemData SMemData;
// typedef struct SMemDataIter SMemDataIter;
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable);
void tsdbMemTableDestroy2(SMemTable *pMemTable);
int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
// int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
// void tsdbMemTableDestroy2(SMemTable2 *pMemTable);
// int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
// int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
/* SMemDataIter */
void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
bool tsdbMemDataIterNext(SMemDataIter *pIter);
void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
// /* SMemDataIter */
// void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
// bool tsdbMemDataIterNext(SMemDataIter *pIter);
// void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
// tsdbCommit2.c ==============================================================================================
int32_t tsdbBegin2(STsdb *pTsdb);
int32_t tsdbCommit2(STsdb *pTsdb);
// // tsdbCommit2.c ==============================================================================================
// int32_t tsdbBegin2(STsdb *pTsdb);
// int32_t tsdbCommit2(STsdb *pTsdb);
// tsdbFile.c ==============================================================================================
typedef int32_t TSDB_FILE_T;
......@@ -124,17 +146,6 @@ int tsdbRLockFS(STsdbFS *pFs);
int tsdbWLockFS(STsdbFS *pFs);
int tsdbUnLockFS(STsdbFS *pFs);
// tsdbMemTable ================
typedef struct STbData STbData;
typedef struct STsdbMemTable STsdbMemTable;
typedef struct SMergeInfo SMergeInfo;
typedef struct STable STable;
int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable);
void tsdbMemTableDestroy(STsdbMemTable *pMemTable);
int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// structs
typedef struct {
int minFid;
......@@ -145,30 +156,27 @@ typedef struct {
#define TSDB_DATA_DIR_LEN 6 // adapt accordingly
struct STsdb {
char *path;
SVnode *pVnode;
TdThreadMutex mutex;
char dir[TSDB_DATA_DIR_LEN];
bool repoLocked;
STsdbKeepCfg keepCfg;
STsdbMemTable *mem;
STsdbMemTable *imem;
SRtn rtn;
STsdbFS *fs;
char *path;
SVnode *pVnode;
TdThreadMutex mutex;
char dir[TSDB_DATA_DIR_LEN];
bool repoLocked;
STsdbKeepCfg keepCfg;
SMemTable *mem;
SMemTable *imem;
SRtn rtn;
STsdbFS *fs;
};
#if 1 // ======================================
struct STable {
uint64_t tid;
uint64_t suid;
uint64_t uid;
STSchema *pSchema; // latest schema
STSchema *pCacheSchema; // cached cache
};
#define TABLE_TID(t) (t)->tid
#define TABLE_UID(t) (t)->uid
// int tsdbPrepareCommit(STsdb *pTsdb);
typedef enum {
TSDB_FILE_HEAD = 0, // .head
......@@ -206,27 +214,44 @@ struct SDFileSet {
SDFile files[TSDB_FILE_MAX];
};
struct TSDBKEY {
int64_t version;
TSKEY ts;
};
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
int8_t level;
SMemSkipListNode *forwards[0];
};
typedef struct SMemSkipList {
uint32_t seed;
int64_t size;
int8_t maxLevel;
int8_t level;
SMemSkipListNode *pHead;
SMemSkipListNode *pTail;
} SMemSkipList;
struct STbData {
tb_uid_t uid;
TSKEY keyMin;
TSKEY keyMax;
int64_t minVer;
int64_t maxVer;
int64_t nrows;
SSkipList *pData;
tb_uid_t suid;
tb_uid_t uid;
TSDBKEY minKey;
TSDBKEY maxKey;
SDelOp *pHead;
SDelOp *pTail;
SMemSkipList sl;
};
struct STsdbMemTable {
SVBufPool *pPool;
T_REF_DECLARE()
SRWLatch latch;
TSKEY keyMin;
TSKEY keyMax;
int64_t minVer;
int64_t maxVer;
int64_t nRow;
SSkipList *pSlIdx; // SSkiplist<STbData>
SHashObj *pHashIdx;
struct SMemTable {
SRWLatch latch;
STsdb *pTsdb;
int32_t nRef;
TSDBKEY minKey;
TSDBKEY maxKey;
int64_t nRow;
int64_t nDelOp;
SArray *aTbData; // SArray<STbData>
};
struct STsdbFSMeta {
......@@ -237,9 +262,11 @@ struct STsdbFSMeta {
// ==================
typedef struct {
STsdbFSMeta meta; // FS meta
SArray *df; // data file array
SArray *sf; // sma data file array v2f1900.index_name_1
STsdbFSMeta meta; // FS meta
SDFile cacheFile; // cache file
SDFile tombstone; // tomestome file
SArray *df; // data file array
SArray *sf; // sma data file array v2f1900.index_name_1
} SFSStatus;
struct STsdbFS {
......@@ -292,16 +319,24 @@ static void *taosTZfree(void *ptr);
static size_t taosTSizeof(void *ptr);
static void taosTMemset(void *ptr, int c);
static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
struct TSDBROW {
int64_t version;
STSRow *pTSRow;
};
static FORCE_INLINE STSRow *tsdbNextIterRow(STbDataIter *pIter) {
TSDBROW row;
if (pIter == NULL) return NULL;
SSkipListNode *node = tSkipListIterGet(pIter);
if (node == NULL) return NULL;
if (tsdbTbDataIterGet(pIter, &row)) {
return row.pTSRow;
}
return (STSRow *)SL_GET_NODE_DATA(node);
return NULL;
}
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
static FORCE_INLINE TSKEY tsdbNextIterKey(STbDataIter *pIter) {
STSRow *row = tsdbNextIterRow(pIter);
if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
......@@ -311,11 +346,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
// tsdbReadImpl
typedef struct SReadH SReadH;
struct TSDBKEY {
int64_t version;
TSKEY ts;
};
typedef struct {
uint64_t suid;
uint64_t uid;
......@@ -354,7 +384,7 @@ typedef struct {
typedef struct {
int32_t delimiter; // For recovery usage
int32_t tid;
uint64_t suid;
uint64_t uid;
SBlock blocks[];
} SBlockInfo;
......@@ -650,11 +680,6 @@ struct SFSIter {
#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
struct TSDBROW {
int64_t version;
STSRow2 tsRow;
};
struct TABLEID {
tb_uid_t suid;
tb_uid_t uid;
......@@ -675,7 +700,7 @@ typedef struct {
TSKEY eKey;
} SDelInfo;
struct SMemTable {
struct SMemTable2 {
STsdb *pTsdb;
int32_t nRef;
TSDBKEY minKey;
......@@ -705,16 +730,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
return 0;
}
typedef struct SMemSkipListNode SMemSkipListNode;
typedef struct SMemSkipList {
uint32_t seed;
int32_t size;
int8_t maxLevel;
int8_t level;
SMemSkipListNode *pHead;
SMemSkipListNode *pTail;
} SMemSkipList;
struct SMemData {
tb_uid_t suid;
tb_uid_t uid;
......@@ -726,13 +741,19 @@ struct SMemData {
};
struct SMemDataIter {
SMemData *pMemData;
STbData *pMemData;
int8_t backward;
TSDBROW *pRow;
SMemSkipListNode *pNode; // current node
TSDBROW row;
};
struct STbDataIter {
STbData *pTbData;
int8_t backward;
SMemSkipListNode *pNode;
};
#endif
#ifdef __cplusplus
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ 9aa1e1d3
......@@ -116,7 +116,9 @@ int tsdbBegin(STsdb* pTsdb);
int32_t tsdbCommit(STsdb* pTsdb);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
SSubmitBlkRsp* pRsp);
int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
uint64_t taskId);
tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
......
source/dnode/vnode/src/meta/metaQuery.c
浏览文件 @ 9aa1e1d3
......@@ -31,7 +31,7 @@ void metaReaderClear(SMetaReader *pReader) {
}
int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
STbDbKey tbDbKey = {.version = version, .uid = uid};
// query table.db
......@@ -54,7 +54,7 @@ _err:
}
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
int64_t version;
// query uid.idx
......@@ -68,7 +68,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
}
int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
SMeta * pMeta = pReader->pMeta;
SMeta *pMeta = pReader->pMeta;
tb_uid_t uid;
// query name.idx
......@@ -82,7 +82,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
}
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
void * pData = NULL;
void *pData = NULL;
int nData = 0;
tb_uid_t uid = 0;
......@@ -134,7 +134,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
int metaTbCursorNext(SMTbCursor *pTbCur) {
int ret;
void * pBuf;
void *pBuf;
STbCfg tbCfg;
for (;;) {
......@@ -155,7 +155,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
void * pData = NULL;
void *pData = NULL;
int nData = 0;
int64_t version;
SSchemaWrapper schema = {0};
......@@ -163,37 +163,47 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
SDecoder dc = {0};
metaRLock(pMeta);
if (sver < 0) {
if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) {
goto _err;
}
version = *(int64_t *)pData;
_query:
if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) {
goto _err;
}
tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData);
version = *(int64_t *)pData;
SMetaEntry me = {0};
tDecoderInit(&dc, pData, nData);
metaDecodeEntry(&dc, &me);
if (me.type == TSDB_SUPER_TABLE) {
tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData);
SMetaEntry me = {0};
tDecoderInit(&dc, pData, nData);
metaDecodeEntry(&dc, &me);
if (me.type == TSDB_SUPER_TABLE) {
if (sver == -1 || sver == me.stbEntry.schemaRow.version) {
pSchema = tCloneSSchemaWrapper(&me.stbEntry.schemaRow);
} else if (me.type == TSDB_NORMAL_TABLE) {
pSchema = tCloneSSchemaWrapper(&me.ntbEntry.schemaRow);
} else {
ASSERT(0);
tDecoderClear(&dc);
goto _exit;
}
} else if (me.type == TSDB_CHILD_TABLE) {
uid = me.ctbEntry.suid;
tDecoderClear(&dc);
goto _query;
} else {
if (tdbTbGet(pMeta->pSkmDb, &(SSkmDbKey){.uid = uid, .sver = sver}, sizeof(SSkmDbKey), &pData, &nData) < 0) {
goto _err;
if (sver == -1 || sver == me.ntbEntry.schemaRow.version) {
pSchema = tCloneSSchemaWrapper(&me.ntbEntry.schemaRow);
tDecoderClear(&dc);
goto _exit;
}
}
tDecoderClear(&dc);
tDecoderInit(&dc, pData, nData);
tDecodeSSchemaWrapper(&dc, &schema);
pSchema = tCloneSSchemaWrapper(&schema);
tDecoderClear(&dc);
// query from skm db
if (tdbTbGet(pMeta->pSkmDb, &(SSkmDbKey){.uid = uid, .sver = sver}, sizeof(SSkmDbKey), &pData, &nData) < 0) {
goto _err;
}
tDecoderInit(&dc, pData, nData);
tDecodeSSchemaWrapper(&dc, &schema);
pSchema = tCloneSSchemaWrapper(&schema);
tDecoderClear(&dc);
_exit:
metaULock(pMeta);
tdbFree(pData);
return pSchema;
......@@ -205,11 +215,11 @@ _err:
}
struct SMCtbCursor {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
......@@ -279,25 +289,13 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
tb_uid_t quid;
SMetaReader mr = {0};
STSchema * pTSchema = NULL;
// SMetaReader mr = {0};
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
STSchemaBuilder sb = {0};
SSchema * pSchema;
metaReaderInit(&mr, pMeta, 0);
metaGetTableEntryByUid(&mr, uid);
if (mr.me.type == TSDB_CHILD_TABLE) {
quid = mr.me.ctbEntry.suid;
} else {
quid = uid;
}
metaReaderClear(&mr);
SSchema *pSchema;
pSW = metaGetTableSchema(pMeta, quid, sver, 0);
pSW = metaGetTableSchema(pMeta, uid, sver, 0);
if (!pSW) return NULL;
tdInitTSchemaBuilder(&sb, pSW->version);
......@@ -321,11 +319,11 @@ int metaGetTbNum(SMeta *pMeta) {
}
typedef struct {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t uid;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int kLen;
int vLen;
} SMSmaCursor;
......@@ -397,7 +395,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) {
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
STSmaWrapper *pSW = NULL;
SArray * pSmaIds = NULL;
SArray *pSmaIds = NULL;
if (!(pSmaIds = metaGetSmaIdsByTable(pMeta, uid))) {
return NULL;
......@@ -421,7 +419,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
metaReaderInit(&mr, pMeta, 0);
int64_t smaId;
int smaIdx = 0;
STSma * pTSma = NULL;
STSma *pTSma = NULL;
for (int i = 0; i < pSW->number; ++i) {
smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i);
if (metaGetTableEntryByUid(&mr, smaId) < 0) {
......@@ -469,7 +467,7 @@ _err:
}
STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
STSma * pTSma = NULL;
STSma *pTSma = NULL;
SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByUid(&mr, indexUid) < 0) {
......@@ -491,7 +489,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
}
SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
SArray * pUids = NULL;
SArray *pUids = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
......@@ -529,7 +527,7 @@ SArray *metaGetSmaIdsByTable(SMeta *pMeta, tb_uid_t uid) {
}
SArray *metaGetSmaTbUids(SMeta *pMeta) {
SArray * pUids = NULL;
SArray *pUids = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t lastUid = 0;
......@@ -591,13 +589,13 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) {
}
typedef struct {
SMeta * pMeta;
TBC * pCur;
SMeta *pMeta;
TBC *pCur;
tb_uid_t suid;
int16_t cid;
int16_t type;
void * pKey;
void * pVal;
void *pKey;
void *pVal;
int32_t kLen;
int32_t vLen;
} SIdxCursor;
......@@ -621,7 +619,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
int32_t nKey = 0;
int32_t nTagData = 0;
void * tagData = NULL;
void *tagData = NULL;
if (IS_VAR_DATA_TYPE(param->type)) {
tagData = varDataVal(param->val);
......@@ -640,7 +638,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
goto END;
}
void * entryKey = NULL, *entryVal = NULL;
void *entryKey = NULL, *entryVal = NULL;
int32_t nEntryKey, nEntryVal;
while (1) {
valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal);
......
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @ 9aa1e1d3
......@@ -141,10 +141,10 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
}
// this interface use suid instead of uid
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true);
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true);
if (pHandle->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
pHandle->msgIter.suid, pHandle->cachedSchemaVer);
pHandle->msgIter.uid, pHandle->cachedSchemaVer);
/*ASSERT(0);*/
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ 9aa1e1d3
......@@ -18,8 +18,8 @@
#define TSDB_MAX_SUBBLOCKS 8
typedef struct {
STable *pTable;
SSkipListIterator *pIter;
STable *pTable;
STbDataIter *pIter;
} SCommitIter;
typedef struct {
......@@ -58,26 +58,26 @@ typedef struct {
#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows)
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
static void tsdbStartCommit(STsdb *pRepo);
static void tsdbEndCommit(STsdb *pTsdb, int eno);
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
static int tsdbNextCommitFid(SCommitH *pCommith);
static void tsdbDestroyCommitH(SCommitH *pCommith);
static int tsdbCreateCommitIters(SCommitH *pCommith);
static void tsdbDestroyCommitIters(SCommitH *pCommith);
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static void tsdbResetCommitFile(SCommitH *pCommith);
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static int tsdbCommitToTable(SCommitH *pCommith, int tid);
static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
static int tsdbComparKeyBlock(const void *arg1, const void *arg2);
static int tsdbWriteBlockInfo(SCommitH *pCommih);
static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
static int tsdbMoveBlock(SCommitH *pCommith, int bidx);
static void tsdbStartCommit(STsdb *pRepo);
static void tsdbEndCommit(STsdb *pTsdb, int eno);
static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
static int tsdbNextCommitFid(SCommitH *pCommith);
static void tsdbDestroyCommitH(SCommitH *pCommith);
static int32_t tsdbCreateCommitIters(SCommitH *pCommith);
static void tsdbDestroyCommitIters(SCommitH *pCommith);
static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static void tsdbResetCommitFile(SCommitH *pCommith);
static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
static int tsdbCommitToTable(SCommitH *pCommith, int tid);
static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
static int tsdbComparKeyBlock(const void *arg1, const void *arg2);
static int tsdbWriteBlockInfo(SCommitH *pCommih);
static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
static int tsdbMoveBlock(SCommitH *pCommith, int bidx);
static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks);
static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit,
bool isLastOneBlock);
......@@ -92,7 +92,7 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
int tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return 0;
STsdbMemTable *pMem;
SMemTable *pMem;
if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {
return -1;
......@@ -244,7 +244,7 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
}
static void tsdbStartCommit(STsdb *pRepo) {
STsdbMemTable *pMem = pRepo->imem;
SMemTable *pMem = pRepo->imem;
tsdbInfo("vgId:%d, start to commit", REPO_ID(pRepo));
......@@ -400,7 +400,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
break;
}
if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->uid <= pIdx->uid))) {
if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->suid <= pIdx->suid || pIter->pTable->uid <= pIdx->uid))) {
if (tsdbCommitToTable(pCommith, mIter) < 0) {
tsdbCloseCommitFile(pCommith, true);
// revert the file change
......@@ -453,57 +453,48 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
return 0;
}
static int tsdbCreateCommitIters(SCommitH *pCommith) {
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
STsdbMemTable *pMem = pRepo->imem;
SSkipListIterator *pSlIter;
SCommitIter *pCommitIter;
SSkipListNode *pNode;
STbData *pTbData;
STSchema *pTSchema = NULL;
static int32_t tsdbCreateCommitIters(SCommitH *pCommith) {
int32_t code = 0;
STsdb *pRepo = TSDB_COMMIT_REPO(pCommith);
SMemTable *pMem = pRepo->imem;
STbData *pTbData;
SCommitIter *pCommitIter;
STSchema *pTSchema = NULL;
pCommith->niters = SL_SIZE(pMem->pSlIdx);
pCommith->niters = taosArrayGetSize(pMem->aTbData);
pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter));
if (pCommith->iters == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
// Loop to create iters for each skiplist
pSlIter = tSkipListCreateIter(pMem->pSlIdx);
if (pSlIter == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
for (int i = 0; i < pCommith->niters; i++) {
tSkipListIterNext(pSlIter);
pNode = tSkipListIterGet(pSlIter);
pTbData = (STbData *)pNode->pData;
for (int32_t iIter = 0; iIter < pCommith->niters; iIter++) {
pTbData = (STbData *)taosArrayGetP(pMem->aTbData, iIter);
pCommitIter = &pCommith->iters[iIter];
pCommitIter = pCommith->iters + i;
pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, -1);
if (pTSchema) {
pCommitIter->pIter = tSkipListCreateIter(pTbData->pData);
tSkipListIterNext(pCommitIter->pIter);
tsdbTbDataIterCreate(pTbData, NULL, 0, &pCommitIter->pIter);
pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable));
pCommitIter->pTable->uid = pTbData->uid;
pCommitIter->pTable->tid = pTbData->uid;
pCommitIter->pTable->suid = pTbData->suid;
pCommitIter->pTable->pSchema = pTSchema;
pCommitIter->pTable->pCacheSchema = NULL;
}
}
tSkipListDestroyIter(pSlIter);
return 0;
return code;
_err:
return code;
}
static void tsdbDestroyCommitIters(SCommitH *pCommith) {
if (pCommith->iters == NULL) return;
for (int i = 1; i < pCommith->niters; i++) {
tSkipListDestroyIter(pCommith->iters[i].pIter);
tsdbTbDataIterDestroy(pCommith->iters[i].pIter);
if (pCommith->iters[i].pTable) {
tdFreeSchema(pCommith->iters[i].pTable->pSchema);
tdFreeSchema(pCommith->iters[i].pTable->pCacheSchema);
......@@ -743,8 +734,8 @@ static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA,
pBlkInfo = *ppBuf;
pBlkInfo->delimiter = TSDB_FILE_DELIMITER;
pBlkInfo->tid = TABLE_TID(pTable);
pBlkInfo->uid = TABLE_UID(pTable);
pBlkInfo->suid = pTable->suid;
pBlkInfo->uid = pTable->uid;
memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock));
if (nSubBlocks > 0) {
......@@ -770,7 +761,8 @@ static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA,
// Set pIdx
pBlock = taosArrayGetLast(pSupA);
pIdx->uid = TABLE_UID(pTable);
pIdx->suid = pTable->suid;
pIdx->uid = pTable->uid;
pIdx->hasLast = pBlock->last ? 1 : 0;
pIdx->maxKey = pBlock->maxKey;
pIdx->numOfBlocks = (uint32_t)nSupBlocks;
......@@ -925,7 +917,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
return -1;
}
STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL};
STable table = {.suid = pIdx->suid, .uid = pIdx->uid, .pSchema = NULL};
pCommith->pTable = &table;
while (bidx < nBlocks) {
......@@ -1186,7 +1178,7 @@ static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFi
}
pBlockData->delimiter = TSDB_FILE_DELIMITER;
pBlockData->uid = TABLE_UID(pTable);
pBlockData->uid = pTable->uid;
pBlockData->numOfCols = nColsNotAllNull;
taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize);
......@@ -1226,7 +1218,7 @@ static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFi
tsdbDebug("vgId:%d, uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64
" numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
REPO_ID(pRepo), pTable->uid, TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
pBlock->numOfCols, pBlock->minKey.ts, pBlock->maxKey.ts);
return 0;
......@@ -1313,7 +1305,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
keyLimit = pBlock[1].minKey.ts - 1;
}
SSkipListIterator titer = *(pIter->pIter);
STbDataIter titer = *(pIter->pIter);
if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1, false) < 0) return -1;
tsdbLoadDataFromCache(TSDB_COMMIT_REPO(pCommith), pIter->pTable, &titer, keyLimit, INT32_MAX, NULL,
......@@ -1522,7 +1514,7 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i
lastKey = key2;
}
tSkipListIterNext(pCommitIter->pIter);
tsdbTbDataIterNext(pCommitIter->pIter);
} else {
if (lastKey != key1) {
if (lastKey != TSKEY_INITIAL_VAL) {
......@@ -1554,7 +1546,7 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i
tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
}
++(*iter);
tSkipListIterNext(pCommitIter->pIter);
tsdbTbDataIterNext(pCommitIter->pIter);
}
if (pTarget->numOfRows >= (maxRows - 1)) break;
......
source/dnode/vnode/src/tsdb/tsdbCommit2.c
浏览文件 @ 9aa1e1d3
......@@ -16,19 +16,19 @@
#include "tsdb.h"
typedef struct {
SMemTable *pMemTable;
int32_t minutes;
int8_t precision;
TSKEY nCommitKey;
int32_t fid;
TSKEY minKey;
TSKEY maxKey;
SReadH readh;
SDFileSet wSet;
SArray *aBlkIdx;
SArray *aSupBlk;
SArray *aSubBlk;
SArray *aDelInfo;
SMemTable2 *pMemTable;
int32_t minutes;
int8_t precision;
TSKEY nCommitKey;
int32_t fid;
TSKEY minKey;
TSKEY maxKey;
SReadH readh;
SDFileSet wSet;
SArray *aBlkIdx;
SArray *aSupBlk;
SArray *aSubBlk;
SArray *aDelInfo;
} SCommitH;
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb);
......@@ -39,7 +39,7 @@ int32_t tsdbBegin2(STsdb *pTsdb) {
int32_t code = 0;
ASSERT(pTsdb->mem == NULL);
code = tsdbMemTableCreate2(pTsdb, (SMemTable **)&pTsdb->mem);
code = tsdbMemTableCreate2(pTsdb, (SMemTable2 **)&pTsdb->mem);
if (code) {
tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
goto _exit;
......@@ -80,8 +80,8 @@ _err:
}
static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) {
int32_t code = 0;
SMemTable *pMemTable = (SMemTable *)pTsdb->mem;
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;
tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode));
......@@ -131,9 +131,9 @@ _err:
}
static int32_t tsdbCommitEnd(SCommitH *pCHandle) {
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SMemTable *pMemTable = (SMemTable *)pTsdb->imem;
int32_t code = 0;
STsdb *pTsdb = pCHandle->pMemTable->pTsdb;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->imem;
// end transaction
code = tsdbEndFSTxn(pTsdb);
......
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ 9aa1e1d3
......@@ -25,8 +25,7 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"meta", // TSDB_FILE_META
};
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname);
// static int tsdbRollBackMFile(SMFile *pMFile);
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname);
static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
static int tsdbRollBackDFile(SDFile *pDFile);
......
source/dnode/vnode/src/tsdb/tsdbMemTable2.c
浏览文件 @ 9aa1e1d3
......@@ -15,11 +15,6 @@
#include "tsdb.h"
struct SMemSkipListNode {
int8_t level;
SMemSkipListNode *forwards[0];
};
typedef struct {
tb_uid_t uid;
STSchema *pTSchema;
......@@ -35,21 +30,21 @@ typedef struct {
#define SL_MOVE_BACKWARD 0x1
#define SL_MOVE_FROM_POS 0x2
static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
static int memDataPCmprFn(const void *p1, const void *p2);
static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version,
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk);
static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
// SMemTable ==============================================
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) {
int32_t code = 0;
SMemTable *pMemTable = NULL;
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable) {
int32_t code = 0;
SMemTable2 *pMemTable = NULL;
pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
pMemTable = (SMemTable2 *)taosMemoryCalloc(1, sizeof(*pMemTable));
if (pMemTable == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
......@@ -75,16 +70,16 @@ _err:
return code;
}
void tsdbMemTableDestroy2(SMemTable *pMemTable) {
void tsdbMemTableDestroy2(SMemTable2 *pMemTable) {
taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/);
taosMemoryFree(pMemTable);
}
int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO
SMemData *pMemData;
TSDBROW row = {.version = version};
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
TSDBROW row = {.version = version};
ASSERT(pMemTable);
ASSERT(pSubmitBlk->nData > 0);
......@@ -112,10 +107,10 @@ _err:
}
int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
int32_t code = 0;
SMemTable *pMemTable = (SMemTable *)pTsdb->mem; // TODO
SMemData *pMemData;
SVBufPool *pPool = pTsdb->pVnode->inUse;
int32_t code = 0;
SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem; // TODO
SMemData *pMemData;
SVBufPool *pPool = pTsdb->pVnode->inUse;
ASSERT(pMemTable);
......@@ -250,7 +245,7 @@ void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) {
}
}
static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
int32_t code = 0;
int32_t idx = 0;
SMemData *pMemDataT = &(SMemData){.suid = suid, .uid = uid};
......@@ -421,7 +416,7 @@ static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY
}
}
static int32_t memDataDoPut(SMemTable *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
static int32_t memDataDoPut(SMemTable2 *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
int8_t forward) {
int32_t code = 0;
int8_t level;
......@@ -475,7 +470,7 @@ _exit:
return code;
}
static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version,
static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
SVSubmitBlk *pSubmitBlk) {
int32_t code = 0;
int32_t n = 0;
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 9aa1e1d3
......@@ -67,15 +67,16 @@ enum {
};
typedef struct STableCheckInfo {
uint64_t tableId;
TSKEY lastKey;
SBlockInfo* pCompInfo;
int32_t compSize;
int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks
uint8_t chosen : 2; // indicate which iterator should move forward
bool initBuf : 1; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator
uint64_t suid;
uint64_t tableId;
TSKEY lastKey;
SBlockInfo* pCompInfo;
int32_t compSize;
int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks
uint8_t chosen : 2; // indicate which iterator should move forward
bool initBuf : 1; // whether to initialize the in-memory skip list iterator or not
STbDataIter* iter; // mem buffer skip list iterator
STbDataIter* iiter; // imem buffer skip list iterator
} STableCheckInfo;
typedef struct STableBlockInfo {
......@@ -107,6 +108,7 @@ typedef struct SBlockLoadSuppInfo {
typedef struct STsdbReadHandle {
STsdb* pTsdb;
uint64_t suid;
SQueryFilePos cur; // current position
int16_t order;
STimeWindow window; // the primary query time window that applies to all queries
......@@ -200,8 +202,8 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle;
int64_t rows = 0;
STsdbMemTable* pMemTable = NULL; // pTsdbReadHandle->pMemTable;
int64_t rows = 0;
SMemTable* pMemTable = NULL; // pTsdbReadHandle->pMemTable;
if (pMemTable == NULL) {
return rows;
}
......@@ -237,6 +239,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j);
STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid};
info.suid = pTsdbReadHandle->suid;
if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) {
info.lastKey = pTsdbReadHandle->window.skey;
......@@ -265,8 +268,8 @@ static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) {
for (int32_t i = 0; i < numOfTables; ++i) {
STableCheckInfo* pCheckInfo = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
pCheckInfo->lastKey = pTsdbReadHandle->window.skey;
pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter);
pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter);
pCheckInfo->iter = tsdbTbDataIterDestroy(pCheckInfo->iter);
pCheckInfo->iiter = tsdbTbDataIterDestroy(pCheckInfo->iiter);
pCheckInfo->initBuf = false;
if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
......@@ -387,6 +390,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
pReadHandle->locateStart = false;
pReadHandle->loadType = pCond->type;
pReadHandle->suid = pCond->suid;
pReadHandle->outputCapacity = 4096; //((STsdb*)tsdb)->config.maxRowsPerFileBlock;
pReadHandle->loadExternalRow = pCond->loadExternalRows;
pReadHandle->currentLoadExternalRows = pCond->loadExternalRows;
......@@ -658,7 +662,7 @@ tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableL
}
#if 0
tsdbReaderT tsdbQueryCacheLastT(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
tsdbReaderT tsdbQueryCacheLastT(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemTable* pMemRef) {
STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTablesT(tsdb, pCond, groupList, qId, pMemRef);
if (pTsdbReadHandle == NULL) {
return NULL;
......@@ -752,23 +756,22 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
pCheckInfo->initBuf = true;
int32_t order = pHandle->order;
STbData** pMem = NULL;
STbData** pIMem = NULL;
STbData* pMem = NULL;
STbData* pIMem = NULL;
int8_t backward = (pHandle->order == TSDB_ORDER_DESC) ? 1 : 0;
TSKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
if (pHandle->pTsdb->mem != NULL) {
pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
tsdbGetTbDataFromMemTable(pHandle->pTsdb->mem, pCheckInfo->suid, pCheckInfo->tableId, &pMem);
if (pMem != NULL) {
pCheckInfo->iter =
tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
tsdbTbDataIterCreate(pMem, &(TSDBKEY){.version = 0, .ts = tLastKey}, backward, &pCheckInfo->iter);
}
}
if (pHandle->pTsdb->imem != NULL) {
pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
tsdbGetTbDataFromMemTable(pHandle->pTsdb->mem, pCheckInfo->suid, pCheckInfo->tableId, &pIMem);
if (pIMem != NULL) {
pCheckInfo->iiter =
tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
tsdbTbDataIterCreate(pIMem, &(TSDBKEY){.version = 0, .ts = tLastKey}, backward, &pCheckInfo->iiter);
}
}
......@@ -777,22 +780,23 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
return false;
}
bool memEmpty = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
bool memEmpty =
(pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tsdbTbDataIterGet(pCheckInfo->iter, NULL));
bool imemEmpty =
(pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tsdbTbDataIterGet(pCheckInfo->iiter, NULL));
if (memEmpty && imemEmpty) { // buffer is empty
return false;
}
if (!memEmpty) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL);
TSDBROW row;
STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
TSKEY key = TD_ROW_KEY(row); // first timestamp in buffer
tsdbTbDataIterGet(pCheckInfo->iter, &row);
TSKEY key = row.pTSRow->ts; // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%" PRId64 ", %s",
pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey,
(*pMem)->nrows, pHandle->idStr);
pHandle, pCheckInfo->tableId, key, order, pMem->minKey.ts, pMem->maxKey.ts, pCheckInfo->lastKey,
pMem->sl.size, pHandle->idStr);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
......@@ -805,15 +809,14 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
}
if (!imemEmpty) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL);
TSDBROW row;
STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
TSKEY key = TD_ROW_KEY(row); // first timestamp in buffer
tsdbTbDataIterGet(pCheckInfo->iter, &row);
TSKEY key = row.pTSRow->ts; // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%" PRId64 ", %s",
pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey,
(*pIMem)->nrows, pHandle->idStr);
pHandle, pCheckInfo->tableId, key, order, pIMem->minKey.ts, pIMem->maxKey.ts, pCheckInfo->lastKey,
pIMem->sl.size, pHandle->idStr);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
......@@ -828,31 +831,23 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
}
static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iter);
tSkipListDestroyIter(pCheckInfo->iiter);
tsdbTbDataIterDestroy(pCheckInfo->iter);
tsdbTbDataIterDestroy(pCheckInfo->iiter);
}
static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) {
TSDBROW row = {0};
STSRow *rmem = NULL, *rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = (STSRow*)SL_GET_NODE_DATA(node);
// TODO: filter max version
// if (TD_ROW_VER(rmem) > maxVer) {
// rmem = NULL;
// }
if (tsdbTbDataIterGet(pCheckInfo->iter, &row)) {
rmem = row.pTSRow;
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = (STSRow*)SL_GET_NODE_DATA(node);
// TODO: filter max version
// if (TD_ROW_VER(rimem) > maxVer) {
// rimem = NULL;
// }
if (tsdbTbDataIterGet(pCheckInfo->iiter, &row)) {
rimem = row.pTSRow;
}
}
......@@ -889,7 +884,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
} else {
pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
tSkipListIterNext(pCheckInfo->iter);
tsdbTbDataIterNext(pCheckInfo->iter);
}
return r1;
} else if (r1 < r2 && ASCENDING_TRAVERSE(order)) {
......@@ -903,28 +898,17 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow,
TDRowVerT maxVer) {
TSDBROW row;
STSRow *rmem = NULL, *rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = (STSRow*)SL_GET_NODE_DATA(node);
#if 0 // TODO: skiplist refactor
if (TD_ROW_VER(rmem) > maxVer) {
rmem = NULL;
}
#endif
if (tsdbTbDataIterGet(pCheckInfo->iter, &row)) {
rmem = row.pTSRow;
}
}
if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = (STSRow*)SL_GET_NODE_DATA(node);
#if 0 // TODO: skiplist refactor
if (TD_ROW_VER(rimem) > maxVer) {
rimem = NULL;
}
#endif
if (tsdbTbDataIterGet(pCheckInfo->iiter, &row)) {
rimem = row.pTSRow;
}
}
......@@ -966,7 +950,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
*extraRow = rimem;
return rmem;
} else {
tSkipListIterNext(pCheckInfo->iter);
tsdbTbDataIterNext(pCheckInfo->iter);
pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
return rimem;
}
......@@ -995,7 +979,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
bool hasNext = false;
if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) {
if (pCheckInfo->iter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iter);
hasNext = tsdbTbDataIterNext(pCheckInfo->iter);
}
if (hasNext) {
......@@ -1003,11 +987,11 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
if (pCheckInfo->iiter != NULL) {
return tSkipListIterGet(pCheckInfo->iiter) != NULL;
return tsdbTbDataIterGet(pCheckInfo->iiter, NULL);
}
} else if (pCheckInfo->chosen == CHECKINFO_CHOSEN_IMEM) {
if (pCheckInfo->iiter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iiter);
hasNext = tsdbTbDataIterNext(pCheckInfo->iiter);
}
if (hasNext) {
......@@ -1015,14 +999,14 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
}
if (pCheckInfo->iter != NULL) {
return tSkipListIterGet(pCheckInfo->iter) != NULL;
return tsdbTbDataIterGet(pCheckInfo->iter, NULL);
}
} else {
if (pCheckInfo->iter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iter);
hasNext = tsdbTbDataIterNext(pCheckInfo->iter);
}
if (pCheckInfo->iiter != NULL) {
hasNext = tSkipListIterNext(pCheckInfo->iiter) || hasNext;
hasNext = tsdbTbDataIterNext(pCheckInfo->iiter) || hasNext;
}
}
......@@ -1126,7 +1110,7 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in
STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index);
pCheckInfo->numOfBlocks = 0;
STable table = {.uid = pCheckInfo->tableId, .tid = pCheckInfo->tableId};
STable table = {.uid = pCheckInfo->tableId, .suid = pCheckInfo->suid};
table.pSchema = pTsdbReadHandle->pSchema;
if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, &table) != TSDB_CODE_SUCCESS) {
......@@ -2924,7 +2908,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) {
// current result is empty
if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey &&
pTsdbReadHandle->cur.rows == 0) {
// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable;
// SMemTable* pMemRef = pTsdbReadHandle->pMemTable;
// doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef);
// doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef);
......@@ -3222,7 +3206,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
}
}
// static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) {
// static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, SMemTable* pMemRef) {
// STsdbReadHandle* pSecQueryHandle = NULL;
//
// if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) {
......
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
浏览文件 @ 9aa1e1d3
......@@ -156,6 +156,24 @@ int tsdbLoadBlockIdx(SReadH *pReadh) {
return 0;
}
static int32_t tsdbBlockIdxCmprFn(const void *p1, const void *p2) {
SBlockIdx *pBlockIdx1 = (SBlockIdx *)p1;
SBlockIdx *pBlockIdx2 = (SBlockIdx *)p2;
if (pBlockIdx1->suid < pBlockIdx2->suid) {
return -1;
} else if (pBlockIdx1->suid > pBlockIdx2->suid) {
return 1;
}
if (pBlockIdx1->uid < pBlockIdx2->uid) {
return -1;
} else if (pBlockIdx1->uid > pBlockIdx2->uid) {
return 1;
}
return 0;
}
int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
STSchema *pSchema = tsdbGetTableSchemaImpl(TSDB_READ_REPO(pReadh), pTable, false, false, -1);
......@@ -171,33 +189,40 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
return -1;
}
size_t size = taosArrayGetSize(pReadh->aBlkIdx);
if (size > 0) {
while (true) {
if (pReadh->cidx >= size) {
pReadh->pBlkIdx = NULL;
break;
}
SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
if (pBlkIdx->uid == TABLE_TID(pTable)) {
if (pBlkIdx->uid == TABLE_UID(pTable)) {
pReadh->pBlkIdx = pBlkIdx;
} else {
pReadh->pBlkIdx = NULL;
}
pReadh->cidx++;
break;
} else if (pBlkIdx->uid > TABLE_TID(pTable)) {
pReadh->pBlkIdx = NULL;
break;
} else {
pReadh->cidx++;
}
}
} else {
uint8_t *p = taosArraySearch(pReadh->aBlkIdx, &(SBlockIdx){.suid = pTable->suid, .uid = pTable->uid},
tsdbBlockIdxCmprFn, TD_EQ);
if (p == NULL) {
pReadh->pBlkIdx = NULL;
}
} else {
pReadh->pBlkIdx = (SBlockIdx *)p;
}
// size_t size = taosArrayGetSize(pReadh->aBlkIdx);
// if (size > 0) {
// while (true) {
// if (pReadh->cidx >= size) {
// pReadh->pBlkIdx = NULL;
// break;
// }
// SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
// if (pBlkIdx->uid == TABLE_TID(pTable)) {
// if (pBlkIdx->uid == TABLE_UID(pTable)) {
// pReadh->pBlkIdx = pBlkIdx;
// } else {
// pReadh->pBlkIdx = NULL;
// }
// pReadh->cidx++;
// break;
// } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
// pReadh->pBlkIdx = NULL;
// break;
// } else {
// pReadh->cidx++;
// }
// }
// } else {
// pReadh->pBlkIdx = NULL;
// }
return 0;
}
......@@ -553,12 +578,12 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
int tlen = 0;
// tlen += taosEncodeVariantI32(buf, pIdx->tid);
tlen += taosEncodeFixedU64(buf, pIdx->suid);
tlen += taosEncodeFixedU64(buf, pIdx->uid);
tlen += taosEncodeVariantU32(buf, pIdx->len);
tlen += taosEncodeVariantU32(buf, pIdx->offset);
tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
tlen += taosEncodeFixedU64(buf, pIdx->uid);
tlen += taosEncodeFixedU64(buf, pIdx->maxKey.ts);
return tlen;
......@@ -570,6 +595,10 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
uint64_t value = 0;
// if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->suid = (int64_t)value;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->uid = (int64_t)value;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
......@@ -577,8 +606,6 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
pIdx->numOfBlocks = numOfBlocks;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->uid = (int64_t)value;
if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
pIdx->maxKey.ts = (TSKEY)value;
return buf;
......
source/dnode/vnode/src/tsdb/tsdbWrite.c
浏览文件 @ 9aa1e1d3
......@@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
SSubmitBlkRsp r = {0};
tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &r) < 0) {
if (tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r) < 0) {
return -1;
}
......
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @ 9aa1e1d3
......@@ -799,7 +799,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname);
}
if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) {
if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) {
submitBlkRsp.code = terrno;
}
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 9aa1e1d3
......@@ -4760,6 +4760,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
pCond->numOfTWindows = 1;
pCond->twindows = taosMemoryCalloc(pCond->numOfTWindows, sizeof(STimeWindow));
pCond->twindows[0] = pTableScanNode->scanRange;
pCond->suid = pTableScanNode->scan.suid;
#if 1
// todo work around a problem, remove it later
......
source/libs/nodes/src/nodesCloneFuncs.c
浏览文件 @ 9aa1e1d3
......@@ -449,6 +449,34 @@ static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
return (SNode*)pDst;
}
static SNode* physiNodeCopy(const SPhysiNode* pSrc, SPhysiNode* pDst) {
CLONE_NODE_FIELD(pOutputDataBlockDesc);
CLONE_NODE_FIELD(pConditions);
CLONE_NODE_LIST_FIELD(pChildren);
return (SNode*)pDst;
}
static SNode* physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, physiNodeCopy);
CLONE_NODE_LIST_FIELD(pExprs);
CLONE_NODE_LIST_FIELD(pFuncs);
CLONE_NODE_FIELD(pTspk);
COPY_SCALAR_FIELD(triggerType);
COPY_SCALAR_FIELD(watermark);
COPY_SCALAR_FIELD(filesFactor);
return (SNode*)pDst;
}
static SNode* physiIntervalCopy(const SIntervalPhysiNode* pSrc, SIntervalPhysiNode* pDst) {
COPY_BASE_OBJECT_FIELD(window, physiWindowCopy);
COPY_SCALAR_FIELD(interval);
COPY_SCALAR_FIELD(offset);
COPY_SCALAR_FIELD(sliding);
COPY_SCALAR_FIELD(intervalUnit);
COPY_SCALAR_FIELD(slidingUnit);
return (SNode*)pDst;
}
static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId);
CLONE_NODE_LIST_FIELD(pSlots);
......@@ -575,6 +603,12 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
return logicIndefRowsFuncCopy((const SIndefRowsFuncLogicNode*)pNode, (SIndefRowsFuncLogicNode*)pDst);
case QUERY_NODE_LOGIC_SUBPLAN:
return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst);
case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
return physiIntervalCopy((const SIntervalPhysiNode*)pNode, (SIntervalPhysiNode*)pDst);
default:
break;
}
......
source/libs/nodes/src/nodesUtilFuncs.c
浏览文件 @ 9aa1e1d3
......@@ -164,9 +164,14 @@ SNodeptr nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateStreamStmt));
case QUERY_NODE_DROP_STREAM_STMT:
return makeNode(type, sizeof(SDropStreamStmt));
case QUERY_NODE_BALANCE_VGROUP_STMT:
return makeNode(type, sizeof(SBalanceVgroupStmt));
case QUERY_NODE_MERGE_VGROUP_STMT:
return makeNode(type, sizeof(SMergeVgroupStmt));
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
return makeNode(type, sizeof(SRedistributeVgroupStmt));
case QUERY_NODE_SPLIT_VGROUP_STMT:
return makeNode(type, sizeof(SSplitVgroupStmt));
case QUERY_NODE_SYNCDB_STMT:
break;
case QUERY_NODE_GRANT_STMT:
......
source/libs/parser/inc/parAst.h
浏览文件 @ 9aa1e1d3
......@@ -187,6 +187,7 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const
SNode* pOptions, SNode* pQuery);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName);
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);
......
source/libs/parser/inc/parInsertData.h
浏览文件 @ 9aa1e1d3
......@@ -116,8 +116,7 @@ static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo
}
static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows) {
pBlocks->suid =
(TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid);
pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid);
pBlocks->uid = dataBuf->pTableMeta->uid;
pBlocks->sversion = dataBuf->pTableMeta->sversion;
pBlocks->schemaLen = dataBuf->createTbReqLen;
......
source/libs/parser/inc/sql.y
浏览文件 @ 9aa1e1d3
......@@ -469,9 +469,10 @@ cmd ::= KILL QUERY NK_INTEGER(A).
cmd ::= KILL TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &A); }
/************************************************ merge/redistribute/ vgroup ******************************************/
cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B). { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
cmd ::= SPLIT VGROUP NK_INTEGER(A). { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
//cmd ::= SPLIT VGROUP NK_INTEGER(A). { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
%type dnode_list { SNodeList* }
%destructor dnode_list { nodesDestroyList($$); }
......
source/libs/parser/src/parAstCreater.c
浏览文件 @ 9aa1e1d3
......@@ -1435,25 +1435,37 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId
return (SNode*)pStmt;
}
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt);
SBalanceVgroupStmt* pStmt = nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
return (SNode*)pStmt;
}
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
CHECK_PARSER_STATUS(pCxt);
SNode* pStmt = nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);
SMergeVgroupStmt* pStmt = nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
return pStmt;
pStmt->vgId1 = taosStr2Int32(pVgId1->z, NULL, 10);
pStmt->vgId2 = taosStr2Int32(pVgId2->z, NULL, 10);
return (SNode*)pStmt;
}
SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes) {
CHECK_PARSER_STATUS(pCxt);
SNode* pStmt = nodesMakeNode(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
SRedistributeVgroupStmt* pStmt = nodesMakeNode(QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
return pStmt;
pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10);
pStmt->pDnodes = pDnodes;
return (SNode*)pStmt;
}
SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
CHECK_PARSER_STATUS(pCxt);
SNode* pStmt = nodesMakeNode(QUERY_NODE_SPLIT_VGROUP_STMT);
SSplitVgroupStmt* pStmt = nodesMakeNode(QUERY_NODE_SPLIT_VGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
return pStmt;
pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10);
return (SNode*)pStmt;
}
SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) {
......
source/libs/parser/src/parTokenizer.c
浏览文件 @ 9aa1e1d3
......@@ -41,6 +41,7 @@ static SKeyword keywordTable[] = {
{"AS", TK_AS},
{"ASC", TK_ASC},
{"AT_ONCE", TK_AT_ONCE},
{"BALANCE", TK_BALANCE},
{"BETWEEN", TK_BETWEEN},
{"BINARY", TK_BINARY},
{"BIGINT", TK_BIGINT},
......@@ -116,6 +117,7 @@ static SKeyword keywordTable[] = {
{"LOCAL", TK_LOCAL},
{"MATCH", TK_MATCH},
{"MAXROWS", TK_MAXROWS},
{"MERGE", TK_MERGE},
{"MINROWS", TK_MINROWS},
{"MINUS", TK_MINUS},
{"MNODE", TK_MNODE},
......@@ -151,6 +153,7 @@ static SKeyword keywordTable[] = {
{"QUERY", TK_QUERY},
{"RATIO", TK_RATIO},
{"READ", TK_READ},
{"REDISTRIBUTE", TK_REDISTRIBUTE},
{"RENAME", TK_RENAME},
{"REPLICA", TK_REPLICA},
{"RESET", TK_RESET},
......@@ -171,6 +174,7 @@ static SKeyword keywordTable[] = {
{"SNODE", TK_SNODE},
{"SNODES", TK_SNODES},
{"SOFFSET", TK_SOFFSET},
// {"SPLIT", TK_SPLIT},
{"STABLE", TK_STABLE},
{"STABLES", TK_STABLES},
{"STATE", TK_STATE},
......@@ -208,6 +212,7 @@ static SKeyword keywordTable[] = {
{"VARCHAR", TK_VARCHAR},
{"VARIABLES", TK_VARIABLES},
{"VERBOSE", TK_VERBOSE},
{"VGROUP", TK_VGROUP},
{"VGROUPS", TK_VGROUPS},
{"VNODES", TK_VNODES},
{"WAL", TK_WAL},
......
source/libs/parser/src/parTranslater.c
浏览文件 @ 9aa1e1d3
......@@ -847,9 +847,9 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO
if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
if(pOp->opType == OP_TYPE_JSON_GET_VALUE){
if (pOp->opType == OP_TYPE_JSON_GET_VALUE) {
pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
}else if(pOp->opType == OP_TYPE_JSON_CONTAINS){
} else if (pOp->opType == OP_TYPE_JSON_CONTAINS) {
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
}
pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes;
......@@ -3611,6 +3611,55 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
}
static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStmt* pStmt) {
SBalanceVgroupReq req = {0};
return buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP, (FSerializeFunc)tSerializeSBalanceVgroupReq, &req);
}
static int32_t translateMergeVgroup(STranslateContext* pCxt, SMergeVgroupStmt* pStmt) {
SMergeVgroupReq req = {.vgId1 = pStmt->vgId1, .vgId2 = pStmt->vgId2};
return buildCmdMsg(pCxt, TDMT_MND_MERGE_VGROUP, (FSerializeFunc)tSerializeSMergeVgroupReq, &req);
}
static int32_t checkDnodeIds(STranslateContext* pCxt, SRedistributeVgroupStmt* pStmt) {
int32_t numOfDnodes = LIST_LENGTH(pStmt->pDnodes);
if (numOfDnodes > 3 || numOfDnodes < 1) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG);
}
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pDnodes) {
SValueNode* pVal = (SValueNode*)pNode;
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
}
pStmt->dnodeId1 = getBigintFromValueNode((SValueNode*)nodesListGetNode(pStmt->pDnodes, 0));
pStmt->dnodeId2 = -1;
pStmt->dnodeId3 = -1;
if (numOfDnodes > 1) {
pStmt->dnodeId2 = getBigintFromValueNode((SValueNode*)nodesListGetNode(pStmt->pDnodes, 1));
}
if (numOfDnodes > 2) {
pStmt->dnodeId3 = getBigintFromValueNode((SValueNode*)nodesListGetNode(pStmt->pDnodes, 2));
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateRedistributeVgroup(STranslateContext* pCxt, SRedistributeVgroupStmt* pStmt) {
SRedistributeVgroupReq req = {.vgId = pStmt->vgId};
int32_t code = checkDnodeIds(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
req.dnodeId1 = pStmt->dnodeId1;
req.dnodeId2 = pStmt->dnodeId2;
req.dnodeId3 = pStmt->dnodeId3;
code = buildCmdMsg(pCxt, TDMT_MND_REDISTRIBUTE_VGROUP, (FSerializeFunc)tSerializeSRedistributeVgroupReq, &req);
}
return code;
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
......@@ -3733,6 +3782,15 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_REVOKE_STMT:
code = translateRevoke(pCxt, (SRevokeStmt*)pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_STMT:
code = translateBalanceVgroup(pCxt, (SBalanceVgroupStmt*)pNode);
break;
case QUERY_NODE_MERGE_VGROUP_STMT:
code = translateMergeVgroup(pCxt, (SMergeVgroupStmt*)pNode);
break;
case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
code = translateRedistributeVgroup(pCxt, (SRedistributeVgroupStmt*)pNode);
break;
default:
break;
}
......
source/libs/parser/src/parUtil.c
浏览文件 @ 9aa1e1d3
......@@ -180,6 +180,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Value too long for column/tag: %s";
case TSDB_CODE_PAR_INVALID_DELETE_WHERE:
return "The DELETE statement must have a definite time window range";
case TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG:
return "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:
......
source/libs/parser/src/sql.c
浏览文件 @ 9aa1e1d3
此差异已折叠。
source/libs/parser/test/parExplainToSyncdbTest.cpp
浏览文件 @ 9aa1e1d3
......@@ -19,7 +19,7 @@ using namespace std;
namespace ParserTest {
class ParserExplainToSyncdbTest : public ParserTestBase {};
class ParserExplainToSyncdbTest : public ParserDdlTest {};
TEST_F(ParserExplainToSyncdbTest, explain) {
useDb("root", "test");
......@@ -43,8 +43,62 @@ TEST_F(ParserExplainToSyncdbTest, grant) {
// todo kill connection
// todo kill query
// todo kill stream
// todo merge vgroup
// todo redistribute vgroup
TEST_F(ParserExplainToSyncdbTest, mergeVgroup) {
useDb("root", "test");
SMergeVgroupReq expect = {0};
auto setMergeVgroupReqFunc = [&](int32_t vgId1, int32_t vgId2) {
expect.vgId1 = vgId1;
expect.vgId2 = vgId2;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_MERGE_VGROUP_STMT);
ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_MERGE_VGROUP);
SMergeVgroupReq req = {0};
ASSERT_EQ(tDeserializeSMergeVgroupReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
ASSERT_EQ(req.vgId1, expect.vgId1);
ASSERT_EQ(req.vgId2, expect.vgId2);
});
setMergeVgroupReqFunc(1, 2);
run("MERGE VGROUP 1 2");
}
TEST_F(ParserExplainToSyncdbTest, redistributeVgroup) {
useDb("root", "test");
SRedistributeVgroupReq expect = {0};
auto setRedistributeVgroupReqFunc = [&](int32_t vgId, int32_t dnodeId1, int32_t dnodeId2 = -1,
int32_t dnodeId3 = -1) {
expect.vgId = vgId;
expect.dnodeId1 = dnodeId1;
expect.dnodeId2 = dnodeId2;
expect.dnodeId3 = dnodeId3;
};
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_REDISTRIBUTE_VGROUP_STMT);
ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_REDISTRIBUTE_VGROUP);
SRedistributeVgroupReq req = {0};
ASSERT_EQ(tDeserializeSRedistributeVgroupReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req),
TSDB_CODE_SUCCESS);
ASSERT_EQ(req.vgId, expect.vgId);
ASSERT_EQ(req.dnodeId1, expect.dnodeId1);
ASSERT_EQ(req.dnodeId2, expect.dnodeId2);
ASSERT_EQ(req.dnodeId3, expect.dnodeId3);
});
setRedistributeVgroupReqFunc(3, 1);
run("REDISTRIBUTE VGROUP 3 DNODE 1");
setRedistributeVgroupReqFunc(5, 10, 20, 30);
run("REDISTRIBUTE VGROUP 5 DNODE 10 DNODE 20 DNODE 30");
}
// todo reset query cache
TEST_F(ParserExplainToSyncdbTest, revoke) {
......
source/libs/parser/test/parInitialATest.cpp
浏览文件 @ 9aa1e1d3
......@@ -305,6 +305,19 @@ TEST_F(ParserInitialATest, alterUser) {
run("ALTER user wxy privilege 'write'");
}
TEST_F(ParserInitialATest, balanceVgroup) {
useDb("root", "test");
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_BALANCE_VGROUP_STMT);
ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_BALANCE_VGROUP);
SBalanceVgroupReq req = {0};
ASSERT_EQ(tDeserializeSBalanceVgroupReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS);
});
run("BALANCE VGROUP");
}
TEST_F(ParserInitialATest, bug001) {
useDb("root", "test");
......
source/libs/stream/src/streamData.c
浏览文件 @ 9aa1e1d3
......@@ -13,7 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tstream.h"
#include "streamInc.h"
#if 0
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
......@@ -74,7 +74,6 @@ FAIL:
}
static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
//
atomic_add_fetch_32(pDataSubmit->dataRef, 1);
}
......
source/libs/stream/src/streamMsg.c
浏览文件 @ 9aa1e1d3
......@@ -116,11 +116,9 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
*ppEpSet = &pTask->fixedEpDispatcher.epSet;
downstreamTaskId = pTask->fixedEpDispatcher.taskId;
} else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
// TODO get ctbName
char ctbName[TSDB_TABLE_FNAME_LEN + 22] = {0};
// TODO get ctbName for each block
SSDataBlock* pBlock = taosArrayGet(data->blocks, 0);
sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId);
// get vg and ep
char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId);
// TODO: get hash function by hashMethod
// get groupId, compute hash value
......
source/util/src/terror.c
浏览文件 @ 9aa1e1d3
......@@ -451,9 +451,89 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_TIMEOUT_ERROR, "Task timeout")
TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order")
// parser
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED, "Permission denied")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED, "Permission denied")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYNTAX_ERROR, "syntax error near")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCOMPLETE_SQL, "Incomplete SQL statement")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLUMN, "Invalid column name")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_AMBIGUOUS_COLUMN, "Column ambiguously defined")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Invalid value type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, "There mustn't be aggregation")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, "ORDER BY item must be the number of a SELECT-list expression")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, "Not a GROUP BY expression")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, "Not SELECTed expression")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SINGLE_GROUP, "Not a single-group group function")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAGS_NOT_MATCHED, "Tags number not matched")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAG_NAME, "Invalid tag name")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG, "Name or password too long")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PASSWD_EMPTY, "Password can not be empty")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PORT, "Port should be an integer that is less than 65535 and greater than 0")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ENDPOINT, "Endpoint should be in the format of 'fqdn:port'")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_EXPRIE_STATEMENT, "This statement is no longer supported")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, "Interval too small")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_DB_NOT_SPECIFIED, "Database not specified")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "Invalid identifier name")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR, "Corresponding super table not in this db")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_RANGE_OPTION, "Invalid option")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STR_OPTION, "Invalid option")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ENUM_OPTION, "Invalid option")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_KEEP_NUM, "Invalid number of keep options")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_KEEP_ORDER, "Invalid keep value, should be keep0 <= keep1 <= keep2")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_KEEP_VALUE, "Invalid option keep")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COMMENT_OPTION, "Invalid option comment")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "Invalid option")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ROLLUP_OPTION, "Invalid option rollup: only one function is allowed")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION, "Invalid option retentions")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST, "GROUP BY and WINDOW-clause can't be used together")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_OPTION_UNIT, "Invalid option unit: only m, h, d allowed")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_KEEP_UNIT, "Invalid option keep unit: only m, h, d allowed")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_AGG_FUNC_NESTING, "Aggregate functions do not support nesting")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE, "Only support STATE_WINDOW on integer/bool/varchar column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STATE_WIN_COL, "Not support STATE_WINDOW on tag column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE, "STATE_WINDOW not support for super table query")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SESSION_GAP, "SESSION gap should be fixed time window, and greater than 0")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SESSION_COL, "Only support SESSION on primary timestamp column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE, "Interval offset cannot be negative")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_OFFSET_UNIT, "Cannot use 'year' as offset when interval is 'month'")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG, "Interval offset should be shorter than interval")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_UNIT, "Does not support sliding when interval is natural month/year")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG, "sliding value no larger than the interval value")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL, "sliding value can not less than 1% of interval value")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_ONLY_ONE_JSON_TAG, "Only one tag if there is a json tag")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_NUM_OF_COL, "Query block has incorrect number of result columns")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL, "Incorrect TIMESTAMP value")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DAYS_VALUE, "Invalid days value, should be keep2 >= keep1 >= keep0 >= days")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_OFFSET_LESS_ZERO, "soffset/offset can not be less than 0")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY, "slimit/soffset only available for PARTITION BY query")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TOPIC_QUERY, "Invalid topic query")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_STABLE, "Cannot drop super table in batch")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Start(end) time of query range required or time range too large")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_DUPLICATED_COLUMN, "Duplicated column names")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAGS_LENGTH, "Tags length exceeds max length")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ROW_LENGTH, "Row length exceeds max length")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TOO_MANY_COLUMNS, "Too many columns")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FIRST_COLUMN, "First column must be timestamp")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN, "Invalid binary/nchar column length")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAGS_NUM, "Invalid number of tag columns")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_INTERNAL_PK, "Invalid _c0 or _rowts expression")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TIMELINE_FUNC, "Invalid timeline function")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PASSWD, "Invalid password")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Invalid alter table statement")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY, "Primary timestamp column cannot be dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar column length could be modified")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TBNAME, "Invalid tbname pseudo column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FUNCTION_NAME, "Invalid function name")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COMMENT_TOO_LONG, "Comment too long")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Some functions are allowed only in the SELECT list of a query. "
"And, cannot be mixed with other non scalar functions or columns.")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, "Window query not supported, since the result of subquery not include valid timestamp column")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_COL, "No columns can be dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COL_JSON, "Only tag can be json type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VALUE_TOO_LONG, "Value too long for column/tag")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statement must have a definite time window range")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes")
//planner
TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "Planner internal error")
......
tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim
浏览文件 @ 9aa1e1d3
......@@ -31,7 +31,7 @@ if $data[0][4] != ready then
goto check_dnode_ready
endi
#sql connect
sql connect
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
......@@ -71,7 +71,7 @@ sql create database db replica $replica vgroups $vgroups
$loop_cnt = 0
check_db_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
sleep 20
if $loop_cnt == 10 then
print ====> db not ready!
return -1
......@@ -93,13 +93,12 @@ $loop_cnt = 0
check_vg_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
if $loop_cnt == 300 then
print ====> vgroups not ready!
return -1
endi
sql show vgroups
print ===> rows: $rows
print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13]
print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13]
if $rows != $vgroups then
return -1
......@@ -132,10 +131,12 @@ if $data[0][8] == leader then
goto check_vg_ready
endi
print ---- vgroup $data[0][0] leader locate on dnode $data[0][7]
goto vg_ready
goto vg_ready
else
goto check_vg_ready
endi
vg_ready:
vg_ready:
print ====> create stable/child table
sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
......@@ -156,27 +157,13 @@ while $i < $tbNum
sql create table $ctb using stb tags( $i )
$ntb = $ntbPrefix . $i
sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
# $x = 0
# while $x < $rowNum
# $binary = ' . binary
# $binary = $binary . $i
# $binary = $binary . '
#
# sql insert into $ctb values ($tstart , $i , $x , $binary )
# sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' )
# $tstart = $tstart + 1
# $x = $x + 1
# endw
# print ====> insert rows: $rowNum into $ctb and $ntb
$i = $i + 1
# $tstart = 1640966400000
endw
$totalTblNum = $tbNum * 2
sleep 1000
sql show tables
print ====> expect $totalTblNum and infinsert $rows in fact
if $rows != $totalTblNum then
return -1
endi
......@@ -185,7 +172,7 @@ print ====> create a normal table for interaction between main and back threads
sql create table interaction (ts timestamp, flag binary(10), childrows int, stbrows int)
print ====> start to run_back to insert data
run_back tsim/tmq/insertDataByRunBack.sim
run_back tsim/sync/insertDataByRunBack.sim
print ====> waiting insert thread starting insert data
......@@ -222,12 +209,15 @@ endi
$dnodeId = dnode . $dnodeId
print ====> stop $dnodeId
system sh/exec.sh -n $dnodeId -s stop -x SIGINT
sleep 1000
print ====> start $dnodeId
system sh/exec.sh -n $dnodeId -s start
$loop_cnt = 0
check_vg_ready_2:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
if $loop_cnt == 300 then
print ====> vgroups switch fail!!!
return -1
endi
......@@ -242,38 +232,38 @@ if $data[0][4] == leader then
if $data[0][6] != NULL then
goto check_vg_ready_2
endi
if $data[0][8] != NULL then
if $data[0][8] != FOLLOWER then
goto check_vg_ready_2
endi
print ---- vgroup $data[0][0] leader switch to dnode $data[0][3]
print ---- vgroup $dnodeId leader switch to dnode $data[0][3]
goto vg_ready_2
endi
if $data[0][6] == leader then
if $data[0][4] != NULL then
goto check_vg_ready_2
endi
if $data[0][8] != NULL then
if $data[0][8] != FOLLOWER then
goto check_vg_ready_2
endi
print ---- vgroup $data[0][0] leader switch to dnode $data[0][5]
print ---- vgroup $dnodeId leader switch to dnode $data[0][5]
goto vg_ready_2
endi
if $data[0][8] == leader then
if $data[0][4] != NULL then
goto check_vg_ready_2
endi
if $data[0][6] != NULL then
if $data[0][6] != FOLLOWER then
goto check_vg_ready_2
endi
print ---- vgroup $data[0][0] leader switch to dnode $data[0][7]
print ---- vgroup $dnodeId leader switch to dnode $data[0][7]
goto vg_ready_2
else
goto check_vg_ready_2
endi
vg_ready_2:
$switch_loop_cnt = $switch_loop_cnt + 1
if $switch_loop_cnt < 3 then
print ====> start $dnodeId
system sh/exec.sh -n $dnodeId -s start
goto switch_leader_loop
endi
......@@ -347,7 +337,7 @@ if $data[0][4] == leader then
if $data[0][6] != NULL then
goto check_vg_ready_1
endi
if $data[0][8] != NULL then
if $data[0][8] != FOLLOWER then
goto check_vg_ready_1
endi
goto vg_ready_1
......@@ -356,7 +346,7 @@ if $data[0][6] == leader then
if $data[0][4] != NULL then
goto check_vg_ready_1
endi
if $data[0][8] != NULL then
if $data[0][8] != FOLLOWER then
goto check_vg_ready_1
endi
goto vg_ready_1
......@@ -365,7 +355,7 @@ if $data[0][8] == leader then
if $data[0][4] != NULL then
goto check_vg_ready_1
endi
if $data[0][6] != NULL then
if $data[0][6] != FOLLOWER then
goto check_vg_ready_1
endi
goto vg_ready_1
......@@ -394,6 +384,73 @@ if $data[0][0] != $totalRowsOfStb then
return -1
endi
print ====> once stop one dnode by loop, and do query every time
$i = 2
loop_stop_dnode:
$dnodeId = dnode . $i
print ====> stop $dnodeId
system sh/exec.sh -n $dnodeId -s stop -x SIGINT
check_vg_ready_3:
sql show vgroups
print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13]
if $data[0][4] == LEADER then
if $data[0][6] == LEADER then
goto check_vg_ready_3
endi
if $data[0][8] == LEADER then
goto check_vg_ready_3
endi
print ---- vgroup $data[0][0] leader locating dnode $data[0][5]
elif $data[0][6] == LEADER then
if $data[0][4] == LEADER then
goto check_vg_ready_3
endi
if $data[0][8] == LEADER then
goto check_vg_ready_3
endi
print ---- vgroup $data[0][0] leader locating dnode $data[0][7]
elif $data[0][8] == LEADER then
if $data[0][4] == LEADER then
goto check_vg_ready_3
endi
if $data[0][6] == LEADER then
goto check_vg_ready_3
endi
print ---- vgroup $data[0][0] leader locating dnode $data[0][9]
else
print ====> no leader vnode!!!
return -1
endi
sql select count(*) from ntb0
print rows: $rows
print $data[0][0] $data[0][1]
if $data[0][0] != $totalRowsOfCtb then
return -1
endi
sql select count(*) from ctb0
print rows: $rows
print $data[0][0] $data[0][1]
if $data[0][0] != $totalRowsOfCtb then
return -1
endi
sql select count(*) from stb
print rows: $rows
print $data[0][0] $data[0][1]
if $data[0][0] != $totalRowsOfStb then
return -1
endi
$i = $i + 1
if $i <= 4 then
print ====> start $dnodeId
system sh/exec.sh -n $dnodeId -s start
goto loop_stop_dnode
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT
......
tests/system-test/1-insert/test_stmt_insert_query_ex.py
浏览文件 @ 9aa1e1d3
......@@ -80,93 +80,9 @@ class TDTestCase:
con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
print(con)
return con
def test_stmt_insert_multi(self,conn):
# type: (TaosConnection) -> None
dbname = "pytest_taos_stmt_multi"
try:
conn.execute("drop database if exists %s" % dbname)
conn.execute("create database if not exists %s" % dbname)
conn.select_db(dbname)
conn.execute(
"create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
ff float, dd double, bb binary(100), nn nchar(100), tt timestamp)",
)
# conn.load_table_info("log")
start = datetime.now()
stmt = conn.statement("insert into log values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
params = new_multi_binds(16)
params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
params[1].bool((True, None, False))
params[2].tinyint([-128, -128, None]) # -128 is tinyint null
params[3].tinyint([0, 127, None])
params[4].smallint([3, None, 2])
params[5].int([3, 4, None])
params[6].bigint([3, 4, None])
params[7].tinyint_unsigned([3, 4, None])
params[8].smallint_unsigned([3, 4, None])
params[9].int_unsigned([3, 4, None])
params[10].bigint_unsigned([3, 4, None])
params[11].float([3, None, 1])
params[12].double([3, None, 1.2])
params[13].binary(["abc", "dddafadfadfadfadfa", None])
params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
params[15].timestamp([None, None, 1626861392591])
# print(type(stmt))
stmt.bind_param_batch(params)
stmt.execute()
end = datetime.now()
print("elapsed time: ", end - start)
assert stmt.affected_rows == 3
#query
querystmt=conn.statement("select ?,bu from log")
queryparam=new_bind_params(1)
print(type(queryparam))
queryparam[0].binary("ts")
querystmt.bind_param(queryparam)
querystmt.execute()
result=querystmt.use_result()
# rows=result.fetch_all()
# print( querystmt.use_result())
# result = conn.query("select * from log")
rows=result.fetch_all()
# rows=result.fetch_all()
print(rows)
assert rows[1][0] == "ts"
assert rows[0][1] == 3
#query
querystmt1=conn.statement("select * from log where bu < ?")
queryparam1=new_bind_params(1)
print(type(queryparam1))
queryparam1[0].int(4)
querystmt1.bind_param(queryparam1)
querystmt1.execute()
result1=querystmt1.use_result()
rows1=result1.fetch_all()
assert str(rows1[0][0]) == "2021-07-21 17:56:32.589000"
assert rows1[0][10] == 3
stmt.close()
# conn.execute("drop database if exists %s" % dbname)
conn.close()
except Exception as err:
# conn.execute("drop database if exists %s" % dbname)
conn.close()
raise err
def test_stmt_set_tbname_tag(self,conn):
dbname = "pytest_taos_stmt_set_tbname_tag"
dbname = "stmt_set_tbname_tag"
try:
conn.execute("drop database if exists %s" % dbname)
......@@ -174,16 +90,16 @@ class TDTestCase:
conn.select_db(dbname)
conn.execute("create table if not exists log(ts timestamp, bo bool, nil tinyint, ti tinyint, si smallint, ii int,\
bi bigint, tu tinyint unsigned, su smallint unsigned, iu int unsigned, bu bigint unsigned, \
ff float, dd double, bb binary(100), nn nchar(100), tt timestamp) tags (t1 timestamp, t2 bool,\
ff float, dd double, bb binary(100), nn nchar(100), tt timestamp , vc varchar(100)) tags (t1 timestamp, t2 bool,\
t3 tinyint, t4 tinyint, t5 smallint, t6 int, t7 bigint, t8 tinyint unsigned, t9 smallint unsigned, \
t10 int unsigned, t11 bigint unsigned, t12 float, t13 double, t14 binary(100), t15 nchar(100), t16 timestamp)")
stmt = conn.statement("insert into ? using log tags (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) \
values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
tags = new_bind_params(16)
tags[0].timestamp(1626861392589123, PrecisionEnum.Microseconds)
tags[1].bool(True)
tags[2].null()
tags[2].bool(False)
tags[3].tinyint(2)
tags[4].smallint(3)
tags[5].int(4)
......@@ -198,7 +114,7 @@ class TDTestCase:
tags[14].nchar("stmt")
tags[15].timestamp(1626861392589, PrecisionEnum.Milliseconds)
stmt.set_tbname_tags("tb1", tags)
params = new_multi_binds(16)
params = new_multi_binds(17)
params[0].timestamp((1626861392589111, 1626861392590111, 1626861392591111))
params[1].bool((True, None, False))
params[2].tinyint([-128, -128, None]) # -128 is tinyint null
......@@ -213,25 +129,32 @@ class TDTestCase:
params[11].float([3, None, 1])
params[12].double([3, None, 1.2])
params[13].binary(["abc", "dddafadfadfadfadfa", None])
params[14].nchar(["涛思数据", None, "a? long string with 中文字符"])
params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
params[15].timestamp([None, None, 1626861392591])
params[16].binary(["涛思数据16", None, "a long string with 中文-字符"])
stmt.bind_param_batch(params)
stmt.execute()
assert stmt.affected_rows == 3
#query
#query all
querystmt1=conn.statement("select * from log where bu < ?")
queryparam1=new_bind_params(1)
print(type(queryparam1))
queryparam1[0].int(5)
queryparam1[0].int(10)
querystmt1.bind_param(queryparam1)
querystmt1.execute()
result1=querystmt1.use_result()
rows1=result1.fetch_all()
print("1",rows1)
print(rows1[0])
print(rows1[1])
print(rows1[2])
assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
assert rows1[0][10] == 3
assert rows1[1][10] == 4
#query: Numeric Functions
querystmt2=conn.statement("select abs(?) from log where bu < ?")
queryparam2=new_bind_params(2)
print(type(queryparam2))
......@@ -242,6 +165,11 @@ class TDTestCase:
result2=querystmt2.use_result()
rows2=result2.fetch_all()
print("2",rows2)
assert rows2[0][0] == 5
assert rows2[1][0] == 5
#query: Numeric Functions and escapes
querystmt3=conn.statement("select abs(?) from log where nn= 'a? long string with 中文字符' ")
queryparam3=new_bind_params(1)
......@@ -252,9 +180,63 @@ class TDTestCase:
result3=querystmt3.use_result()
rows3=result3.fetch_all()
print("3",rows3)
# assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
# assert rows1[0][10] == 3
# assert rows1[1][10] == 4
assert rows3 == []
#query: string Functions
querystmt3=conn.statement("select CHAR_LENGTH(?) from log ")
queryparam3=new_bind_params(1)
print(type(queryparam3))
queryparam3[0].binary('中文字符')
querystmt3.bind_param(queryparam3)
querystmt3.execute()
result3=querystmt3.use_result()
rows3=result3.fetch_all()
print("4",rows3)
assert rows3[0][0] == 12, 'fourth case is failed'
assert rows3[1][0] == 12, 'fourth case is failed'
# #query: conversion Functions
# querystmt4=conn.statement("select cast( ? as bigint) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].binary('1232a')
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("5",rows4)
# assert rows4[0][0] == 1232
# assert rows4[1][0] == 1232
# querystmt4=conn.statement("select cast( ? as binary(10)) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].int(123)
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("6",rows4)
# assert rows4[0][0] == '123'
# assert rows4[1][0] == '123'
# #query: datatime Functions
# querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].timestamp(1626861392591111)
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("7",rows4)
# assert rows4[0][0] == 1, 'seventh case is failed'
# assert rows4[1][0] == 1, 'seventh case is failed'
# conn.execute("drop database if exists %s" % dbname)
conn.close()
......@@ -269,8 +251,6 @@ class TDTestCase:
config = buildPath+ "../sim/dnode1/cfg/"
host="localhost"
connectstmt=self.newcon(host,config)
self.test_stmt_insert_multi(connectstmt)
connectstmt=self.newcon(host,config)
self.test_stmt_set_tbname_tag(connectstmt)
return
......
tests/system-test/1-insert/test_stmt_muti_insert_query.py
浏览文件 @ 9aa1e1d3
......@@ -124,7 +124,7 @@ class TDTestCase:
print("elapsed time: ", end - start)
assert stmt.affected_rows == 3
#query
#query 1
querystmt=conn.statement("select ?,bu from log")
queryparam=new_bind_params(1)
print(type(queryparam))
......@@ -141,8 +141,9 @@ class TDTestCase:
print(rows)
assert rows[1][0] == "ts"
assert rows[0][1] == 3
assert rows[2][1] == None
#query
#query 2
querystmt1=conn.statement("select * from log where bu < ?")
queryparam1=new_bind_params(1)
print(type(queryparam1))
......
tests/system-test/1-insert/test_stmt_set_tbname_tag.py
浏览文件 @ 9aa1e1d3
......@@ -82,7 +82,7 @@ class TDTestCase:
return con
def test_stmt_set_tbname_tag(self,conn):
dbname = "pytest_taos_stmt_set_tbname_tag"
dbname = "stmt_set_tbname_tag"
try:
conn.execute("drop database if exists %s" % dbname)
......@@ -99,7 +99,7 @@ class TDTestCase:
tags = new_bind_params(16)
tags[0].timestamp(1626861392589123, PrecisionEnum.Microseconds)
tags[1].bool(True)
tags[2].null()
tags[2].bool(False)
tags[3].tinyint(2)
tags[4].smallint(3)
tags[5].int(4)
......@@ -114,7 +114,7 @@ class TDTestCase:
tags[14].nchar("stmt")
tags[15].timestamp(1626861392589, PrecisionEnum.Milliseconds)
stmt.set_tbname_tags("tb1", tags)
params = new_multi_binds(16)
params = new_multi_binds(17)
params[0].timestamp((1626861392589111, 1626861392590111, 1626861392591111))
params[1].bool((True, None, False))
params[2].tinyint([-128, -128, None]) # -128 is tinyint null
......@@ -129,28 +129,153 @@ class TDTestCase:
params[11].float([3, None, 1])
params[12].double([3, None, 1.2])
params[13].binary(["abc", "dddafadfadfadfadfa", None])
params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
params[14].nchar(["涛思数据", None, "a long string with 中文?字符"])
params[15].timestamp([None, None, 1626861392591])
params[16].binary(["涛思数据16", None, "a long string with 中文-字符"])
params[16].binary(["涛思数据16", None, None])
stmt.bind_param_batch(params)
stmt.execute()
assert stmt.affected_rows == 3
#query
#query all
querystmt1=conn.statement("select * from log where bu < ?")
queryparam1=new_bind_params(1)
print(type(queryparam1))
queryparam1[0].int(5)
queryparam1[0].int(10)
querystmt1.bind_param(queryparam1)
querystmt1.execute()
result1=querystmt1.use_result()
rows1=result1.fetch_all()
print(rows1)
# assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
# assert rows1[0][10] == 3
# assert rows1[1][10] == 4
print(rows1[0])
print(rows1[1])
print(rows1[2])
assert str(rows1[0][0]) == "2021-07-21 17:56:32.589111"
assert rows1[0][10] == 3
assert rows1[1][10] == 4
#query: Numeric Functions
querystmt2=conn.statement("select abs(?) from log where bu < ?")
queryparam2=new_bind_params(2)
print(type(queryparam2))
queryparam2[0].int(5)
queryparam2[1].int(5)
querystmt2.bind_param(queryparam2)
querystmt2.execute()
result2=querystmt2.use_result()
rows2=result2.fetch_all()
print("2",rows2)
assert rows2[0][0] == 5
assert rows2[1][0] == 5
#query: Numeric Functions and escapes
querystmt3=conn.statement("select abs(?) from log where nn= 'a? long string with 中文字符' ")
queryparam3=new_bind_params(1)
print(type(queryparam3))
queryparam3[0].int(5)
querystmt3.bind_param(queryparam3)
querystmt3.execute()
result3=querystmt3.use_result()
rows3=result3.fetch_all()
print("3",rows3)
assert rows3 == []
# #query: string Functions
# querystmt3=conn.statement("select CHAR_LENGTH(?) from log ")
# queryparam3=new_bind_params(1)
# print(type(queryparam3))
# queryparam3[0].binary('中文字符')
# querystmt3.bind_param(queryparam3)
# querystmt3.execute()
# result3=querystmt3.use_result()
# rows3=result3.fetch_all()
# print("4",rows3)
# assert rows3[0][0] == 12, 'fourth case is failed'
# assert rows3[1][0] == 12, 'fourth case is failed'
# #query: conversion Functions
# querystmt4=conn.statement("select cast( ? as bigint) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].binary('1232a')
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("5",rows4)
# assert rows4[0][0] == 1232
# assert rows4[1][0] == 1232
# querystmt4=conn.statement("select cast( ? as binary(10)) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].int(123)
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("6",rows4)
# assert rows4[0][0] == '123'
# assert rows4[1][0] == '123'
# #query: datatime Functions
# querystmt4=conn.statement(" select timediff('2021-07-21 17:56:32.590111',?,1s) from log ")
# queryparam4=new_bind_params(1)
# print(type(queryparam4))
# queryparam4[0].timestamp(1626861392591111)
# querystmt4.bind_param(queryparam4)
# querystmt4.execute()
# result4=querystmt4.use_result()
# rows4=result4.fetch_all()
# print("7",rows4)
# assert rows4[0][0] == 1, 'seventh case is failed'
# assert rows4[1][0] == 1, 'seventh case is failed'
#query: aggregate Functions
querystmt4=conn.statement(" select count(?) from log ")
queryparam4=new_bind_params(1)
print(type(queryparam4))
queryparam4[0].int(123)
querystmt4.bind_param(queryparam4)
querystmt4.execute()
result4=querystmt4.use_result()
rows4=result4.fetch_all()
print("8",rows4)
assert rows4[0][0] == 3, ' 8 case is failed'
#query: selector Functions 9
querystmt4=conn.statement(" select bottom(bu,?) from log group by bu ; ")
queryparam4=new_bind_params(1)
print(type(queryparam4))
queryparam4[0].int(2)
querystmt4.bind_param(queryparam4)
querystmt4.execute()
result4=querystmt4.use_result()
rows4=result4.fetch_all()
print("9",rows4)
assert rows4[0][0] == 4, ' 9 case is failed'
assert rows4[1][0] == 3, ' 9 case is failed'
# #query: time-series specific Functions 10
querystmt4=conn.statement(" select twa(?) from log; ")
queryparam4=new_bind_params(1)
print(type(queryparam4))
queryparam4[0].int(15)
querystmt4.bind_param(queryparam4)
querystmt4.execute()
result4=querystmt4.use_result()
rows4=result4.fetch_all()
print("10",rows4)
assert rows4[0][0] == 15, ' 10 case is failed'
# conn.execute("drop database if exists %s" % dbname)
conn.close()
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或