diff --git a/docs/en/14-reference/03-connector/60-r-lang.mdx b/docs/en/14-reference/03-connector/60-r-lang.mdx index 852b2022a5c77bd6a2a0388f262c75653129077f..f1cbb89f7b1bbda310578cdee064821b8444cc09 100644 --- a/docs/en/14-reference/03-connector/60-r-lang.mdx +++ b/docs/en/14-reference/03-connector/60-r-lang.mdx @@ -24,8 +24,9 @@ install.packages("RJDBC", repos='http://cran.us.r-project.org') ``` :::note -On Linux systems, installing the RJDBC package may require installing the necessary components for compilation. For example, on Ubuntu, you can execute the command ``apt install -y libbz2-dev libpcre2-dev libicu-dev`` to install the required components. -On Windows systems, you need to set the **JAVA_HOME** environment variable. +1. The default R language package version 4.2 which shipped with Ubuntu might lead unresponsive bug. Please install latest version of R language package from the [official website](https://www.r-project.org/). +2. On Linux systems, installing the RJDBC package may require installing the necessary components for compilation. For example, on Ubuntu, you can execute the command ``apt install -y libbz2-dev libpcre2-dev libicu-dev`` to install the required components. +3. On Windows systems, you need to set the **JAVA_HOME** environment variable. ::: 3. Download the TDengine JDBC driver: Visit the Maven website and download the TDengine JDBC driver (taos-jdbcdriver-X.X.X-dist.jar) to your local machine. diff --git a/docs/examples/R/connect_rest.r b/docs/examples/R/connect_rest.r index bc5da3c15a9ec6a664678b38398082ebc0918dd2..a5221d2c3b2091880e4c87b1ed39df6db0b62d75 100644 --- a/docs/examples/R/connect_rest.r +++ b/docs/examples/R/connect_rest.r @@ -12,5 +12,9 @@ driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver- driver = JDBC("com.taosdata.jdbc.rs.RestfulDriver", driver_path) conn = dbConnect(driver, "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata") dbGetQuery(conn, "SELECT server_version()") +dbSendUpdate(conn, "create database if not exists rtest") +dbSendUpdate(conn, "create table if not exists rtest.test (ts timestamp, current float, voltage int, devname varchar(20))") +dbSendUpdate(conn, "insert into rtest.test values (now, 1.2, 220, 'test')") +dbGetQuery(conn, "select * from rtest.test") dbDisconnect(conn) # ANCHOR_END: demo diff --git a/docs/zh/08-connector/43-r-lang.mdx b/docs/zh/08-connector/43-r-lang.mdx index 3a4ed39748836d9cc502ce7a14d495e1a629a6b2..a181f68abac0e79e3f6db512711e644709f8b2dd 100644 --- a/docs/zh/08-connector/43-r-lang.mdx +++ b/docs/zh/08-connector/43-r-lang.mdx @@ -24,8 +24,9 @@ install.packages("RJDBC", repos='http://cran.us.r-project.org') ``` :::note -在 Linux 上安装 RJDBC 包可能需要安装编译需要的组件,以 Ubuntu 为例执行 `apt install -y libbz2-dev libpcre2-dev libicu-dev` 命令安装。 -在 Windows 系统上需要设置 JAVA_HOME 环境变量。 +1. Ubuntu 系统自带的 R 语言软件版本 4.2 在调用 RJDBC 库会产生无响应 bug,请安装 R 语言[官网](https://www.r-project.org/)的安装包。 +2. 在 Linux 上安装 RJDBC 包可能需要安装编译需要的组件,以 Ubuntu 为例执行 `apt install -y libbz2-dev libpcre2-dev libicu-dev` 命令安装。 +3. 在 Windows 系统上需要设置 JAVA_HOME 环境变量。 ::: 3. 下载 TDengine JDBC 驱动程序:访问 maven.org 网站,下载 TDengine JDBC 驱动程序(taos-jdbcdriver-X.X.X-dist.jar)。 diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index b4ae30910c14b2c5965e3d19776f6fac09cdd3a1..b241ae9b4125d4fd0fbaf40ef7477d819229572f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -30,6 +30,7 @@ extern "C" { typedef struct SStreamTask SStreamTask; +#define SSTREAM_TASK_VER 1 enum { STREAM_STATUS__NORMAL = 0, STREAM_STATUS__STOP, @@ -266,13 +267,13 @@ typedef struct SCheckpointInfo { } SCheckpointInfo; typedef struct SStreamStatus { - int8_t taskStatus; - int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set - int8_t schedStatus; - int8_t keepTaskStatus; - bool transferState; - int8_t timerActive; // timer is active - int8_t pauseAllowed; // allowed task status to be set to be paused + int8_t taskStatus; + int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set + int8_t schedStatus; + int8_t keepTaskStatus; + bool transferState; + int8_t timerActive; // timer is active + int8_t pauseAllowed; // allowed task status to be set to be paused } SStreamStatus; typedef struct SHistDataRange { @@ -309,6 +310,7 @@ typedef struct { } STaskTimestamp; struct SStreamTask { + int64_t ver; SStreamId id; SSTaskBasicInfo info; STaskOutputInfo outputInfo; @@ -589,10 +591,10 @@ bool streamTaskShouldPause(const SStreamStatus* pStatus); bool streamTaskIsIdle(const SStreamTask* pTask); int32_t streamTaskEndScanWAL(SStreamTask* pTask); -SStreamChildEpInfo * streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); -int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize); +SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); +int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize); -char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); // recover and fill history void streamTaskCheckDownstreamTasks(SStreamTask* pTask); @@ -628,7 +630,8 @@ int32_t streamDispatchTransferStateMsg(SStreamTask* pTask); // agg level int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask); -int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq *pReq, SRpcHandleInfo* pRpcInfo); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, + SRpcHandleInfo* pRpcInfo); int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); // stream task meta @@ -642,7 +645,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int32_t taskId); -int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); // todo remove it +int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); // todo remove it SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); @@ -659,7 +662,6 @@ int32_t streamTaskReleaseState(SStreamTask* pTask); int32_t streamTaskReloadState(SStreamTask* pTask); int32_t streamAlignTransferState(SStreamTask* pTask); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index a8a719edda4d204baf753d7ccb622ed85f50d9a0..3dab144eefcf92787b016062285990862fa3ebb5 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -70,6 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, innerSz) < 0) return -1; for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); + pTask->ver = SSTREAM_TASK_VER; if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; } } @@ -154,7 +155,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { return 0; } -static void* freeStreamTasks(SArray* pTaskLevel) { +static void *freeStreamTasks(SArray *pTaskLevel) { int32_t numOfLevel = taosArrayGetSize(pTaskLevel); for (int32_t i = 0; i < numOfLevel; i++) { SArray *pLevel = taosArrayGetP(pTaskLevel, i); @@ -192,14 +193,14 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); if (pVgEpNew == NULL) return NULL; pVgEpNew->vgId = pVgEp->vgId; -// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); + // pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); pVgEpNew->epSet = pVgEp->epSet; return pVgEpNew; } void tDeleteSMqVgEp(SMqVgEp *pVgEp) { if (pVgEp) { -// taosMemoryFreeClear(pVgEp->qmsg); + // taosMemoryFreeClear(pVgEp->qmsg); taosMemoryFree(pVgEp); } } @@ -207,14 +208,14 @@ void tDeleteSMqVgEp(SMqVgEp *pVgEp) { int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); -// tlen += taosEncodeString(buf, pVgEp->qmsg); + // tlen += taosEncodeString(buf, pVgEp->qmsg); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); - if(sver == 1){ + if (sver == 1) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); buf = POINTER_SHIFT(buf, size); @@ -223,7 +224,7 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) { return (void *)buf; } -SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char* cgroup) { +SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup) { SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,12 +261,12 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char* cgroup) { } void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer, bool delete) { - if(pConsumer == NULL) return; + if (pConsumer == NULL) return; taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree); - if(delete){ + if (delete) { taosMemoryFree(pConsumer); } } @@ -392,7 +393,7 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s taosArrayPush(pConsumer->assignedTopics, &topic); } - if(sver > 1){ + if (sver > 1) { buf = taosDecodeFixedI8(buf, &pConsumer->withTbName); buf = taosDecodeFixedI8(buf, &pConsumer->autoCommit); buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval); @@ -401,18 +402,18 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s return (void *)buf; } -//SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { -// SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); -// if (pConsumerEpNew == NULL) return NULL; -// pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; -// pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); -// return pConsumerEpNew; -//} +// SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { +// SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); +// if (pConsumerEpNew == NULL) return NULL; +// pConsumerEpNew->consumerId = pConsumerEpOld->consumerId; +// pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL); +// return pConsumerEpNew; +// } // -//void tDeleteSMqConsumerEp(void *data) { -// SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; -// taosArrayDestroy(pConsumerEp->vgs); -//} +// void tDeleteSMqConsumerEp(void *data) { +// SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data; +// taosArrayDestroy(pConsumerEp->vgs); +// } int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { int32_t tlen = 0; @@ -420,7 +421,7 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows); tlen += taosEncodeFixedI32(buf, szVgs); - for (int32_t j= 0; j < szVgs; ++j) { + for (int32_t j = 0; j < szVgs; ++j) { OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j); tlen += taosEncodeFixedI32(buf, offRows->vgId); tlen += taosEncodeFixedI64(buf, offRows->rows); @@ -434,28 +435,28 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { // do nothing } } -//#if 0 -// int32_t sz = taosArrayGetSize(pConsumerEp->vgs); -// tlen += taosEncodeFixedI32(buf, sz); -// for (int32_t i = 0; i < sz; i++) { -// SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); -// tlen += tEncodeSMqVgEp(buf, pVgEp); -// } -//#endif + // #if 0 + // int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + // tlen += taosEncodeFixedI32(buf, sz); + // for (int32_t i = 0; i < sz; i++) { + // SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + // tlen += tEncodeSMqVgEp(buf, pVgEp); + // } + // #endif return tlen; } void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); - if (sver > 1){ + if (sver > 1) { int32_t szVgs = 0; buf = taosDecodeFixedI32(buf, &szVgs); - if(szVgs > 0){ + if (szVgs > 0) { pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); if (NULL == pConsumerEp->offsetRows) return NULL; - for (int32_t j= 0; j < szVgs; ++j) { - OffsetRows* offRows = taosArrayReserve(pConsumerEp->offsetRows, 1); + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(pConsumerEp->offsetRows, 1); buf = taosDecodeFixedI32(buf, &offRows->vgId); buf = taosDecodeFixedI64(buf, &offRows->rows); buf = taosDecodeFixedI8(buf, &offRows->offset.type); @@ -470,21 +471,21 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s } } } -//#if 0 -// int32_t sz; -// buf = taosDecodeFixedI32(buf, &sz); -// pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); -// for (int32_t i = 0; i < sz; i++) { -// SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); -// buf = tDecodeSMqVgEp(buf, pVgEp); -// taosArrayPush(pConsumerEp->vgs, &pVgEp); -// } -//#endif + // #if 0 + // int32_t sz; + // buf = taosDecodeFixedI32(buf, &sz); + // pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *)); + // for (int32_t i = 0; i < sz; i++) { + // SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + // buf = tDecodeSMqVgEp(buf, pVgEp); + // taosArrayPush(pConsumerEp->vgs, &pVgEp); + // } + // #endif return (void *)buf; } -SMqSubscribeObj *tNewSubscribeObj(const char* key) { +SMqSubscribeObj *tNewSubscribeObj(const char *key) { SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSubObj == NULL) { return NULL; @@ -577,7 +578,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t szVgs = taosArrayGetSize(pSub->offsetRows); tlen += taosEncodeFixedI32(buf, szVgs); - for (int32_t j= 0; j < szVgs; ++j) { + for (int32_t j = 0; j < szVgs; ++j) { OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j); tlen += taosEncodeFixedI32(buf, offRows->vgId); tlen += taosEncodeFixedI64(buf, offRows->rows); @@ -617,14 +618,14 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver); buf = taosDecodeStringTo(buf, pSub->dbName); - if (sver > 1){ + if (sver > 1) { int32_t szVgs = 0; buf = taosDecodeFixedI32(buf, &szVgs); - if(szVgs > 0){ + if (szVgs > 0) { pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); if (NULL == pSub->offsetRows) return NULL; - for (int32_t j= 0; j < szVgs; ++j) { - OffsetRows* offRows = taosArrayReserve(pSub->offsetRows, 1); + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(pSub->offsetRows, 1); buf = taosDecodeFixedI32(buf, &offRows->vgId); buf = taosDecodeFixedI64(buf, &offRows->rows); buf = taosDecodeFixedI8(buf, &offRows->offset.type); @@ -639,71 +640,71 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) { } } buf = taosDecodeString(buf, &pSub->qmsg); - }else{ + } else { pSub->qmsg = taosStrdup(""); } return (void *)buf; } -//SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { -// SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); -// if (pEntryNew == NULL) return NULL; -// pEntryNew->epoch = pEntry->epoch; -// pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); -// return pEntryNew; -//} +// SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); +// if (pEntryNew == NULL) return NULL; +// pEntryNew->epoch = pEntry->epoch; +// pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pEntryNew; +// } // -//void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { -// taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); -//} - -//int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { -// int32_t tlen = 0; -// tlen += taosEncodeFixedI32(buf, pEntry->epoch); -// tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); -// return tlen; -//} +// void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { +// taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp); +// } + +// int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { +// int32_t tlen = 0; +// tlen += taosEncodeFixedI32(buf, pEntry->epoch); +// tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +// } // -//void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { -// buf = taosDecodeFixedI32(buf, &pEntry->epoch); -// buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); -// return (void *)buf; -//} - -//SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { -// SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); -// if (pLogNew == NULL) return pLogNew; -// memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); -// pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); -// return pLogNew; -//} +// void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { +// buf = taosDecodeFixedI32(buf, &pEntry->epoch); +// buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +// } + +// SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); +// if (pLogNew == NULL) return pLogNew; +// memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); +// pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp); +// return pLogNew; +// } // -//void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { -// taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); -//} - -//int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { -// int32_t tlen = 0; -// tlen += taosEncodeString(buf, pLog->key); -// tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); -// return tlen; -//} +// void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { +// taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp); +// } + +// int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pLog->key); +// tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry); +// return tlen; +// } // -//void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { -// buf = taosDecodeStringTo(buf, pLog->key); -// buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); -// return (void *)buf; -//} +// void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) { +// buf = taosDecodeStringTo(buf, pLog->key); +// buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); +// return (void *)buf; +// } // -//int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { -// int32_t tlen = 0; -// tlen += taosEncodeString(buf, pOffset->key); -// tlen += taosEncodeFixedI64(buf, pOffset->offset); -// return tlen; -//} +// int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { +// int32_t tlen = 0; +// tlen += taosEncodeString(buf, pOffset->key); +// tlen += taosEncodeFixedI64(buf, pOffset->offset); +// return tlen; +// } // -//void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { -// buf = taosDecodeStringTo(buf, pOffset->key); -// buf = taosDecodeFixedI64(buf, &pOffset->offset); -// return buf; -//} +// void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { +// buf = taosDecodeStringTo(buf, pOffset->key); +// buf = taosDecodeFixedI64(buf, &pOffset->offset); +// return buf; +// } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 06ab1bb638eb71c092393280d7e0b8648f30a9b6..0a4bb980dfb0530f070dd5a33610b1b399c246d6 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_STREAM_VER_NUMBER 2 +#define MND_STREAM_VER_NUMBER 3 #define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_MAX_NUM 60 @@ -140,10 +140,12 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { void *buf = NULL; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) { + goto STREAM_DECODE_OVER; + } - if (sver != 1 && sver != 2) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + if (sver != MND_STREAM_VER_NUMBER) { + terrno = 0; goto STREAM_DECODE_OVER; } @@ -429,9 +431,11 @@ FAIL: return 0; } -int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { +int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); + + pTask->ver = SSTREAM_TASK_VER; tEncodeStreamTask(&encoder, pTask); int32_t size = encoder.pos; @@ -1264,7 +1268,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char idstr[128] = {0}; + char idstr[128] = {0}; int32_t len = tintToHex(pTask->id.taskId, &idstr[4]); idstr[2] = '0'; idstr[3] = 'x'; @@ -1304,7 +1308,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock colDataSetVal(pColInfo, numOfRows, (const char *)&level, false); // status - char status[20 + VARSTR_HEADER_SIZE] = {0}; + char status[20 + VARSTR_HEADER_SIZE] = {0}; int8_t taskStatus = atomic_load_8(&pTask->status.taskStatus); if (taskStatus == TASK_STATUS__NORMAL) { memcpy(varDataVal(status), "normal", 6); @@ -1370,7 +1374,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { return 0; } -int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray* tasks) { +int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray *tasks) { int32_t size = taosArrayGetSize(tasks); for (int32_t i = 0; i < size; i++) { SArray *pTasks = taosArrayGetP(tasks, i); @@ -1491,7 +1495,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } - static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6eb2be34b3ca1654e25a83c6bf0ed9aefd11bf0d..844bfb07fcb9273bac35de2f4db832ac33497b05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -468,7 +468,8 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT, + .precision = pFunc->node.resType.precision}; return TSDB_CODE_SUCCESS; } @@ -491,7 +492,8 @@ static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in // pseudo column do not need to check parameters pFunc->node.resType = - (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP, + .precision = pFunc->node.resType.precision}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8534f3b0a11503101bf0a49c1f0a8aaf6811a4c7..4a0ce81e6808ed1184036537c204bdf5002569da 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -218,11 +218,11 @@ _EXIT: } void streamBackendCleanup(void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)arg; - RocksdbCfInst** pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL); + void* pIter = taosHashIterate(pHandle->cfInst, NULL); while (pIter != NULL) { - RocksdbCfInst* inst = *pIter; + RocksdbCfInst* inst = *(RocksdbCfInst**)pIter; destroyRocksdbCfInst(inst); - taosHashIterate(pHandle->cfInst, pIter); + pIter = taosHashIterate(pHandle->cfInst, pIter); } taosHashCleanup(pHandle->cfInst); @@ -833,7 +833,10 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t qDebug("succ to open rocksdb cf"); } // close default cf - if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) rocksdb_column_family_handle_destroy(cfHandle[0]); + if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) { + rocksdb_column_family_handle_destroy(cfHandle[0]); + cfHandle[0] = NULL; + } rocksdb_options_destroy(cfOpts[0]); handle->db = db; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ae077388685c0c732d956a466016ea2660db63c2..7886091401a5b2baf2e4a0f78686526cd8c9fdf0 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -202,6 +202,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; int32_t len; int32_t code; + pTask->ver = SSTREAM_TASK_VER; tEncodeSize(tEncodeStreamTask, pTask, len, code); if (code < 0) { return -1; @@ -331,7 +332,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int32_t taskId) { qDebug("s-task:0x%x set task status:%s", taskId, streamGetTaskStatusStr(TASK_STATUS__DROPPING)); - while(1) { + while (1) { taosRLockLatch(&pMeta->lock); ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); @@ -443,9 +444,20 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { taosArrayDestroy(pRecycleList); return -1; } - tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeStreamTask(&decoder, pTask); + if (tDecodeStreamTask(&decoder, pTask) < 0) { + tDecoderClear(&decoder); + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + taosArrayDestroy(pRecycleList); + tFreeStreamTask(pTask); + qError( + "stream read incompatible data, rm %s/vnode/vnode*/tq/stream if taosd cannot start, and rebuild stream " + "manually", + tsDataDir); + return -1; + } tDecoderClear(&decoder); if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { @@ -500,13 +512,13 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { } if (taosArrayGetSize(pRecycleList) > 0) { - for(int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { - int32_t taskId = *(int32_t*) taosArrayGet(pRecycleList, i); + for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { + int32_t taskId = *(int32_t*)taosArrayGet(pRecycleList, i); streamMetaRemoveTask(pMeta, taskId); } } - qDebug("vgId:%d load %d task from disk", pMeta->vgId, (int32_t) taosArrayGetSize(pMeta->pTaskList)); + qDebug("vgId:%d load %d task from disk", pMeta->vgId, (int32_t)taosArrayGetSize(pMeta->pTaskList)); taosArrayDestroy(pRecycleList); return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 1eb8d119168d23b96a851e0946e968c4e1b1fef8..dc4e5ff4a65a3326c3079e45ef9cd09b99343375 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -26,13 +26,14 @@ static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { return 0; } -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, SArray* pTaskList) { +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, + SArray* pTaskList) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - + pTask->ver = SSTREAM_TASK_VER; pTask->id.taskId = tGenIdPI32(); pTask->id.streamId = streamId; pTask->info.taskLevel = taskLevel; @@ -72,6 +73,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->ver) < 0) return -1; if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->info.totalLevel) < 0) return -1; @@ -135,6 +137,9 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->ver) < 0) return -1; + if (pTask->ver != SSTREAM_TASK_VER) return -1; + if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->info.totalLevel) < 0) return -1; @@ -163,7 +168,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI64(pDecoder, &pTask->dataRange.window.skey)) return -1; if (tDecodeI64(pDecoder, &pTask->dataRange.window.ekey)) return -1; - int32_t epSz; + int32_t epSz = -1; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; pTask->pUpstreamEpInfoList = taosArrayInit(epSz, POINTER_BYTES); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8715a0d13b3adbea7c6105ec119dc20e8f8fe0c2..747eb909a0d9a2ec7d3f00dc79d065abc73c8d72 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -135,6 +135,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3404.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py diff --git a/tests/system-test/99-TDcase/TS-3311.py b/tests/system-test/99-TDcase/TS-3311.py new file mode 100644 index 0000000000000000000000000000000000000000..ce39597a7b5dd20ad76818c4078b02a3736e219e --- /dev/null +++ b/tests/system-test/99-TDcase/TS-3311.py @@ -0,0 +1,121 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def create_tables(self): + tdSql.execute("create database if not exists dbus precision 'us'") + tdSql.execute("create database if not exists dbns precision 'ns'") + + tdSql.execute("use dbus") + + tdSql.execute(f"CREATE STABLE `stb_us` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `ctb1_us` USING `stb_us` (`t1`) TAGS (1)") + tdSql.execute(f"CREATE TABLE `ctb2_us` USING `stb_us` (`t1`) TAGS (2)") + + tdSql.execute("use dbns") + + tdSql.execute(f"CREATE STABLE `stb_ns` (`ts` TIMESTAMP, `ip_value` FLOAT, `ip_quality` INT) TAGS (`t1` INT)") + tdSql.execute(f"CREATE TABLE `ctb1_ns` USING `stb_ns` (`t1`) TAGS (1)") + tdSql.execute(f"CREATE TABLE `ctb2_ns` USING `stb_ns` (`t1`) TAGS (2)") + + def insert_data(self): + tdLog.debug("start to insert data ............") + + tdSql.execute(f"INSERT INTO `dbus`.`ctb1_us` VALUES ('2023-07-01 00:00:00.000', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `dbus`.`ctb2_us` VALUES ('2023-08-01 00:00:00.000', 20.30000, 200)") + + tdSql.execute(f"INSERT INTO `dbns`.`ctb1_ns` VALUES ('2023-07-01 00:00:00.000', 10.30000, 100)") + tdSql.execute(f"INSERT INTO `dbns`.`ctb2_ns` VALUES ('2023-08-01 00:00:00.000', 20.30000, 200)") + + tdLog.debug("insert data ............ [OK]") + + def run(self): + tdSql.prepare() + self.create_tables() + self.insert_data() + tdLog.printNoPrefix("======== test TS-3311") + + # test ns + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(1n)") + tdSql.checkRows(2) + + tdSql.checkData(0, 0, '2023-07-01 00:00:00.000000000') + tdSql.checkData(1, 0, '2023-08-01 00:00:00.000000000') + + tdSql.checkData(0, 1, '2023-08-01 00:00:00.000000000') + tdSql.checkData(1, 1, '2023-09-01 00:00:00.000000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(12n)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbns`.`stb_ns` interval(1y)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000000') + + + ## test us + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(1n)") + tdSql.checkRows(2) + + tdSql.checkData(0, 0, '2023-07-01 00:00:00.000000') + tdSql.checkData(1, 0, '2023-08-01 00:00:00.000000') + + tdSql.checkData(0, 1, '2023-08-01 00:00:00.000000') + tdSql.checkData(1, 1, '2023-09-01 00:00:00.000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(12n)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000') + + tdSql.query(f"select _wstart, _wend, count(*) from `dbus`.`stb_us` interval(1y)") + tdSql.checkRows(1) + + tdSql.checkData(0, 0, '2023-01-01 00:00:00.000000') + tdSql.checkData(0, 1, '2024-01-01 00:00:00.000000') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())