diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index c214c96f3569f0e792d11bd0db359de45dbaebb4..e798a0c42af1765848f3c98f4f5fdbaa41bc73aa 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -832,10 +832,8 @@ typedef struct { } SCreateDnodeMsg, SDropDnodeMsg; typedef struct { - int32_t dnodeId; - int32_t mnodeNum; - SDnodeEp mnodeEps[]; -} SCreateMnodeMsg; + int32_t dnodeId; +} SCreateMnodeMsg, SDropMnodeMsg; typedef struct { int32_t dnodeId; @@ -901,16 +899,7 @@ typedef struct { typedef struct { char queryId[TSDB_KILL_MSG_LEN + 1]; -} SKillQueryMsg, SKillStreamMsg, SKillConnMsg; - -typedef struct { - int32_t vnode; - int32_t sid; - uint64_t uid; - uint64_t stime; // stream starting time - int32_t status; - char tableFname[TSDB_TABLE_FNAME_LEN]; -} SAlterStreamMsg; +} SKillQueryMsg, SKillConnMsg; typedef struct { char user[TSDB_USER_LEN]; @@ -925,8 +914,7 @@ typedef struct { int8_t reserved1[7]; char name[TSDB_STEP_NAME_LEN]; char desc[TSDB_STEP_DESC_LEN]; - char reserved2[64]; -} SStartupStep; +} SStartupMsg; // mq related typedef struct { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0ce2e3da14d1cec204fc755db13da53f08295bff..5460ae54017afe33261388ff301011012ac6e0ff 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -22,22 +22,16 @@ extern "C" { #include #include #include "taosdef.h" +#include "taosmsg.h" #define TAOS_CONN_SERVER 0 #define TAOS_CONN_CLIENT 1 extern int tsRpcHeadSize; -typedef struct SRpcEpSet { - int8_t inUse; - int8_t numOfEps; - uint16_t port[TSDB_MAX_REPLICA]; - char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; -} SRpcEpSet; - typedef struct SRpcCorEpSet { int32_t version; - SRpcEpSet epSet; + SEpSet epSet; } SRpcCorEpSet; typedef struct SRpcConnInfo { @@ -72,7 +66,7 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *, SRpcEpSet *); + void (*cfp)(SRpcMsg *, SEpSet *); // call back to retrieve the client auth info, for server app only int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); @@ -85,11 +79,11 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); +void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(int64_t rid); diff --git a/include/server/mnode/mnode.h b/include/server/mnode/mnode.h index 9bbc2e4b10118f89bdeb9b9cacdb65b5f382c56c..e0fcdec560df967c54106a99bd695d49f0f041b7 100644 --- a/include/server/mnode/mnode.h +++ b/include/server/mnode/mnode.h @@ -20,41 +20,23 @@ extern "C" { #endif -typedef enum { MN_STATUS_UNINIT = 0, MN_STATUS_INIT = 1, MN_STATUS_READY = 2, MN_STATUS_CLOSING = 3 } EMnStatus; - typedef struct { - /** - * Send messages to other dnodes, such as create vnode message. - * - * @param epSet, the endpoint list of the dnodes. - * @param rpcMsg, message to be sent. - */ - void (*SendMsgToDnode)(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); + int64_t numOfDnode; + int64_t numOfMnode; + int64_t numOfVgroup; + int64_t numOfDatabase; + int64_t numOfSuperTable; + int64_t numOfChildTable; + int64_t numOfColumn; + int64_t totalPoints; + int64_t totalStorage; + int64_t compStorage; +} SMnodeStat; - /** - * Send messages to mnode, such as config message. - * - * @param rpcMsg, message to be sent. - */ +typedef struct { + void (*SendMsgToDnode)(struct SEpSet *epSet, struct SRpcMsg *rpcMsg); void (*SendMsgToMnode)(struct SRpcMsg *rpcMsg); - - /** - * Send redirect message to dnode or shell. - * - * @param rpcMsg, message to be sent. - * @param forShell, used to identify whether to send to shell or dnode. - */ void (*SendRedirectMsg)(struct SRpcMsg *rpcMsg, bool forShell); - - /** - * Get the corresponding endpoint information from dnodeId. - * - * @param dnode, the instance of dDnode module. - * @param dnodeId, the id ot dnode. - * @param ep, the endpoint of dnode. - * @param fqdn, the fqdn of dnode. - * @param port, the port of dnode. - */ void (*GetDnodeEp)(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); } SMnodeFp; @@ -64,77 +46,16 @@ typedef struct { int32_t dnodeId; } SMnodePara; -/** - * Initialize and start mnode module. - * - * @param para, initialization parameters. - * @return Error code. - */ int32_t mnodeInit(SMnodePara para); - -/** - * Stop and cleanup mnode module. - */ -void mnodeCleanup(); - -/** - * Deploy mnode instances in dnode. - * - * @return Error Code. - */ +void mnodeCleanup(); int32_t mnodeDeploy(); +void mnodeUnDeploy(); +int32_t mnodeStart(); +void mnodeStop(); -/** - * Delete the mnode instance deployed in dnode. - */ -void mnodeUnDeploy(); - -/** - * Whether the mnode is in service. - * - * @return Server status. - */ -EMnStatus mnodeGetStatus(); - -typedef struct { - int64_t numOfDnode; - int64_t numOfMnode; - int64_t numOfVgroup; - int64_t numOfDatabase; - int64_t numOfSuperTable; - int64_t numOfChildTable; - int64_t numOfColumn; - int64_t totalPoints; - int64_t totalStorage; - int64_t compStorage; -} SMnodeStat; - -/** - * Get the statistical information of Mnode. - * - * @param stat, statistical information. - * @return Error Code. - */ int32_t mnodeGetStatistics(SMnodeStat *stat); - -/** - * Get the auth information of Mnode. - * - * @param user, username. - * @param spi, security parameter index. - * @param encrypt, encrypt algorithm. - * @param secret, key for authentication. - * @param ckey, ciphering key. - * @return Error Code. - */ int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); -/** - * Interface for processing messages. - * - * @param rpcMsg, message to be processed. - * @return Error code. - */ void mnodeProcessMsg(SRpcMsg *rpcMsg); #ifdef __cplusplus diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index a9f40aad77581165ad29102e645620de4e8f935d..c4f8df79c11ecae8c7d3611ef6dfb3196bc77ee8 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -66,41 +66,12 @@ typedef struct SVnodeMsg { char pCont[]; } SVnodeMsg; -/** - * Start initialize vnode module. - * - * @return Error code. - */ int32_t vnodeInit(); +void vnodeCleanup(); -/** - * Cleanup vnode module. - */ -void vnodeCleanup(); - -/** - * Get the statistical information of vnode. - * - * @param pVnode, - * @param pStat, statistical information. - * @return Error Code. - */ int32_t vnodeGetStatistics(SVnode *pVnode, SVnodeStatisic *pStat); - -/** - * Get the status of all vnodes. - * - * @param pVnode, - * @param status, status information. - * @return Error Code. - */ int32_t vnodeGetStatus(SVnode *pVnode, SVnodeStatus *pStatus); -/** - * Operation functions of vnode - * - * @return Error Code. - */ SVnode *vnodeOpen(int32_t vgId, const char *path); void vnodeClose(SVnode *pVnode); int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); @@ -109,14 +80,7 @@ int32_t vnodeDrop(SVnode *pVnode); int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); -/** - * Interface for processing messages. - * - * @param pVnode, - * @param pMsg, message to be processed. - * - */ -int32_t vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg); +void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 76c5f575a5c64bcf98065ba9bb14d2412816dcb6..0579ae46bcb48e3577c592ab881c39c6e7350bae 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -216,11 +216,13 @@ int32_t* taosGetErrno(); // dnode #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) //"Message not processed") #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) //"Dnode out of memory") -#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) //"No permission for disk files in dnode") -#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) //"Invalid message length") -#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) //"Action in progress") -#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") -#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" +#define TSDB_CODE_DND_DNODE_ID_NOT_MATCHED TAOS_DEF_ERROR_CODE(0, 0x0402) //"Dnode Id not matched") +#define TSDB_CODE_DND_MNODE_ALREADY_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0403) //"Mnode already deployed") +#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0404) //"No permission for disk files in dnode") +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0405) //"Invalid message length") +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0406) //"Action in progress") +#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0407) //"Too many vnode directories") +#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0408) //"Dnode is exiting" // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") diff --git a/source/dnode/mgmt/inc/dnodeDnode.h b/source/dnode/mgmt/inc/dnodeDnode.h index 9a2b564bd444e14e4db170c82eb5190f5d7b847e..2ca1368e63254d35cb8d8eddaf0b367c7af7d895 100644 --- a/source/dnode/mgmt/inc/dnodeDnode.h +++ b/source/dnode/mgmt/inc/dnodeDnode.h @@ -23,13 +23,13 @@ extern "C" { int32_t dnodeInitDnode(); void dnodeCleanupDnode(); -void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet); +void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet); int32_t dnodeGetDnodeId(); int64_t dnodeGetClusterId(); void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port); -void dnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); +void dnodeGetMnodeEpSetForPeer(SEpSet *epSet); +void dnodeGetMnodeEpSetForShell(SEpSet *epSet); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); #ifdef __cplusplus diff --git a/source/dnode/mgmt/inc/dnodeInt.h b/source/dnode/mgmt/inc/dnodeInt.h index 1306f2d700a7894d1a0d336a99216e42f556f281..472c467ad6df6add6c0a4599e582ad059a47e45e 100644 --- a/source/dnode/mgmt/inc/dnodeInt.h +++ b/source/dnode/mgmt/inc/dnodeInt.h @@ -35,14 +35,14 @@ extern int32_t dDebugFlag; #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} typedef enum { DN_RUN_STAT_INIT, DN_RUN_STAT_RUNNING, DN_RUN_STAT_STOPPED } EDnStat; -typedef void (*MsgFp)(SRpcMsg *pMsg, SRpcEpSet *pEpSet); +typedef void (*MsgFp)(SRpcMsg *pMsg, SEpSet *pEpSet); int32_t dnodeInit(); void dnodeCleanup(); EDnStat dnodeGetRunStat(); void dnodeSetRunStat(); -void dnodeGetStartup(SStartupStep *); +void dnodeGetStartup(SStartupMsg *); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/dnodeMnode.h b/source/dnode/mgmt/inc/dnodeMnode.h index e696acf4253e5fb77710afee04e1d2af81de4f08..28702dcb842d5c9373e109c31362983d848a6c0e 100644 --- a/source/dnode/mgmt/inc/dnodeMnode.h +++ b/source/dnode/mgmt/inc/dnodeMnode.h @@ -23,9 +23,8 @@ extern "C" { int32_t dnodeInitMnode(); void dnodeCleanupMnode(); -void dnodeProcessMnodeMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet); - -void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg); +void dnodeProcessMnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet); +int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/dnodeTransport.h b/source/dnode/mgmt/inc/dnodeTransport.h index e8223f4c06930f866f776bee348caaf451363197..95ca1b81e59cc89ff75c22cc458d70cf3ffffe03 100644 --- a/source/dnode/mgmt/inc/dnodeTransport.h +++ b/source/dnode/mgmt/inc/dnodeTransport.h @@ -24,7 +24,7 @@ extern "C" { int32_t dnodeInitTrans(); void dnodeCleanupTrans(); void dnodeSendMsgToMnode(SRpcMsg *rpcMsg); -void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(SEpSet *epSet, SRpcMsg *rpcMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/dnodeVnodes.h b/source/dnode/mgmt/inc/dnodeVnodes.h index 46dcf549d4c66087c271f0c8676d18c4a40a154f..6e9bce9ae5c721f494836ebcc776c183e4d342b8 100644 --- a/source/dnode/mgmt/inc/dnodeVnodes.h +++ b/source/dnode/mgmt/inc/dnodeVnodes.h @@ -23,7 +23,7 @@ extern "C" { int32_t dnodeInitVnodes(); void dnodeCleanupVnodes(); -void dnodeProcessVnodesMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet); +void dnodeProcessVnodesMsg(SRpcMsg *pMsg, SEpSet *pEpSet); void dnodeGetVnodes(SVnodeLoads *pVloads); #ifdef __cplusplus diff --git a/source/dnode/mgmt/src/dnodeDnode.c b/source/dnode/mgmt/src/dnodeDnode.c index 64694e9a46733452d105a9d10840fe0fe462e98b..5bf5b1d56a0af9e3c5462f1bd76e959ef662dfa2 100644 --- a/source/dnode/mgmt/src/dnodeDnode.c +++ b/source/dnode/mgmt/src/dnodeDnode.c @@ -27,15 +27,14 @@ static struct { int64_t clusterId; SDnodeEps *dnodeEps; SHashObj *dnodeHash; - SRpcEpSet mnodeEpSetForShell; - SRpcEpSet mnodeEpSetForPeer; + SEpSet mnodeEpSetForShell; + SEpSet mnodeEpSetForPeer; char file[PATH_MAX + 20]; uint32_t rebootTime; int8_t dropped; int8_t threadStop; pthread_t *threadId; pthread_mutex_t mutex; - MsgFp msgFp[TSDB_MSG_TYPE_MAX]; } tsDnode = {0}; int32_t dnodeGetDnodeId() { @@ -67,13 +66,13 @@ void dnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) { pthread_mutex_unlock(&tsDnode.mutex); } -void dnodeGetMnodeEpSetForPeer(SRpcEpSet *pEpSet) { +void dnodeGetMnodeEpSetForPeer(SEpSet *pEpSet) { pthread_mutex_lock(&tsDnode.mutex); *pEpSet = tsDnode.mnodeEpSetForPeer; pthread_mutex_unlock(&tsDnode.mutex); } -void dnodeGetMnodeEpSetForShell(SRpcEpSet *pEpSet) { +void dnodeGetMnodeEpSetForShell(SEpSet *pEpSet) { pthread_mutex_lock(&tsDnode.mutex); *pEpSet = tsDnode.mnodeEpSetForShell; pthread_mutex_unlock(&tsDnode.mutex); @@ -82,7 +81,7 @@ void dnodeGetMnodeEpSetForShell(SRpcEpSet *pEpSet) { void dnodeSendRedirectMsg(SRpcMsg *pMsg, bool forShell) { int32_t msgType = pMsg->msgType; - SRpcEpSet epSet = {0}; + SEpSet epSet = {0}; if (forShell) { dnodeGetMnodeEpSetForShell(&epSet); } else { @@ -107,7 +106,7 @@ void dnodeSendRedirectMsg(SRpcMsg *pMsg, bool forShell) { rpcSendRedirectRsp(pMsg->handle, &epSet); } -static void dnodeUpdateMnodeEpSet(SRpcEpSet *pEpSet) { +static void dnodeUpdateMnodeEpSet(SEpSet *pEpSet) { if (pEpSet == NULL || pEpSet->numOfEps <= 0) { dError("mnode is changed, but content is invalid, discard it"); return; @@ -127,7 +126,7 @@ static void dnodeUpdateMnodeEpSet(SRpcEpSet *pEpSet) { pthread_mutex_unlock(&tsDnode.mutex); } -static void dnodePrintEps() { +static void dnodePrintDnodes() { dDebug("print dnode endpoint list, num:%d", tsDnode.dnodeEps->dnodeNum); for (int32_t i = 0; i < tsDnode.dnodeEps->dnodeNum; i++) { SDnodeEp *ep = &tsDnode.dnodeEps->dnodeEps[i]; @@ -135,7 +134,7 @@ static void dnodePrintEps() { } } -static void dnodeResetEps(SDnodeEps *pEps) { +static void dnodeResetDnodes(SDnodeEps *pEps) { assert(pEps != NULL); int32_t size = sizeof(SDnodeEps) + pEps->dnodeNum * sizeof(SDnodeEp); @@ -171,7 +170,7 @@ static void dnodeResetEps(SDnodeEps *pEps) { taosHashPut(tsDnode.dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp)); } - dnodePrintEps(); + dnodePrintDnodes(); } static bool dnodeIsEpChanged(int32_t dnodeId, char *epStr) { @@ -189,7 +188,7 @@ static bool dnodeIsEpChanged(int32_t dnodeId, char *epStr) { return changed; } -static int32_t dnodeReadEps() { +static int32_t dnodeReadDnodes() { int32_t len = 0; int32_t maxLen = 30000; char *content = calloc(1, maxLen + 1); @@ -199,59 +198,59 @@ static int32_t dnodeReadEps() { fp = fopen(tsDnode.file, "r"); if (!fp) { dDebug("file %s not exist", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s since content is null", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } content[len] = 0; root = cJSON_Parse(content); if (root == NULL) { dError("failed to read %s since invalid json format", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId"); if (!dnodeId || dnodeId->type != cJSON_String) { dError("failed to read %s since dnodeId not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tsDnode.dnodeId = atoi(dnodeId->valuestring); cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { dError("failed to read %s since clusterId not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tsDnode.clusterId = atoll(clusterId->valuestring); cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); if (!dropped || dropped->type != cJSON_String) { dError("failed to read %s since dropped not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tsDnode.dropped = atoi(dropped->valuestring); cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos"); if (!dnodeInfos || dnodeInfos->type != cJSON_Array) { dError("failed to read %s since dnodeInfos not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos); if (dnodeInfosSize <= 0) { dError("failed to read %s since dnodeInfos size:%d invalid", tsDnode.file, dnodeInfosSize); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tsDnode.dnodeEps = calloc(1, dnodeInfosSize * sizeof(SDnodeEp) + sizeof(SDnodeEps)); if (tsDnode.dnodeEps == NULL) { dError("failed to calloc dnodeEpList since %s", strerror(errno)); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tsDnode.dnodeEps->dnodeNum = dnodeInfosSize; @@ -264,36 +263,36 @@ static int32_t dnodeReadEps() { cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId"); if (!dnodeId || dnodeId->type != cJSON_String) { dError("failed to read %s, dnodeId not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } pEp->dnodeId = atoi(dnodeId->valuestring); cJSON *isMnode = cJSON_GetObjectItem(dnodeInfo, "isMnode"); if (!isMnode || isMnode->type != cJSON_String) { dError("failed to read %s, isMnode not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } pEp->isMnode = atoi(isMnode->valuestring); cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { dError("failed to read %s, dnodeFqdn not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } tstrncpy(pEp->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort"); if (!dnodePort || dnodePort->type != cJSON_String) { dError("failed to read %s, dnodePort not found", tsDnode.file); - goto PRASE_EPS_OVER; + goto PRASE_DNODE_OVER; } pEp->dnodePort = atoi(dnodePort->valuestring); } dInfo("succcessed to read file %s", tsDnode.file); - dnodePrintEps(); + dnodePrintDnodes(); -PRASE_EPS_OVER: +PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); @@ -303,13 +302,13 @@ PRASE_EPS_OVER: return -1; } - dnodeResetEps(tsDnode.dnodeEps); + dnodeResetDnodes(tsDnode.dnodeEps); terrno = 0; return 0; } -static int32_t dnodeWriteEps() { +static int32_t dnodeWriteDnodes() { FILE *fp = fopen(tsDnode.file, "w"); if (!fp) { dError("failed to write %s since %s", tsDnode.file, strerror(errno)); @@ -391,7 +390,7 @@ static void dnodeUpdateCfg(SDnodeCfg *pCfg) { tsDnode.dropped = pCfg->dropped; dInfo("dnodeId is set to %d, clusterId is set to %" PRId64, pCfg->dnodeId, pCfg->clusterId); - dnodeWriteEps(); + dnodeWriteDnodes(); pthread_mutex_unlock(&tsDnode.mutex); } @@ -401,13 +400,13 @@ static void dnodeUpdateDnodeEps(SDnodeEps *pEps) { pthread_mutex_lock(&tsDnode.mutex); if (pEps->dnodeNum != tsDnode.dnodeEps->dnodeNum) { - dnodeResetEps(pEps); - dnodeWriteEps(); + dnodeResetDnodes(pEps); + dnodeWriteDnodes(); } else { int32_t size = pEps->dnodeNum * sizeof(SDnodeEp) + sizeof(SDnodeEps); if (memcmp(tsDnode.dnodeEps, pEps, size) != 0) { - dnodeResetEps(pEps); - dnodeWriteEps(); + dnodeResetDnodes(pEps); + dnodeWriteDnodes(); } } @@ -448,21 +447,26 @@ static void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) { static void dnodeProcessStartupReq(SRpcMsg *pMsg) { dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont); - SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep)); - dnodeGetStartup(pStep); + SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg)); + dnodeGetStartup(pStartup); - dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished); + dInfo("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStep, .contLen = sizeof(SStartupStep)}; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; rpcSendResponse(&rpcRsp); rpcFreeCont(pMsg->pCont); } static void *dnodeThreadRoutine(void *param) { int32_t ms = tsStatusInterval * 1000; + while (!tsDnode.threadStop) { + if (dnodeGetRunStat() != DN_RUN_STAT_RUNNING) { + continue; + } else { + dnodeSendStatusMsg(); + } taosMsleep(ms); - dnodeSendStatusMsg(); } } @@ -488,9 +492,9 @@ int32_t dnodeInitDnode() { return TSDB_CODE_DND_OUT_OF_MEMORY; } - int32_t code = dnodeReadEps(); + int32_t code = dnodeReadDnodes(); if (code != 0) { - dError("failed to read dnode endpoint file since %s", tstrerror(code)); + dError("failed to read file:%s since %s", tsDnode.file, tstrerror(code)); return code; } @@ -523,7 +527,7 @@ void dnodeCleanupDnode() { dInfo("dnode-dnode is cleaned up"); } -void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { +void dnodeProcessDnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { int32_t msgType = pMsg->msgType; if (msgType == TSDB_MSG_TYPE_STATUS_RSP && pEpSet) { diff --git a/source/dnode/mgmt/src/dnodeInt.c b/source/dnode/mgmt/src/dnodeInt.c index a1e7bc1c7e25ade263bec0147f4d9902fa25d2fa..e853ba7f147d83cc087dd553ca0b4a2f38a7fccd 100644 --- a/source/dnode/mgmt/src/dnodeInt.c +++ b/source/dnode/mgmt/src/dnodeInt.c @@ -26,30 +26,30 @@ #include "wal.h" static struct { - EDnStat runStatus; - SStartupStep startup; - SSteps *steps; + SStartupMsg startup; + EDnStat runStat; + SSteps *steps; } tsInt; -EDnStat dnodeGetRunStat() { return tsInt.runStatus; } +EDnStat dnodeGetRunStat() { return tsInt.runStat; } -void dnodeSetRunStat(EDnStat stat) { tsInt.runStatus = stat; } +void dnodeSetRunStat(EDnStat stat) { tsInt.runStat = stat; } static void dnodeReportStartup(char *name, char *desc) { - SStartupStep *startup = &tsInt.startup; - tstrncpy(startup->name, name, strlen(startup->name)); - tstrncpy(startup->desc, desc, strlen(startup->desc)); - startup->finished = 0; + SStartupMsg *pStartup = &tsInt.startup; + tstrncpy(pStartup->name, name, strlen(pStartup->name)); + tstrncpy(pStartup->desc, desc, strlen(pStartup->desc)); + pStartup->finished = 0; } static void dnodeReportStartupFinished(char *name, char *desc) { - SStartupStep *startup = &tsInt.startup; - tstrncpy(startup->name, name, strlen(startup->name)); - tstrncpy(startup->desc, desc, strlen(startup->desc)); - startup->finished = 1; + SStartupMsg *pStartup = &tsInt.startup; + tstrncpy(pStartup->name, name, strlen(pStartup->name)); + tstrncpy(pStartup->desc, desc, strlen(pStartup->desc)); + pStartup->finished = 1; } -void dnodeGetStartup(SStartupStep *pStep) { memcpy(pStep, &tsInt.startup, sizeof(SStartupStep)); } +void dnodeGetStartup(SStartupMsg *pStartup) { memcpy(pStartup, &tsInt.startup, sizeof(SStartupMsg)); } static int32_t dnodeCheckRunning(char *dir) { char filepath[256] = {0}; @@ -99,7 +99,7 @@ static int32_t dnodeInitDir() { } static int32_t dnodeInitMain() { - tsInt.runStatus = DN_RUN_STAT_STOPPED; + tsInt.runStat = DN_RUN_STAT_STOPPED; tscEmbedded = 1; taosIgnSIGPIPE(); taosBlockSIGPIPE(); diff --git a/source/dnode/mgmt/src/dnodeMnode.c b/source/dnode/mgmt/src/dnodeMnode.c index 6346b3386fc4aa9091f2c9688f5a26747d731576..4414d57f3e6ec9ca7cb7ca5c4b0e2c9c3495c2c0 100644 --- a/source/dnode/mgmt/src/dnodeMnode.c +++ b/source/dnode/mgmt/src/dnodeMnode.c @@ -17,9 +17,211 @@ #include "dnodeMnode.h" #include "dnodeDnode.h" #include "dnodeTransport.h" +#include "cJSON.h" #include "mnode.h" +static struct { + int8_t deployed; + int8_t dropped; + char file[PATH_MAX + 20]; + pthread_mutex_t mutex; +} tsMnode = {0}; + +static int32_t dnodeReadMnode() { + int32_t len = 0; + int32_t maxLen = 300; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + + fp = fopen(tsMnode.file, "r"); + if (!fp) { + dDebug("file %s not exist", tsMnode.file); + goto PRASE_MNODE_OVER; + } + + len = (int32_t)fread(content, 1, maxLen, fp); + if (len <= 0) { + dError("failed to read %s since content is null", tsMnode.file); + goto PRASE_MNODE_OVER; + } + + content[len] = 0; + root = cJSON_Parse(content); + if (root == NULL) { + dError("failed to read %s since invalid json format", tsMnode.file); + goto PRASE_MNODE_OVER; + } + + cJSON *deployed = cJSON_GetObjectItem(root, "deployed"); + if (!deployed || deployed->type != cJSON_String) { + dError("failed to read %s since deployed not found", tsMnode.file); + goto PRASE_MNODE_OVER; + } + tsMnode.deployed = atoi(deployed->valuestring); + + cJSON *dropped = cJSON_GetObjectItem(root, "dropped"); + if (!dropped || dropped->type != cJSON_String) { + dError("failed to read %s since dropped not found", tsMnode.file); + goto PRASE_MNODE_OVER; + } + tsMnode.dropped = atoi(dropped->valuestring); + + dInfo("succcessed to read file %s", tsMnode.file); + +PRASE_MNODE_OVER: + if (content != NULL) free(content); + if (root != NULL) cJSON_Delete(root); + if (fp != NULL) fclose(fp); + + return 0; +} + +static int32_t dnodeWriteMnode() { + FILE *fp = fopen(tsMnode.file, "w"); + if (!fp) { + dError("failed to write %s since %s", tsMnode.file, strerror(errno)); + return -1; + } + + int32_t len = 0; + int32_t maxLen = 300; + char *content = calloc(1, maxLen + 1); + + len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"deployed\": \"%d\",\n", tsMnode.dropped); + len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\",\n", tsMnode.dropped); + len += snprintf(content + len, maxLen - len, "}\n"); + + fwrite(content, 1, len, fp); + taosFsyncFile(fileno(fp)); + fclose(fp); + free(content); + terrno = 0; + + dInfo("successed to write %s", tsMnode.file); + return 0; +} + +static int32_t dnodeStartMnode(SCreateMnodeMsg *pCfg) { + int32_t code = 0; + + if (pCfg->dnodeId != dnodeGetDnodeId()) { + code = TSDB_CODE_DND_DNODE_ID_NOT_MATCHED; + dError("failed to start mnode since %s", tstrerror(code)); + return code; + } + + if (tsMnode.dropped) { + code = TSDB_CODE_DND_MNODE_ALREADY_DROPPED; + dError("failed to start mnode since %s", tstrerror(code)); + return code; + } + + if (tsMnode.deployed) { + dError("failed to start mnode since its already deployed"); + return 0; + } + + tsMnode.deployed = 1; + tsMnode.dropped = 0; + + code = dnodeWriteMnode(); + if (code != 0) { + tsMnode.deployed = 0; + dError("failed to start mnode since %s", tstrerror(code)); + return code; + } + + code = mnodeDeploy(); + if (code != 0) { + tsMnode.deployed = 0; + dError("failed to start mnode since %s", tstrerror(code)); + return code; + } + + code = mnodeStart(); + if (code != 0) { + tsMnode.deployed = 0; + dError("failed to start mnode since %s", tstrerror(code)); + return code; + } + + tsMnode.deployed = 1; + return 0; +} + +static int32_t dnodeDropMnode(SDropMnodeMsg *pCfg) { + int32_t code = 0; + + if (pCfg->dnodeId != dnodeGetDnodeId()) { + code = TSDB_CODE_DND_DNODE_ID_NOT_MATCHED; + dError("failed to drop mnode since %s", tstrerror(code)); + return code; + } + + if (tsMnode.dropped) { + code = TSDB_CODE_DND_MNODE_ALREADY_DROPPED; + dError("failed to drop mnode since %s", tstrerror(code)); + return code; + } + + if (!tsMnode.deployed) { + dError("failed to drop mnode since not deployed"); + return 0; + } + + mnodeStop(); + + tsMnode.deployed = 0; + tsMnode.dropped = 1; + + code = dnodeWriteMnode(); + if (code != 0) { + tsMnode.deployed = 1; + tsMnode.dropped = 0; + dError("failed to drop mnode since %s", tstrerror(code)); + return code; + } + + mnodeUnDeploy(); + + tsMnode.deployed = 0; + return 0; +} + +static void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg) { + SCreateMnodeMsg *pCfg = pMsg->pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + + int32_t code = dnodeStartMnode(pCfg); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); +} + +static void dnodeProcessDropMnodeReq(SRpcMsg *pMsg) { + SDropMnodeMsg *pCfg = pMsg->pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + + int32_t code = dnodeDropMnode(pCfg); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rspMsg); + rpcFreeCont(pMsg->pCont); +} + +static bool dnodeNeedDeployMnode() { + if (dnodeGetDnodeId() > 0) return false; + if (dnodeGetClusterId() > 0) return false; + if (strcmp(tsFirst, tsLocalEp) != 0) return false; + return true; +} + int32_t dnodeInitMnode() { + tsMnode.dropped = 0; + tsMnode.deployed = 0; + snprintf(tsMnode.file, sizeof(tsMnode.file), "%s/mnode.json", tsDnodeDir); + SMnodePara para; para.fp.GetDnodeEp = dnodeGetDnodeEp; para.fp.SendMsgToDnode = dnodeSendMsgToDnode; @@ -28,42 +230,64 @@ int32_t dnodeInitMnode() { para.dnodeId = dnodeGetDnodeId(); para.clusterId = dnodeGetClusterId(); - return mnodeInit(para); -} - -void dnodeCleanupMnode() { mnodeCleanup(); } + int32_t code = mnodeInit(para); + if (code != 0) { + dError("failed to init mnode module since %s", tstrerror(code)); + return code; + } -static int32_t dnodeStartMnode(SRpcMsg *pMsg) { - SCreateMnodeMsg *pCfg = pMsg->pCont; - pCfg->dnodeId = htonl(pCfg->dnodeId); - pCfg->mnodeNum = htonl(pCfg->mnodeNum); - for (int32_t i = 0; i < pCfg->mnodeNum; ++i) { - pCfg->mnodeEps[i].dnodeId = htonl(pCfg->mnodeEps[i].dnodeId); - pCfg->mnodeEps[i].dnodePort = htons(pCfg->mnodeEps[i].dnodePort); + code = dnodeReadMnode(); + if (code != 0) { + dError("failed to read file:%s since %s", tsMnode.file, tstrerror(code)); + return code; } - if (pCfg->dnodeId != dnodeGetDnodeId()) { - dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); - return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; + if (tsMnode.dropped) { + dError("mnode already dropped, undeploy it"); + mnodeUnDeploy(); + return 0; } - if (mnodeGetStatus() == MN_STATUS_READY) return 0; + if (!tsMnode.deployed) { + bool needDeploy = dnodeNeedDeployMnode(); + if (needDeploy) { + code = mnodeDeploy(); + } else { + return 0; + } - return mnodeDeploy(); -} + if (code != 0) { + dError("failed to deploy mnode since %s", tstrerror(code)); + return code; + } -void dnodeProcessCreateMnodeReq(SRpcMsg *pMsg) { - int32_t code = dnodeStartMnode(pMsg); + tsMnode.deployed = 1; + } - SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; + return mnodeStart(); +} - rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); +void dnodeCleanupMnode() { + if (tsMnode.deployed) { + mnodeStop(); + } + + mnodeCleanup(); } -void dnodeProcessMnodeMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - mnodeProcessMsg(pMsg); - // tsDnode.msgFp[TSDB_MSG_TYPE_CREATE_MNODE_IN] = dnodeProcessCreateMnodeReq; +void dnodeProcessMnodeMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_CREATE_MNODE_IN: + dnodeProcessCreateMnodeReq(pMsg); + break; + case TSDB_MSG_TYPE_DROP_MNODE_IN: + dnodeProcessDropMnodeReq(pMsg); + break; + default: + mnodeProcessMsg(pMsg); + } +} - // tsTrans.msgFp[TSDB_MSG_TYPE_DROP_MNODE_IN] = dnodeProcessDropMnodeReq; +int32_t dnodeGetUserAuthFromMnode(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + return mnodeRetriveAuth(user, spi, encrypt, secret, ckey); } \ No newline at end of file diff --git a/source/dnode/mgmt/src/dnodeTransport.c b/source/dnode/mgmt/src/dnodeTransport.c index 222d59ba556a91e967ff1e8ac06c687a102551fd..18e97e2b9fceded8d0162eba9f1c74c9ac0abfad 100644 --- a/source/dnode/mgmt/src/dnodeTransport.c +++ b/source/dnode/mgmt/src/dnodeTransport.c @@ -24,12 +24,11 @@ #include "dnodeDnode.h" #include "dnodeMnode.h" #include "dnodeVnodes.h" -#include "mnode.h" -#include "vnode.h" + static struct { - void *serverRpc; - void *clientRpc; + void *peerRpc; void *shellRpc; + void *clientRpc; MsgFp msgFp[TSDB_MSG_TYPE_MAX]; } tsTrans; @@ -119,8 +118,8 @@ static void dnodeInitMsgFp() { tsTrans.msgFp[TSDB_MSG_TYPE_STATUS_RSP] = dnodeProcessDnodeMsg; } -static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; +static void dnodeProcessPeerReq(SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rspMsg = {.handle = pMsg->handle}; int32_t msgType = pMsg->msgType; if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) { @@ -154,7 +153,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } } -static int32_t dnodeInitServer() { +static int32_t dnodeInitPeerServer() { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = tsDnodeDnodePort; @@ -165,8 +164,8 @@ static int32_t dnodeInitServer() { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - tsTrans.serverRpc = rpcOpen(&rpcInit); - if (tsTrans.serverRpc == NULL) { + tsTrans.peerRpc = rpcOpen(&rpcInit); + if (tsTrans.peerRpc == NULL) { dError("failed to init peer rpc server"); return -1; } @@ -175,15 +174,15 @@ static int32_t dnodeInitServer() { return 0; } -static void dnodeCleanupServer() { - if (tsTrans.serverRpc) { - rpcClose(tsTrans.serverRpc); - tsTrans.serverRpc = NULL; +static void dnodeCleanupPeerServer() { + if (tsTrans.peerRpc) { + rpcClose(tsTrans.peerRpc); + tsTrans.peerRpc = NULL; dInfo("dnode peer server is closed"); } } -static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { +static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SEpSet *pEpSet) { int32_t msgType = pMsg->msgType; if (dnodeGetRunStat() == DN_RUN_STAT_STOPPED) { @@ -205,7 +204,8 @@ static void dnodeProcessPeerRsp(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } static int32_t dnodeInitClient() { - char secret[TSDB_KEY_LEN] = "secret"; + char secret[TSDB_KEY_LEN] = "secret"; + SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND-C"; @@ -236,8 +236,8 @@ static void dnodeCleanupClient() { } } -static void dnodeProcessShellReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0}; +static void dnodeProcessShellReq(SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rspMsg = {.handle = pMsg->handle}; int32_t msgType = pMsg->msgType; if (dnodeGetRunStat() == DN_RUN_STAT_STOPPED) { @@ -272,41 +272,29 @@ static void dnodeProcessShellReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } } -void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL); } - -void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { - SRpcEpSet epSet = {0}; - dnodeGetMnodeEpSetForPeer(&epSet); - dnodeSendMsgToDnode(&epSet, rpcMsg); -} - static void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SRpcEpSet epSet = {0}; + SEpSet epSet = {0}; dnodeGetMnodeEpSetForPeer(&epSet); rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp); } static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); + int32_t code = dnodeGetUserAuthFromMnode(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pMsg; - rpcMsg.contLen = sizeof(SAuthMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_AUTH; - dDebug("user:%s, send auth msg to mnodes", user); + SRpcMsg rpcMsg = {.pCont = pMsg, .contLen = sizeof(SAuthMsg), .msgType = TSDB_MSG_TYPE_AUTH}; SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { dError("user:%s, auth msg received from mnodes, error:%s", user, tstrerror(rpcRsp.code)); } else { - SAuthRsp *pRsp = rpcRsp.pCont; dDebug("user:%s, auth msg received from mnodes", user); + SAuthRsp *pRsp = rpcRsp.pCont; memcpy(secret, pRsp->secret, TSDB_KEY_LEN); memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); *spi = pRsp->spi; @@ -317,7 +305,7 @@ static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, c return rpcRsp.code; } -static int32_t dnodeInitShell() { +static int32_t dnodeInitShellServer() { int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; @@ -344,7 +332,7 @@ static int32_t dnodeInitShell() { return 0; } -static void dnodeCleanupShell() { +static void dnodeCleanupShellServer() { if (tsTrans.shellRpc) { rpcClose(tsTrans.shellRpc); tsTrans.shellRpc = NULL; @@ -356,11 +344,11 @@ int32_t dnodeInitTrans() { return -1; } - if (dnodeInitServer() != 0) { + if (dnodeInitPeerServer() != 0) { return -1; } - if (dnodeInitShell() != 0) { + if (dnodeInitShellServer() != 0) { return -1; } @@ -368,7 +356,15 @@ int32_t dnodeInitTrans() { } void dnodeCleanupTrans() { - dnodeCleanupShell(); - dnodeCleanupServer(); + dnodeCleanupShellServer(); + dnodeCleanupPeerServer(); dnodeCleanupClient(); } + +void dnodeSendMsgToDnode(SEpSet *epSet, SRpcMsg *rpcMsg) { rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL); } + +void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) { + SEpSet epSet = {0}; + dnodeGetMnodeEpSetForPeer(&epSet); + dnodeSendMsgToDnode(&epSet, rpcMsg); +} \ No newline at end of file diff --git a/source/dnode/mgmt/src/dnodeVnodes.c b/source/dnode/mgmt/src/dnodeVnodes.c index f9032b419e9e9b1c89a07a21723057a7e7a16139..59f5cf0fd0393a60a1010d9e3fbf8f42e9c0f29b 100644 --- a/source/dnode/mgmt/src/dnodeVnodes.c +++ b/source/dnode/mgmt/src/dnodeVnodes.c @@ -21,6 +21,6 @@ int32_t dnodeInitVnodes() { return vnodeInit(); } void dnodeCleanupVnodes() { vnodeCleanup(); } -void dnodeProcessVnodesMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { vnodeProcessMsg(NULL, NULL); } +void dnodeProcessVnodesMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { vnodeProcessMsg(NULL, NULL); } void dnodeGetVnodes(SVnodeLoads *pVloads) {} \ No newline at end of file diff --git a/source/dnode/mnode/inc/mnodeInt.h b/source/dnode/mnode/inc/mnodeInt.h index 0ce47cbe36f314a50ee0bef3e91b0411958b23a8..5fcd7173eea6effbb5a2821f9d676159d1132287 100644 --- a/source/dnode/mnode/inc/mnodeInt.h +++ b/source/dnode/mnode/inc/mnodeInt.h @@ -22,12 +22,14 @@ extern "C" { #endif +typedef enum { MN_STATUS_UNINIT = 0, MN_STATUS_INIT = 1, MN_STATUS_READY = 2, MN_STATUS_CLOSING = 3 } EMnStatus; + tmr_h mnodeGetTimer(); int32_t mnodeGetDnodeId(); int64_t mnodeGetClusterId(); EMnStatus mnodeGetStatus(); -void mnodeSendMsgToDnode(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg); +void mnodeSendMsgToDnode(struct SEpSet *epSet, struct SRpcMsg *rpcMsg); void mnodeSendMsgToMnode(struct SRpcMsg *rpcMsg); void mnodeSendRedirectMsg(struct SRpcMsg *rpcMsg, bool forShell); void mnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); diff --git a/source/dnode/mnode/inc/mnodeMnode.h b/source/dnode/mnode/inc/mnodeMnode.h index cee96c7bf676f8b6f312a87c5c4228c3acf1b15d..0c8069a917a327737daa9b7e467ae9b678ef08b4 100644 --- a/source/dnode/mnode/inc/mnodeMnode.h +++ b/source/dnode/mnode/inc/mnodeMnode.h @@ -24,8 +24,8 @@ extern "C" { int32_t mnodeInitMnode(); void mnodeCleanupMnode(); -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect); -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect); +void mnodeGetMnodeEpSetForPeer(SEpSet *epSet, bool redirect); +void mnodeGetMnodeEpSetForShell(SEpSet *epSet, bool redirect); #ifdef __cplusplus } diff --git a/source/dnode/mnode/src/mnodeMnode.c b/source/dnode/mnode/src/mnodeMnode.c index d2bcd25fc77a0bc355be226df2703f3d7f785817..60e1627ad4b8614e7dc7c291fa237baf2b1cf46d 100644 --- a/source/dnode/mnode/src/mnodeMnode.c +++ b/source/dnode/mnode/src/mnodeMnode.c @@ -20,5 +20,5 @@ int32_t mnodeInitMnode() { return 0; } void mnodeCleanupMnode() {} -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect) {} -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect) {} \ No newline at end of file +void mnodeGetMnodeEpSetForPeer(SEpSet *epSet, bool redirect) {} +void mnodeGetMnodeEpSetForShell(SEpSet *epSet, bool redirect) {} \ No newline at end of file diff --git a/source/dnode/mnode/src/mnodeWorker.c b/source/dnode/mnode/src/mnodeWorker.c index 5ac44a3a065729c48ba2f4c9c6e73833b5ab2e93..e4b4c47ddf082dc74a7273784dd05a47a29f1baa 100644 --- a/source/dnode/mnode/src/mnodeWorker.c +++ b/source/dnode/mnode/src/mnodeWorker.c @@ -297,10 +297,10 @@ static void mnodeProcessWriteReq(SMnMsg *pMsg, void *unused) { if (!mnodeIsMaster()) { SMnRsp *rpcRsp = &pMsg->rpcRsp; - SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + SEpSet *epSet = rpcMallocCont(sizeof(SEpSet)); mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; - rpcRsp->len = sizeof(SRpcEpSet); + rpcRsp->len = sizeof(SEpSet); mDebug("msg:%p, app:%p type:%s in write queue, is redirected, numOfEps:%d inUse:%d", pMsg, ahandle, taosMsg[msgType], epSet->numOfEps, epSet->inUse); @@ -334,14 +334,14 @@ static void mnodeProcessReadReq(SMnMsg *pMsg, void *unused) { if (!mnodeIsMaster()) { SMnRsp *rpcRsp = &pMsg->rpcRsp; - SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + SEpSet *epSet = rpcMallocCont(sizeof(SEpSet)); if (!epSet) { code = TSDB_CODE_MND_OUT_OF_MEMORY; goto PROCESS_READ_REQ_END; } mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; - rpcRsp->len = sizeof(SRpcEpSet); + rpcRsp->len = sizeof(SEpSet); mDebug("msg:%p, app:%p type:%s in mread queue is redirected, numOfEps:%d inUse:%d", pMsg, ahandle, taosMsg[msgType], epSet->numOfEps, epSet->inUse); @@ -375,10 +375,10 @@ static void mnodeProcessPeerReq(SMnMsg *pMsg, void *unused) { if (!mnodeIsMaster()) { SMnRsp *rpcRsp = &pMsg->rpcRsp; - SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + SEpSet *epSet = rpcMallocCont(sizeof(SEpSet)); mnodeGetMnodeEpSetForPeer(epSet, true); rpcRsp->rsp = epSet; - rpcRsp->len = sizeof(SRpcEpSet); + rpcRsp->len = sizeof(SEpSet); mDebug("msg:%p, ahandle:%p type:%s in mpeer queue is redirected, numOfEps:%d inUse:%d", pMsg, ahandle, taosMsg[msgType], epSet->numOfEps, epSet->inUse); diff --git a/source/dnode/mnode/src/mondeInt.c b/source/dnode/mnode/src/mondeInt.c index 343384ba6742314000bb57818b6049e9be536dad..669273d8dc8cc8adc6982813e448642ed6957f03 100644 --- a/source/dnode/mnode/src/mondeInt.c +++ b/source/dnode/mnode/src/mondeInt.c @@ -54,7 +54,7 @@ int64_t mnodeGetClusterId() { return tsMint.clusterId; } EMnStatus mnodeGetStatus() { return tsMint.state; } -void mnodeSendMsgToDnode(struct SRpcEpSet *epSet, struct SRpcMsg *rpcMsg) { +void mnodeSendMsgToDnode(struct SEpSet *epSet, struct SRpcMsg *rpcMsg) { (*tsMint.fp.SendMsgToDnode)(epSet, rpcMsg); } @@ -250,3 +250,6 @@ void mnodeCleanup() { mInfo("mnode is cleaned up"); } } + +int32_t mnodeStart() { return 0; } +void mnodeStop() {} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/impl/src/vnodeInt.c index 7a395706e947ff3f01226e210f8039159cfefc0c..6f83542ceff8ab2831899965f34be78006d542bf 100644 --- a/source/dnode/vnode/impl/src/vnodeInt.c +++ b/source/dnode/vnode/impl/src/vnodeInt.c @@ -30,4 +30,4 @@ int32_t vnodeDrop(SVnode *pVnode) { return 0; } int32_t vnodeCompact(SVnode *pVnode) { return 0; } int32_t vnodeSync(SVnode *pVnode) { return 0; } -int32_t vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg) { return 0; } +void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg) {} diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index e047964b9483aff01afc5c55a1013f97dbdcfc27..fa49c20c7748f50ce944165e1e9e2c5d9a33da49 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -54,7 +54,7 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *, SRpcEpSet *); + void (*cfp)(SRpcMsg *, SEpSet *); int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; @@ -70,7 +70,7 @@ typedef struct { typedef struct { SRpcInfo *pRpc; // associated SRpcInfo - SRpcEpSet epSet; // ip list provided by app + SEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app struct SRpcConn *pConn; // pConn allocated char msgType; // message type @@ -84,7 +84,7 @@ typedef struct { int64_t rid; // refId returned by taosAddRef SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API - SRpcEpSet *pSet; // for synchronous API + SEpSet *pSet; // for synchronous API char msg[0]; // RpcHead starts from here } SRpcReqContext; @@ -383,7 +383,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { +void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -486,15 +486,15 @@ void rpcSendResponse(const SRpcMsg *pRsp) { return; } -void rpcSendRedirectRsp(void *thandle, const SRpcEpSet *pEpSet) { +void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SRpcEpSet); + rpcMsg.contLen = sizeof(SEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; - memcpy(rpcMsg.pCont, pEpSet, sizeof(SRpcEpSet)); + memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; @@ -516,7 +516,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); @@ -794,9 +794,9 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { } static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { - SRpcConn *pConn; - SRpcInfo *pRpc = pContext->pRpc; - SRpcEpSet *pEpSet = &pContext->epSet; + SRpcConn *pConn; + SRpcInfo *pRpc = pContext->pRpc; + SEpSet *pEpSet = &pContext->epSet; pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); if ( pConn == NULL || pConn->user[0] == 0) { @@ -926,7 +926,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { SRpcReqContext *pContext = pConn->pContext; if (pHead->code == TSDB_CODE_RPC_REDIRECT) { - if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SRpcEpSet)) { + if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SEpSet)) { // if EpSet is not included in the msg, treat it as NOT_READY pHead->code = TSDB_CODE_RPC_NOT_READY; } else { @@ -1126,12 +1126,12 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { pContext->pConn = NULL; if (pContext->pRsp) { // for synchronous API - memcpy(pContext->pSet, &pContext->epSet, sizeof(SRpcEpSet)); + memcpy(pContext->pSet, &pContext->epSet, sizeof(SEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { // for asynchronous API - SRpcEpSet *pEpSet = NULL; + SEpSet *pEpSet = NULL; if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; @@ -1175,7 +1175,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - SRpcEpSet *pEpSet = (SRpcEpSet*)pHead->content; + SEpSet *pEpSet = (SEpSet*)pHead->content; if (pEpSet->numOfEps > 0) { memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8e5d7a47fd9797d5aba796ab809859238e79dbf0..22fbeb1883b7bb744a0fb21ca9dbb525054e69c1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -228,6 +228,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists" // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, "Dnode out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_DNODE_ID_NOT_MATCHED, "Dnode Id not matched") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MNODE_ALREADY_DROPPED, "Mnode already deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") diff --git a/src/kit/shell/src/tnettest.c b/src/kit/shell/src/tnettest.c index 2a147ee4f17a38e9a00a6110fcc6f2d21fb7b131..c8a6b36fc721b6ac9c107fa0fa7fea82ec65d79f 100644 --- a/src/kit/shell/src/tnettest.c +++ b/src/kit/shell/src/tnettest.c @@ -338,7 +338,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { return pRpcConn; } -static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupStep *pStep) { +static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupMsg *pStep) { SRpcEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; @@ -374,7 +374,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p } int32_t code = 0; - if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupStep)) { + if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupMsg)) { memcpy(pStep, rspMsg.pCont, rspMsg.contLen); code = 1; } @@ -384,8 +384,8 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p return code; } -static int32_t taosNetParseStartup(SStartupStep *pCont) { - SStartupStep *pStep = pCont; +static int32_t taosNetParseStartup(SStartupMsg *pCont) { + SStartupMsg *pStep = pCont; uInfo("step:%s desc:%s", pStep->name, pStep->desc); if (pStep->finished) { @@ -398,7 +398,7 @@ static int32_t taosNetParseStartup(SStartupStep *pCont) { static void taosNetTestStartup(char *host, int32_t port) { uInfo("check startup, host:%s port:%d\n", host, port); - SStartupStep *pStep = malloc(sizeof(SStartupStep)); + SStartupMsg *pStep = malloc(sizeof(SStartupMsg)); while (1) { int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep); if (code > 0) { diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt deleted file mode 100644 index 320445f7f784884f8aa009e37182fc57a38bb96f..0000000000000000000000000000000000000000 --- a/src/plugins/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -ADD_SUBDIRECTORY(monitor) -ADD_SUBDIRECTORY(http) -IF (TD_LINUX AND TD_MQTT) - ADD_SUBDIRECTORY(mqtt) -ENDIF () \ No newline at end of file diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt deleted file mode 100644 index 89fdc141b66adafb9f882dd6f59eca54053aff6c..0000000000000000000000000000000000000000 --- a/src/plugins/http/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -ADD_LIBRARY(http ${SRC}) -TARGET_LINK_LIBRARIES(http z) - -IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(http taos_static) -ELSE () - TARGET_LINK_LIBRARIES(http taos) -ENDIF () - -IF (TD_ADMIN) - TARGET_LINK_LIBRARIES(http admin) -ENDIF () diff --git a/src/plugins/http/inc/httpAuth.h b/src/plugins/http/inc/httpAuth.h deleted file mode 100644 index 2ce9725d4bb57c9d24e3eddffb54707b0129b1cf..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpAuth.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_TOKEN_H -#define TDENGINE_HTTP_TOKEN_H - -int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len); -int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len); -int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h deleted file mode 100644 index af52fdd1ebbfe70fee3f278f94aa95faa1f4caba..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpContext.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_CONTEXT_H -#define TDENGINE_HTTP_CONTEXT_H - -#include "httpInt.h" - -bool httpInitContexts(); -void httpCleanupContexts(); -const char *httpContextStateStr(HttpContextState state); - -HttpContext *httpCreateContext(SOCKET fd); -bool httpInitContext(HttpContext *pContext); -HttpContext *httpGetContext(void * pContext); -void httpReleaseContext(HttpContext *pContext/*, bool clearRes*/); -void httpCloseContextByServer(HttpContext *pContext); -void httpCloseContextByApp(HttpContext *pContext); -void httpNotifyContextClose(HttpContext *pContext); -bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState); - -#endif diff --git a/src/plugins/http/inc/httpGcHandle.h b/src/plugins/http/inc/httpGcHandle.h deleted file mode 100644 index a3688c6c386e5d73c5084a54ad887ba305e425ea..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpGcHandle.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_GC_HANDLE_H -#define TDENGINE_GC_HANDLE_H - -#include "http.h" -#include "httpInt.h" -#include "httpUtil.h" -#include "httpResp.h" -#include "httpSql.h" - -#define GC_ROOT_URL_POS 0 -#define GC_ACTION_URL_POS 1 -#define GC_USER_URL_POS 2 -#define GC_PASS_URL_POS 3 - -void gcInitHandle(HttpServer* pServer); -bool gcProcessRequest(struct HttpContext* pContext); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpGcJson.h b/src/plugins/http/inc/httpGcJson.h deleted file mode 100644 index eb79528c711521aed5e3f3bcf6eacd93f64f01ce..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpGcJson.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_GC_JSON_H -#define TDENGINE_GC_JSON_H -#include "../../../../include/client/taos.h" -#include "httpHandle.h" -#include "httpJson.h" - -void gcInitQueryJson(HttpContext *pContext); -void gcCleanQueryJson(HttpContext *pContext); - -void gcStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); -void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd); -bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); - -void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpGzip.h b/src/plugins/http/inc/httpGzip.h deleted file mode 100644 index aeac79c9759078efb64669037c54122b2900d132..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpGzip.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef HTTP_GZIP_H -#define HTTP_GZIP_H - -#define EHTTP_GZIP_CHUNK_SIZE_DEFAULT (1024*16) - -typedef struct ehttp_gzip_s ehttp_gzip_t; - -typedef struct ehttp_gzip_callbacks_s ehttp_gzip_callbacks_t; -typedef struct ehttp_gzip_conf_s ehttp_gzip_conf_t; - -struct ehttp_gzip_callbacks_s { - void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len); -}; - -struct ehttp_gzip_conf_s { - int32_t get_header:2; // 0: not fetching header info - int32_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT -}; - -ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); -ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); -void ehttp_gzip_destroy(ehttp_gzip_t *gzip); - -int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len); -int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip); - -#endif // _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_ - diff --git a/src/plugins/http/inc/httpHandle.h b/src/plugins/http/inc/httpHandle.h deleted file mode 100644 index 3e6356d8053e5b953235a8134ee607c34789edbb..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpHandle.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_HANDLE_H -#define TDENGINE_HTTP_HANDLE_H - -// http request handler -void httpProcessRequest(HttpContext *pContext); -bool httpProcessData(HttpContext *pContext); - -#endif diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h deleted file mode 100644 index 6c567e23bc817957d7f376ef101f8e5ca88559e6..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpInt.h +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_INT_H -#define TDENGINE_HTTP_INT_H - -#include "os.h" -#include -#include "pthread.h" -#include "semaphore.h" -#include "tmempool.h" -#include "taosdef.h" -#include "tutil.h" -#include "zlib.h" -#include "http.h" -#include "httpLog.h" -#include "httpJson.h" -#include "httpParser.h" - -#define HTTP_MAX_CMD_SIZE 1024 -#define HTTP_MAX_BUFFER_SIZE 1024*1024*8 -#define HTTP_LABEL_SIZE 8 -#define HTTP_MAX_EVENTS 10 -#define HTTP_BUFFER_INIT 4096 -#define HTTP_BUFFER_SIZE 8388608 -#define HTTP_STEP_SIZE 4096 //http message get process step by step -#define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size -#define HTTP_GC_TARGET_SIZE 512 -#define HTTP_WRITE_RETRY_TIMES 500 -#define HTTP_WRITE_WAIT_TIME_MS 5 -#define HTTP_PASSWORD_LEN TSDB_UNI_LEN -#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + HTTP_PASSWORD_LEN) - -typedef enum HttpReqType { - HTTP_REQTYPE_OTHERS = 0, - HTTP_REQTYPE_LOGIN = 1, - HTTP_REQTYPE_HEARTBEAT = 2, - HTTP_REQTYPE_SINGLE_SQL = 3, - HTTP_REQTYPE_MULTI_SQL = 4 -} HttpReqType; - -typedef enum { - HTTP_SERVER_INIT, - HTTP_SERVER_RUNNING, - HTTP_SERVER_CLOSING, - HTTP_SERVER_CLOSED -} HttpServerStatus; - -typedef enum { - HTTP_CONTEXT_STATE_READY, - HTTP_CONTEXT_STATE_HANDLING, - HTTP_CONTEXT_STATE_DROPPING, - HTTP_CONTEXT_STATE_CLOSED -} HttpContextState; - -typedef enum { - HTTP_CMD_TYPE_UN_SPECIFIED, - HTTP_CMD_TYPE_CREATE_DB, - HTTP_CMD_TYPE_CREATE_STBALE, - HTTP_CMD_TYPE_INSERT -} HttpSqlCmdType; - -typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState; - -typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType; - -struct HttpContext; -struct HttpThread; - -typedef struct { - char id[HTTP_SESSION_ID_LEN]; - int32_t refCount; - void * taos; -} HttpSession; - -typedef struct { - // used by single cmd - char *nativSql; - int32_t numOfRows; - int32_t code; - - // these are the locations in the buffer - int32_t tagNames[TSDB_MAX_TAGS]; - int32_t tagValues[TSDB_MAX_TAGS]; - int32_t timestamp; - int32_t metric; - int32_t stable; - int32_t table; - int32_t values; - int32_t sql; - - // used by multi-cmd - int8_t cmdType; - int8_t cmdReturnType; - int8_t cmdState; - int8_t tagNum; -} HttpSqlCmd; - -typedef struct { - HttpSqlCmd *cmds; - int16_t pos; - int16_t size; - int16_t maxSize; - int32_t bufferPos; - int32_t bufferSize; - char * buffer; -} HttpSqlCmds; - -typedef struct { - char *module; - bool (*fpDecode)(struct HttpContext *pContext); -} HttpDecodeMethod; - -typedef struct { - void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); - void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd); - bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); - void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows); - void (*initJsonFp)(struct HttpContext *pContext); - void (*cleanJsonFp)(struct HttpContext *pContext); - bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code); - void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code); -} HttpEncodeMethod; - -typedef enum { - EHTTP_CONTEXT_PROCESS_FAILED = 0x01, - EHTTP_CONTEXT_PARSER_FAILED = 0x02 -} EHTTP_CONTEXT_FAILED_CAUSE; - -typedef struct HttpContext { - int32_t refCount; - SOCKET fd; - uint32_t accessTimes; - uint32_t lastAccessTime; - int32_t state; - uint8_t reqType; - uint8_t parsed; - uint8_t error; - char ipstr[22]; - char user[TSDB_USER_LEN]; // parsed from auth token or login message - char pass[HTTP_PASSWORD_LEN]; - char db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN]; - TAOS * taos; - void * ppContext; - HttpSession *session; - z_stream gzipStream; - HttpParser *parser; - HttpSqlCmd singleCmd; - HttpSqlCmds *multiCmds; - JsonBuf * jsonBuf; - HttpEncodeMethod *encodeMethod; - HttpDecodeMethod *decodeMethod; - struct HttpThread *pThread; -} HttpContext; - -typedef struct HttpThread { - pthread_t thread; - HttpContext * pHead; - pthread_mutex_t threadMutex; - bool stop; - EpollFd pollFd; - int32_t numOfContexts; - int32_t threadId; - char label[HTTP_LABEL_SIZE << 1]; - bool (*processData)(HttpContext *pContext); -} HttpThread; - -typedef struct HttpServer { - char label[HTTP_LABEL_SIZE]; - uint32_t serverIp; - uint16_t serverPort; - int8_t stop; - int8_t reserve; - SOCKET fd; - int32_t numOfThreads; - int32_t methodScannerLen; - int32_t requestNum; - int32_t status; - pthread_t thread; - HttpThread * pThreads; - void * contextCache; - void * sessionCache; - pthread_mutex_t serverMutex; - HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE]; - bool (*processData)(HttpContext *pContext); -} HttpServer; - -extern const char *httpKeepAliveStr[]; -extern const char *httpVersionStr[]; -extern HttpServer tsHttpServer; - -#endif diff --git a/src/plugins/http/inc/httpJson.h b/src/plugins/http/inc/httpJson.h deleted file mode 100644 index 3595ad926fa08a7ba87fc758c2b0472d34617488..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpJson.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_JSON_H -#define TDENGINE_HTTP_JSON_H - -#include -#include - -#define JSON_BUFFER_SIZE 16384 -struct HttpContext; - -enum { JsonNumber, JsonString, JsonBoolean, JsonArray, JsonObject, JsonNull }; - -extern char JsonItmTkn; -extern char JsonObjStt; -extern char JsonObjEnd; -extern char JsonArrStt; -extern char JsonArrEnd; -extern char JsonStrStt; -extern char JsonStrEnd; -extern char JsonPairTkn; -extern char JsonNulTkn[]; -extern char JsonTrueTkn[]; -extern char JsonFalseTkn[]; - -typedef struct { - int32_t size; - int32_t total; - char* lst; - char buf[JSON_BUFFER_SIZE]; - struct HttpContext* pContext; -} JsonBuf; - -// http response -int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz); -int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz); -int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz); - -// builder callback -typedef void (*httpJsonBuilder)(JsonBuf* buf, void* jsnHandle); - -// buffer -void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext); -void httpWriteJsonBufHead(JsonBuf* buf); -int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast); -void httpWriteJsonBufEnd(JsonBuf* buf); - -// value -void httpJsonString(JsonBuf* buf, char* sVal, int32_t len); -void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len); -void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen); -void httpJsonInt64(JsonBuf* buf, int64_t num); -void httpJsonUInt64(JsonBuf* buf, uint64_t num); -void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision); -void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision); -void httpJsonInt(JsonBuf* buf, int32_t num); -void httpJsonUInt(JsonBuf* buf, uint32_t num); -void httpJsonFloat(JsonBuf* buf, float num); -void httpJsonDouble(JsonBuf* buf, double num); -void httpJsonNull(JsonBuf* buf); -void httpJsonBool(JsonBuf* buf, int32_t val); - -// pair -void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen); -void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen); -void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len); -void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num); -void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num); -void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num); -void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num); -void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num); -void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen); - -// object -void httpJsonPairArray(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle); -void httpJsonPairObject(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle); -void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle); -void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuidler, void* jsonHandle); - -// print -void httpJsonTestBuf(JsonBuf* buf, int32_t safety); -void httpJsonToken(JsonBuf* buf, char c); -void httpJsonItemToken(JsonBuf* buf); -void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len); - -// quick -void httpJsonPairStatus(JsonBuf* buf, int32_t code); - -// http json printer -JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext); -void httpFreeJsonBuf(struct HttpContext* pContext); - -#endif diff --git a/src/plugins/http/inc/httpLog.h b/src/plugins/http/inc/httpLog.h deleted file mode 100644 index 9c145a43e81629781ca43f0e9af48612f8011a22..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpLog.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_LOG_H -#define TDENGINE_HTTP_LOG_H - -#include "tlog.h" - -extern int32_t httpDebugFlag; - -#define httpFatal(...) { if (httpDebugFlag & DEBUG_FATAL) { taosPrintLog("HTP FATAL ", 255, __VA_ARGS__); }} -#define httpError(...) { if (httpDebugFlag & DEBUG_ERROR) { taosPrintLog("HTP ERROR ", 255, __VA_ARGS__); }} -#define httpWarn(...) { if (httpDebugFlag & DEBUG_WARN) { taosPrintLog("HTP WARN ", 255, __VA_ARGS__); }} -#define httpInfo(...) { if (httpDebugFlag & DEBUG_INFO) { taosPrintLog("HTP ", 255, __VA_ARGS__); }} -#define httpDebug(...) { if (httpDebugFlag & DEBUG_DEBUG) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }} -#define httpTrace(...) { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }} -#define httpTraceL(...){ if (httpDebugFlag & DEBUG_TRACE) { taosPrintLongString("HTP ", httpDebugFlag, __VA_ARGS__); }} - -#endif diff --git a/src/plugins/http/inc/httpMetricsHandle.h b/src/plugins/http/inc/httpMetricsHandle.h deleted file mode 100644 index e05a8ce6874452e616bf14245db515ef7e8c182c..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpMetricsHandle.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_HTTPMETRICSHANDLE_H -#define TDENGINE_HTTPMETRICSHANDLE_H - -#include "http.h" -#include "httpInt.h" -#include "httpUtil.h" -#include "httpResp.h" - -void metricsInitHandle(HttpServer* httpServer); - -bool metricsProcessRequest(struct HttpContext* httpContext); - -#endif // TDENGINE_HTTPMETRICHANDLE_H diff --git a/src/plugins/http/inc/httpParser.h b/src/plugins/http/inc/httpParser.h deleted file mode 100644 index f7b55958c88bf9961a6a57e14aa2de286d004ece..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpParser.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef HTTP_PARSER_H -#define HTTP_PARSER_H -#include "httpGzip.h" - -#define HTTP_MAX_URL 6 // http url stack size - -#define HTTP_CODE_CONTINUE 100 -#define HTTP_CODE_SWITCHING_PROTOCOL 101 -#define HTTP_CODE_PROCESSING 102 -#define HTTP_CODE_EARLY_HINTS 103 -#define HTTP_CODE_OK 200 -#define HTTP_CODE_CREATED 201 -#define HTTP_CODE_ACCEPTED 202 -#define HTTP_CODE_NON_AUTHORITATIVE_INFO 203 -#define HTTP_CODE_NO_CONTENT 204 -#define HTTP_CODE_RESET_CONTENT 205 -#define HTTP_CODE_PARTIAL_CONTENT 206 -#define HTTP_CODE_MULTI_STATUS 207 -#define HTTP_CODE_ALREADY_REPORTED 208 -#define HTTP_CODE_IM_USED 226 -#define HTTP_CODE_MULTIPLE_CHOICE 300 -#define HTTP_CODE_MOVED_PERMANENTLY 301 -#define HTTP_CODE_FOUND 302 -#define HTTP_CODE_SEE_OTHER 303 -#define HTTP_CODE_NOT_MODIFIED 304 -#define HTTP_CODE_USE_PROXY 305 -#define HTTP_CODE_UNUSED 306 -#define HTTP_CODE_TEMPORARY_REDIRECT 307 -#define HTTP_CODE_PERMANENT_REDIRECT 308 -#define HTTP_CODE_BAD_REQUEST 400 -#define HTTP_CODE_UNAUTHORIZED 401 -#define HTTP_CODE_PAYMENT_REQUIRED 402 -#define HTTP_CODE_FORBIDDEN 403 -#define HTTP_CODE_NOT_FOUND 404 -#define HTTP_CODE_METHOD_NOT_ALLOWED 405 -#define HTTP_CODE_NOT_ACCEPTABLE 406 -#define HTTP_CODE_PROXY_AUTH_REQUIRED 407 -#define HTTP_CODE_REQUEST_TIMEOUT 408 -#define HTTP_CODE_CONFLICT 409 -#define HTTP_CODE_GONE 410 -#define HTTP_CODE_LENGTH_REQUIRED 411 -#define HTTP_CODE_PRECONDITION_FAILED 412 -#define HTTP_CODE_PAYLOAD_TOO_LARGE 413 -#define HTTP_CODE_URI_TOO_LARGE 414 -#define HTTP_CODE_UNSUPPORTED_MEDIA_TYPE 415 -#define HTTP_CODE_RANGE_NOT_SATISFIABLE 416 -#define HTTP_CODE_EXPECTATION_FAILED 417 -#define HTTP_CODE_IM_A_TEAPOT 418 -#define HTTP_CODE_MISDIRECTED_REQUEST 421 -#define HTTP_CODE_UNPROCESSABLE_ENTITY 422 -#define HTTP_CODE_LOCKED 423 -#define HTTP_CODE_FAILED_DEPENDENCY 424 -#define HTTP_CODE_TOO_EARLY 425 -#define HTTP_CODE_UPGRADE_REQUIRED 426 -#define HTTP_CODE_PRECONDITION_REQUIRED 428 -#define HTTP_CODE_TOO_MANY_REQUESTS 429 -#define HTTP_CODE_REQ_HDR_FIELDS_TOO_LARGE 431 -#define HTTP_CODE_UNAVAIL_4_LEGAL_REASONS 451 -#define HTTP_CODE_INTERNAL_SERVER_ERROR 500 -#define HTTP_CODE_NOT_IMPLEMENTED 501 -#define HTTP_CODE_BAD_GATEWAY 502 -#define HTTP_CODE_SERVICE_UNAVAILABLE 503 -#define HTTP_CODE_GATEWAY_TIMEOUT 504 -#define HTTP_CODE_HTTP_VER_NOT_SUPPORTED 505 -#define HTTP_CODE_VARIANT_ALSO_NEGOTIATES 506 -#define HTTP_CODE_INSUFFICIENT_STORAGE 507 -#define HTTP_CODE_LOOP_DETECTED 508 -#define HTTP_CODE_NOT_EXTENDED 510 -#define HTTP_CODE_NETWORK_AUTH_REQUIRED 511 - -typedef enum HTTP_PARSER_STATE { - HTTP_PARSER_BEGIN, - HTTP_PARSER_REQUEST_OR_RESPONSE, - HTTP_PARSER_METHOD, - HTTP_PARSER_TARGET, - HTTP_PARSER_HTTP_VERSION, - HTTP_PARSER_SP, - HTTP_PARSER_STATUS_CODE, - HTTP_PARSER_REASON_PHRASE, - HTTP_PARSER_CRLF, - HTTP_PARSER_HEADER, - HTTP_PARSER_HEADER_KEY, - HTTP_PARSER_HEADER_VAL, - HTTP_PARSER_CHUNK_SIZE, - HTTP_PARSER_CHUNK, - HTTP_PARSER_END, - HTTP_PARSER_ERROR, - HTTP_PARSER_OPTIONAL_SP -} HTTP_PARSER_STATE; - -typedef enum HTTP_AUTH_TYPE { - HTTP_INVALID_AUTH, - HTTP_BASIC_AUTH, - HTTP_TAOSD_AUTH -} HTTP_AUTH_TYPE; - -typedef enum HTTP_VERSION { - HTTP_VERSION_10 = 0, - HTTP_VERSION_11 = 1, - HTTP_VERSION_12 = 2, - HTTP_INVALID_VERSION -} HTTP_VERSION; - -typedef enum HTTP_KEEPALIVE { - HTTP_KEEPALIVE_NO_INPUT = 0, - HTTP_KEEPALIVE_ENABLE = 1, - HTTP_KEEPALIVE_DISABLE = 2 -} HTTP_KEEPALIVE; - -typedef struct HttpString { - char * str; - int32_t pos; - int32_t size; -} HttpString; - -typedef struct HttpStatus { - int32_t code; - char * desc; -} HttpStatus; - -typedef struct HttpStack{ - int8_t *stacks; - int32_t pos; - int32_t size; -} HttpStack; - -struct HttpContext; -typedef struct HttpParser { - struct HttpContext *pContext; - ehttp_gzip_t *gzip; - HttpStack stacks; - HttpString str; - HttpString body; - HttpString path[HTTP_MAX_URL]; - char * method; - char * target; - char * version; - char * reasonPhrase; - char * key; - char * val; - char * authContent; - int8_t httpVersion; - int8_t acceptEncodingGzip; - int8_t acceptEncodingChunked; - int8_t contentLengthSpecified; - int8_t contentChunked; - int8_t transferGzip; - int8_t transferChunked; - int8_t keepAlive; - int8_t authType; - int32_t contentLength; - int32_t chunkSize; - int32_t receivedChunkSize; - int32_t receivedSize; - int32_t statusCode; - int8_t inited; - int8_t parsed; - int16_t httpCode; - int32_t parseCode; -} HttpParser; - -void httpInitParser(HttpParser *parser); -HttpParser *httpCreateParser(struct HttpContext *pContext); -void httpClearParser(HttpParser *parser); -void httpDestroyParser(HttpParser *parser); -int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len); -char * httpGetStatusDesc(int32_t statusCode); - -#endif diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h deleted file mode 100644 index 1ffbd5148164ab4d1f74bdac781c8fcf731ad772..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpQueue.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_QUEUE_H -#define TDENGINE_HTTP_QUEUE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows); - -bool httpInitResultQueue(); -void httpCleanupResultQueue(); -void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/plugins/http/inc/httpResp.h b/src/plugins/http/inc/httpResp.h deleted file mode 100644 index a528bcc39e6e23f47ecc9bc2557a3e12d7cf335f..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpResp.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_RESP_H -#define TDENGINE_HTTP_RESP_H - -#include "httpInt.h" - -enum _httpRespTempl { - HTTP_RESPONSE_JSON_OK, - HTTP_RESPONSE_JSON_ERROR, - HTTP_RESPONSE_OK, - HTTP_RESPONSE_ERROR, - HTTP_RESPONSE_CHUNKED_UN_COMPRESS, - HTTP_RESPONSE_CHUNKED_COMPRESS, - HTTP_RESPONSE_OPTIONS, - HTTP_RESPONSE_GRAFANA, - HTTP_RESP_END -}; - -extern const char *httpRespTemplate[]; - -void httpSendErrorResp(HttpContext *pContext, int32_t errNo); -void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg); -void httpSendSuccResp(HttpContext *pContext, char *desc); -void httpSendOptionResp(HttpContext *pContext, char *desc); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpRestHandle.h b/src/plugins/http/inc/httpRestHandle.h deleted file mode 100644 index df405685e91a520d8b948fdc056d9386626368a2..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpRestHandle.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_REST_HANDLE_H -#define TDENGINE_REST_HANDLE_H - -#include "http.h" -#include "httpInt.h" -#include "httpUtil.h" -#include "httpResp.h" -#include "httpSql.h" - -#define REST_ROOT_URL_POS 0 -#define REST_ACTION_URL_POS 1 -#define REST_USER_USEDB_URL_POS 2 -#define REST_PASS_URL_POS 3 - -void restInitHandle(HttpServer* pServer); -bool restProcessRequest(struct HttpContext* pContext); - -#endif diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h deleted file mode 100644 index d1f5a4552114b400b4eb4b1b5b614202001cdeb9..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpRestJson.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_REST_JSON_H -#define TDENGINE_REST_JSON_H -#include -#include "../../../../include/client/taos.h" -#include "httpHandle.h" -#include "httpJson.h" - -#define REST_JSON_SUCCESS "succ" -#define REST_JSON_SUCCESS_LEN 4 -#define REST_JSON_FAILURE "error" -#define REST_JSON_FAILURE_LEN 5 -#define REST_JSON_STATUS "status" -#define REST_JSON_STATUS_LEN 6 -#define REST_JSON_CODE "code" -#define REST_JSON_CODE_LEN 4 -#define REST_JSON_DESC "desc" -#define REST_JSON_DESC_LEN 4 -#define REST_JSON_DATA "data" -#define REST_JSON_DATA_LEN 4 -#define REST_JSON_HEAD "head" -#define REST_JSON_HEAD_LEN 4 -#define REST_JSON_HEAD_INFO "column_meta" -#define REST_JSON_HEAD_INFO_LEN 11 -#define REST_JSON_ROWS "rows" -#define REST_JSON_ROWS_LEN 4 -#define REST_JSON_AFFECT_ROWS "affected_rows" -#define REST_JSON_AFFECT_ROWS_LEN 13 - -#define REST_TIMESTAMP_FMT_LOCAL_STRING 0 -#define REST_TIMESTAMP_FMT_TIMESTAMP 1 -#define REST_TIMESTAMP_FMT_UTC_STRING 2 - -void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows); - -void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); -bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); -bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); -bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); -void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd); - -#endif diff --git a/src/plugins/http/inc/httpServer.h b/src/plugins/http/inc/httpServer.h deleted file mode 100644 index 58ed3545f35e9d832a330c142d398f2316291aa6..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpServer.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_SERVER_H -#define TDENGINE_HTTP_SERVER_H - -#include "httpInt.h" - -bool httpInitConnect(); -void httpCleanUpConnect(); - -void *httpInitServer(char *ip, uint16_t port, char *label, int32_t numOfThreads, void *fp, void *shandle); -void httpCleanUpServer(HttpServer *pServer); - -#endif diff --git a/src/plugins/http/inc/httpSession.h b/src/plugins/http/inc/httpSession.h deleted file mode 100644 index 393e720f69c899cd9b78babdd0c550f14132a014..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpSession.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_SESSION_H -#define TDENGINE_HTTP_SESSION_H - -bool httpInitSessions(); -void httpCleanUpSessions(); - -// http session method -void httpCreateSession(HttpContext *pContext, void *taos); -void httpGetSession(HttpContext *pContext); -void httpReleaseSession(HttpContext *pContext); - -#endif diff --git a/src/plugins/http/inc/httpSql.h b/src/plugins/http/inc/httpSql.h deleted file mode 100644 index 325545af47aa5dc8810eee0172f197f14b19c442..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpSql.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_SQL_H -#define TDENGINE_HTTP_SQL_H - - -int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...); -int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...); -int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize); -int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext); - -bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize); -bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize); -bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize); -void httpFreeMultiCmds(HttpContext *pContext); - -HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext); -HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext); -int32_t httpCurSqlCmdPos(HttpContext *pContext); - -void httpTrimTableName(char *name); -int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name); -char * httpGetCmdsString(HttpContext *pContext, int32_t pos); - -int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql); -void httpCheckFreeEscapedSql(char *oldSql, char *newSql); - -#endif diff --git a/src/plugins/http/inc/httpSystem.h b/src/plugins/http/inc/httpSystem.h deleted file mode 100644 index afe49edb2f76a907ae869aa85a05ab41625a5531..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpSystem.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_SYSTEM_H -#define TDENGINE_HTTP_SYSTEM_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -int32_t httpInitSystem(); -int32_t httpStartSystem(); -void httpStopSystem(); -void httpCleanUpSystem(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/plugins/http/inc/httpTgHandle.h b/src/plugins/http/inc/httpTgHandle.h deleted file mode 100644 index 6a3a7bfa4a450da1dab035a094093a143d8736c8..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpTgHandle.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TG_HANDLE_H -#define TDENGINE_TG_HANDLE_H - -#include "http.h" -#include "httpInt.h" -#include "httpUtil.h" -#include "httpResp.h" -#include "httpSql.h" - -#define TG_ROOT_URL_POS 0 -#define TG_DB_URL_POS 1 -#define TG_USER_URL_POS 2 -#define TG_PASS_URL_POS 3 - -void tgInitHandle(HttpServer *pServer); -void tgCleanupHandle(); - -bool tgProcessRquest(struct HttpContext *pContext); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpTgJson.h b/src/plugins/http/inc/httpTgJson.h deleted file mode 100644 index a7ec4f8cb66b6686c8e86ec217286494bc55734b..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpTgJson.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TG_JSON_H -#define TDENGINE_TG_JSON_H - -#include "../../../../include/client/taos.h" -#include "httpHandle.h" -#include "httpJson.h" - -void tgInitQueryJson(HttpContext *pContext); -void tgCleanQueryJson(HttpContext *pContext); -void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); -void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd); -void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows); -bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code); -void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code); - -#endif \ No newline at end of file diff --git a/src/plugins/http/inc/httpUtil.h b/src/plugins/http/inc/httpUtil.h deleted file mode 100644 index 21690ebca96d35423e126a9e747d8ce6bb5a43a0..0000000000000000000000000000000000000000 --- a/src/plugins/http/inc/httpUtil.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_HTTP_UTIL_H -#define TDENGINE_HTTP_UTIL_H - -bool httpCheckUsedbSql(char *sql); -bool httpCheckAlterSql(char *sql); -void httpTimeToString(int32_t t, char *buf, int32_t buflen); - -bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp); -bool httpParseRequest(HttpContext *pContext); -int32_t httpCheckReadCompleted(HttpContext *pContext); -void httpReadDirtyData(HttpContext *pContext); - -int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData); -int32_t httpGzipCompressInit(HttpContext *pContext); -int32_t httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen, - char *outDestData, int32_t *outDestDataLen, bool isTheLast); - -// http request parser -void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod); - -#endif diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c deleted file mode 100644 index 2ef5406823b63862ba8e529ed4caea5bf7e4ec32..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpAuth.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tkey.h" -#include "tutil.h" -#include "http.h" -#include "httpInt.h" -#include "httpAuth.h" - -#define KEY_DES_4 4971256377704625728L - -int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { - token[len] = '\0'; - int32_t outlen = 0; - char * base64 = (char *)base64_decode(token, len, &outlen); - if (base64 == NULL || outlen == 0) { - httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); - free(base64); - return -1; - } - - char *user = strstr(base64, ":"); - if (user == NULL) { - httpError("context:%p, fd:%d, basic token:%s invalid format", pContext, pContext->fd, token); - free(base64); - return -1; - } - - int32_t user_len = (int32_t)(user - base64); - if (user_len < 1 || user_len >= TSDB_USER_LEN) { - httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token); - free(base64); - return -1; - } - strncpy(pContext->user, base64, (size_t)user_len); - pContext->user[user_len] = 0; - - char * password = user + 1; - int32_t pass_len = (int32_t)((base64 + outlen) - password); - if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) { - httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); - free(base64); - return -1; - } - strncpy(pContext->pass, password, (size_t)pass_len); - pContext->pass[pass_len] = 0; - - free(base64); - httpDebug("context:%p, fd:%d, basic token parsed success, user:%s", pContext, pContext->fd, pContext->user); - return 0; -} - -int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { - token[len] = '\0'; - int32_t outlen = 0; - unsigned char *base64 = base64_decode(token, len, &outlen); - if (base64 == NULL || outlen == 0) { - httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); - if (base64) free(base64); - return 01; - } - if (outlen != (TSDB_USER_LEN + HTTP_PASSWORD_LEN)) { - httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token); - free(base64); - return -1; - } - - char *descrypt = taosDesDecode(KEY_DES_4, (char *)base64, outlen); - if (descrypt == NULL) { - httpError("context:%p, fd:%d, taosd token:%s descrypt error", pContext, pContext->fd, token); - free(base64); - return -1; - } else { - tstrncpy(pContext->user, descrypt, sizeof(pContext->user)); - tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass)); - - httpDebug("context:%p, fd:%d, taosd token:%s parsed success, user:%s", pContext, pContext->fd, token, - pContext->user); - free(base64); - free(descrypt); - return 0; - } -} - -int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { - char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; - size_t size = sizeof(pContext->user); - tstrncpy(buffer, pContext->user, size); - size = sizeof(pContext->pass); - tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size); - - char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + HTTP_PASSWORD_LEN); - char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + HTTP_PASSWORD_LEN); - - size_t len = strlen(base64); - tstrncpy(token, base64, len + 1); - free(encrypt); - free(base64); - - httpDebug("context:%p, fd:%d, generate taosd token:%s", pContext, pContext->fd, token); - - return 0; -} diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c deleted file mode 100644 index 11945453c56ab7fdd1fc8b0c4f2510bbbdda1a6e..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpContext.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "tsocket.h" -#include "tutil.h" -#include "ttimer.h" -#include "tglobal.h" -#include "tcache.h" -#include "hash.h" -#include "httpInt.h" -#include "httpResp.h" -#include "httpSql.h" -#include "httpSession.h" -#include "httpContext.h" -#include "httpParser.h" - -static void httpDestroyContext(void *data); - -static void httpRemoveContextFromEpoll(HttpContext *pContext) { - HttpThread *pThread = pContext->pThread; - if (pContext->fd >= 0) { - epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); -#ifdef WINDOWS - SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); -#else - SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1); -#endif - taosCloseSocket(fd); - } -} - -static void httpDestroyContext(void *data) { - HttpContext *pContext = *(HttpContext **)data; - if (pContext->fd > 0) taosCloseSocket(pContext->fd); - - HttpThread *pThread = pContext->pThread; - httpRemoveContextFromEpoll(pContext); - httpReleaseSession(pContext); - atomic_sub_fetch_32(&pThread->numOfContexts, 1); - - httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, - data, pContext->pThread->label, pContext->pThread->numOfContexts); - pContext->pThread = 0; - pContext->state = HTTP_CONTEXT_STATE_CLOSED; - - // avoid double free - httpFreeJsonBuf(pContext); - httpFreeMultiCmds(pContext); - - if (pContext->parser) { - httpDestroyParser(pContext->parser); - pContext->parser = NULL; - } - - tfree(pContext); -} - -bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_CACHE_PTR_KEY, 2, true, httpDestroyContext, "restc"); - if (tsHttpServer.contextCache == NULL) { - httpError("failed to init context cache"); - return false; - } - - return true; -} - -void httpCleanupContexts() { - if (tsHttpServer.contextCache != NULL) { - SCacheObj *cache = tsHttpServer.contextCache; - httpInfo("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); - taosCacheCleanup(tsHttpServer.contextCache); - tsHttpServer.contextCache = NULL; - } -} - -const char *httpContextStateStr(HttpContextState state) { - switch (state) { - case HTTP_CONTEXT_STATE_READY: - return "ready"; - case HTTP_CONTEXT_STATE_HANDLING: - return "handling"; - case HTTP_CONTEXT_STATE_DROPPING: - return "dropping"; - case HTTP_CONTEXT_STATE_CLOSED: - return "closed"; - default: - return "unknown"; - } -} - -void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); } - -bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { - return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); -} - -HttpContext *httpCreateContext(SOCKET fd) { - HttpContext *pContext = calloc(1, sizeof(HttpContext)); - if (pContext == NULL) return NULL; - - pContext->fd = fd; - pContext->lastAccessTime = taosGetTimestampSec(); - pContext->state = HTTP_CONTEXT_STATE_READY; - pContext->parser = httpCreateParser(pContext); - - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &pContext, - sizeof(TSDB_CACHE_PTR_TYPE), 3000); - pContext->ppContext = ppContext; - httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - - // set the ref to 0 - taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); - - return pContext; -} - -HttpContext *httpGetContext(void *ptr) { - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)ptr; - HttpContext **ppContext = taosCacheAcquireByKey(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); - - if (ppContext) { - HttpContext *pContext = *ppContext; - if (pContext) { - int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1); - httpTrace("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount); - return pContext; - } - } - return NULL; -} - -void httpReleaseContext(HttpContext *pContext/*, bool clearRes*/) { - int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1); - if (refCount < 0) { - httpError("context:%p, is already released, refCount:%d", pContext, refCount); - return; - } - /* - if (clearRes) { - if (pContext->parser) { - httpClearParser(pContext->parser); - } - memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - } - */ - HttpContext **ppContext = pContext->ppContext; - httpTrace("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount); - - if (tsHttpServer.contextCache != NULL) { - taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false); - } else { - httpDebug("context:%p, won't be destroyed for cache is already released", pContext); - // httpDestroyContext((void **)(&ppContext)); - } -} - -bool httpInitContext(HttpContext *pContext) { - pContext->accessTimes++; - pContext->lastAccessTime = taosGetTimestampSec(); - - pContext->reqType = HTTP_REQTYPE_OTHERS; - pContext->encodeMethod = NULL; - memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - - httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); - return true; -} - -void httpCloseContextByApp(HttpContext *pContext) { - HttpParser *parser = pContext->parser; - pContext->parsed = false; - bool keepAlive = true; - - if (pContext->error == true) { - keepAlive = false; - } else if (parser && parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { - keepAlive = false; - } else if (parser && parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { - keepAlive = false; - } - - if (keepAlive) { - if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) { - httpTrace("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) { - httpRemoveContextFromEpoll(pContext); - httpTrace("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { - httpRemoveContextFromEpoll(pContext); - httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd); - } else { - httpRemoveContextFromEpoll(pContext); - httpError("context:%p, fd:%d, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd, - httpContextStateStr(pContext->state), pContext->state); - } - } else { - httpRemoveContextFromEpoll(pContext); - httpTrace("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd, - httpContextStateStr(pContext->state), pContext->state); - } - - httpReleaseContext(pContext/*, true*/); -} - -void httpCloseContextByServer(HttpContext *pContext) { - if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) { - httpTrace("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) { - httpTrace("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) { - httpTrace("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd); - } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) { - httpTrace("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd); - } else { - httpError("context:%p, fd:%d, unknown state:%d", pContext, pContext->fd, pContext->state); - } - - pContext->parsed = false; - httpRemoveContextFromEpoll(pContext); - httpReleaseContext(pContext/*, true*/); -} diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c deleted file mode 100644 index 883afcc4ecba7d4959cb6dfe1cc7f8c1d83b7cae..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpGcHandle.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "cJSON.h" -#include "httpLog.h" -#include "httpGcHandle.h" -#include "httpGcJson.h" - -static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest}; -static HttpEncodeMethod gcHeartBeatMethod = { - .startJsonFp = NULL, - .stopJsonFp = gcSendHeartBeatResp, - .buildQueryJsonFp = NULL, - .buildAffectRowJsonFp = NULL, - .initJsonFp = NULL, - .cleanJsonFp = NULL, - .checkFinishedFp = NULL, - .setNextCmdFp = NULL -}; - -static HttpEncodeMethod gcQueryMethod = { - .startJsonFp = NULL, - .stopJsonFp = gcStopQueryJson, - .buildQueryJsonFp = gcBuildQueryJson, - .buildAffectRowJsonFp = NULL, - .initJsonFp = gcInitQueryJson, - .cleanJsonFp = gcCleanQueryJson, - .checkFinishedFp = NULL, - .setNextCmdFp = NULL -}; - -void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); } - -bool gcGetUserFromUrl(HttpContext* pContext) { - HttpParser* pParser = pContext->parser; - if (pParser->path[GC_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].str, TSDB_USER_LEN); - return true; -} - -bool gcGetPassFromUrl(HttpContext* pContext) { - HttpParser* pParser = pContext->parser; - if (pParser->path[GC_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, HTTP_PASSWORD_LEN); - return true; -} - -bool gcProcessLoginRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, user:%s, process grafana login msg", pContext, pContext->fd, pContext->user); - pContext->reqType = HTTP_REQTYPE_LOGIN; - return true; -} - -/** - * Process the query request - * @param fd for http send back - * @param context is taos conn - * @param filter, the request format is json, such as - */ - -// https://github.com/grafana/grafana/blob/master/docs/sources/plugins/developing/datasources.md -// input -//[{ -// "refId": "A", -// "alias" : "taosd", -// "sql" : "select first(taosd) from sys.mem where ts > now-6h and ts < now interval(20000a)" -//}, -//{ -// "refId": "B", -// "alias" : "system", -// "sql" : "select first(taosd) from sys.mem where ts > now-6h and ts < now interval(20000a)" -//}] -// output -//[{ -// "datapoints": [[339.386719, -// 1537873132000], -// [339.656250, -// 1537873162400], -// [339.656250, -// 1537873192600], -// [339.656250, -// 1537873222800], -// [339.589844, -// 1537873253200], -// [339.964844, -// 1537873283400], -// [340.093750, -// 1537873313800], -// [340.093750, -// 1537873344000], -// [340.093750, -// 1537873374200], -// [340.093750, -// 1537873404600]], -// "refId": "A", -// "target" : "taosd" -//}, -//{ -// "datapoints": [[339.386719, -// 1537873132000], -// [339.656250, -// 1537873162400], -// [339.656250, -// 1537873192600], -// [339.656250, -// 1537873222800], -// [339.589844, -// 1537873253200], -// [339.964844, -// 1537873283400], -// [340.093750, -// 1537873313800], -// [340.093750, -// 1537873344000], -// [340.093750, -// 1537873374200], -// [340.093750, -// 1537873404600]], -// "refId": "B", -// "target" : "system" -//}] - -bool gcProcessQueryRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, process grafana query msg", pContext, pContext->fd); - - char* filter = pContext->parser->body.str; - if (filter == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT); - return false; - } - - cJSON* root = cJSON_Parse(filter); - if (root == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR); - return false; - } - - int32_t size = cJSON_GetArraySize(root); - if (size <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_NULL); - cJSON_Delete(root); - return false; - } - - if (size > 100) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_SIZE); - cJSON_Delete(root); - return false; - } - - if (!httpMallocMultiCmds(pContext, size, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - -#define ESCAPE_ERROR_PROC(code, context, root) \ - do { \ - if (code != TSDB_CODE_SUCCESS) { \ - httpSendErrorResp(context, code); \ - \ - cJSON_Delete(root); \ - return false; \ - } \ - } while (0) - - for (int32_t i = 0; i < size; ++i) { - cJSON* query = cJSON_GetArrayItem(root, i); - if (query == NULL) continue; - - cJSON* refId = cJSON_GetObjectItem(query, "refId"); - if (refId == NULL || refId->valuestring == NULL || strlen(refId->valuestring) == 0) { - httpDebug("context:%p, fd:%d, user:%s, refId is null", pContext, pContext->fd, pContext->user); - continue; - } - - char *newStr = NULL; - int32_t retCode = 0; - - retCode = httpCheckAllocEscapeSql(refId->valuestring, &newStr); - ESCAPE_ERROR_PROC(retCode, pContext, root); - - int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, newStr); - httpCheckFreeEscapedSql(refId->valuestring, newStr); - if (refIdBuffer == -1) { - httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user); - break; - } - - cJSON* alias = cJSON_GetObjectItem(query, "alias"); - int32_t aliasBuffer = -1; - if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { - retCode = httpCheckAllocEscapeSql(alias->valuestring, &newStr); - ESCAPE_ERROR_PROC(retCode, pContext, root); - - aliasBuffer = httpAddToSqlCmdBuffer(pContext, newStr); - httpCheckFreeEscapedSql(alias->valuestring, newStr); - if (aliasBuffer == -1) { - httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user); - break; - } - } - if (aliasBuffer == -1) { - aliasBuffer = httpAddToSqlCmdBuffer(pContext, ""); - } - - cJSON* sql = cJSON_GetObjectItem(query, "sql"); - if (sql == NULL || sql->valuestring == NULL || strlen(sql->valuestring) == 0) { - httpDebug("context:%p, fd:%d, user:%s, sql is null", pContext, pContext->fd, pContext->user); - continue; - } - - retCode = httpCheckAllocEscapeSql(sql->valuestring, &newStr); - ESCAPE_ERROR_PROC(retCode, pContext, root); - - int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, newStr); - httpCheckFreeEscapedSql(sql->valuestring, newStr); - if (sqlBuffer == -1) { - httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user); - break; - } - - HttpSqlCmd* cmd = httpNewSqlCmd(pContext); - if (cmd == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - - cmd->sql = sqlBuffer; - cmd->values = refIdBuffer; - cmd->table = aliasBuffer; - cmd->numOfRows = 0; // hack way as target flags - cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way - - if (cmd->timestamp == -1) { - httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd, - pContext->user); - break; - } - } - -#undef ESCAPE_ERROR_PROC - - pContext->reqType = HTTP_REQTYPE_MULTI_SQL; - pContext->encodeMethod = &gcQueryMethod; - pContext->multiCmds->pos = 0; - - return true; -} - -bool gcProcessHeartbeatRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, process grafana heartbeat msg", pContext, pContext->fd); - pContext->reqType = HTTP_REQTYPE_HEARTBEAT; - pContext->encodeMethod = &gcHeartBeatMethod; - return true; -} - -/** - * Process get/post/options msg, such as login and logout - */ -bool gcProcessRequest(struct HttpContext* pContext) { - if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "login")) { - gcGetUserFromUrl(pContext); - gcGetPassFromUrl(pContext); - } - - if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); - return false; - } - - if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "query")) { - return gcProcessQueryRequest(pContext); - } else if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "heartbeat")) { - return gcProcessHeartbeatRequest(pContext); - } else if (httpUrlMatch(pContext, GC_ACTION_URL_POS, "login")) { - return gcProcessLoginRequest(pContext); - } else { - return gcProcessHeartbeatRequest(pContext); - } -} diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c deleted file mode 100644 index 2d361d37940a93c3627ef53883a342d12183e6a1..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpGcJson.c +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "httpGcHandle.h" -#include "httpGcJson.h" -#include "httpJson.h" -#include "httpResp.h" - -unsigned char *base64_decode(const char *value, int inlen, int *outlen); - -void gcInitQueryJson(HttpContext *pContext) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - httpInitJsonBuf(jsonBuf, pContext); - httpWriteJsonBufHead(jsonBuf); - - // data array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); -} - -void gcCleanQueryJson(HttpContext *pContext) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // array end - httpJsonToken(jsonBuf, JsonArrEnd); - - httpWriteJsonBufEnd(jsonBuf); -} - -void gcWriteTargetStartJson(JsonBuf *jsonBuf, char *refId, char *target) { - if (strlen(target) == 0) { - target = refId; - } - - // object begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonObjStt); - - // target section - httpJsonPair(jsonBuf, "refId", 5, refId, (int32_t)strlen(refId)); - httpJsonPair(jsonBuf, "target", 6, target, (int32_t)strlen(target)); - - // data begin - httpJsonPairHead(jsonBuf, "datapoints", 10); - - // data array begin - httpJsonToken(jsonBuf, JsonArrStt); -} - -void gcWriteTargetEndJson(JsonBuf *jsonBuf) { - // data array end - httpJsonToken(jsonBuf, JsonArrEnd); - - // object end - httpJsonToken(jsonBuf, JsonObjEnd); -} - -void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // write end of target - if (cmd->numOfRows != 0) { - gcWriteTargetEndJson(jsonBuf); - } -} - -bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return false; - - int32_t num_fields = taos_num_fields(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - if (num_fields == 0) { - return false; - } - - int32_t precision = taos_result_precision(result); - - // such as select count(*) from sys.cpu - // such as select count(*) from sys.cpu group by ipaddr - // such as select count(*) from sys.cpu interval(1d) - // such as select count(*) from sys.cpu interval(1d) group by ipaddr - // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) - int32_t dataFields = -1; - int32_t groupFields = -1; - bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; - if (hasTimestamp) { - dataFields = 1; - if (num_fields > 2) groupFields = num_fields - 1; - } else { - dataFields = 0; - if (num_fields > 1) groupFields = num_fields - 1; - } - - char *refIdBuffer = httpGetCmdsString(pContext, cmd->values); - char *aliasBuffer = httpGetCmdsString(pContext, cmd->table); - char *targetBuffer = httpGetCmdsString(pContext, cmd->timestamp); - - if (groupFields == -1 && cmd->numOfRows == 0) { - gcWriteTargetStartJson(jsonBuf, refIdBuffer, aliasBuffer); - } - cmd->numOfRows += numOfRows; - - for (int32_t k = 0; k < numOfRows; ++k) { - TAOS_ROW row = taos_fetch_row(result); - if (row == NULL) { - cmd->numOfRows--; - continue; - } - int32_t *length = taos_fetch_lengths(result); - - // for group by - if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; - int32_t len; - len = snprintf(target, HTTP_GC_TARGET_SIZE, "%s{", aliasBuffer); - for (int32_t i = dataFields + 1; i < num_fields; i++) { - if (row[i] == NULL) { - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:nil", fields[i].name); - - if (i < num_fields - 1) { - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", "); - } - - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d", fields[i].name, *((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d", fields[i].name, *((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d,", fields[i].name, *((int32_t *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%" PRId64, fields[i].name, *((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%.5f", fields[i].name, GET_FLOAT_VAL(row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%.9f", fields[i].name, GET_DOUBLE_VAL(row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - if (row[i] != NULL) { - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name); - memcpy(target + len, (char *)row[i], length[i]); - len = (int32_t)strlen(target); - } - break; - default: - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, "-"); - break; - } - if (i < num_fields - 1) { - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", "); - } - } - len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "}"); - - if (strcmp(target, targetBuffer) != 0) { - // first target not write this section - if (strlen(targetBuffer) != 0) { - gcWriteTargetEndJson(jsonBuf); - } - - // start new target - gcWriteTargetStartJson(jsonBuf, refIdBuffer, target); - strncpy(targetBuffer, target, HTTP_GC_TARGET_SIZE); - } - } // end of group by - - // data row array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - for (int32_t i = dataFields; i >= 0; i--) { - httpJsonItemToken(jsonBuf); - if (row == NULL || i >= num_fields || row[i] == NULL) { - httpJsonOriginString(jsonBuf, "null", 4); - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - httpJsonInt(jsonBuf, *((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - httpJsonInt(jsonBuf, *((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - httpJsonInt(jsonBuf, *((int32_t *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - httpJsonInt64(jsonBuf, *((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - httpJsonDouble(jsonBuf, GET_DOUBLE_VAL(row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char *)row[i], fields[i].bytes); - break; - case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t ts = convertTimePrecision(*((int64_t *)row[i]), precision, TSDB_TIME_PRECISION_MILLI); - httpJsonInt64(jsonBuf, ts); - break; - } - default: - httpJsonString(jsonBuf, "-", 1); - break; - } - } - - if (dataFields == 0) { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, "-", 1); - } - - // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - } - - return true; -} - -void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - char *desc = "Grafana server receive a quest from you!"; - - httpInitJsonBuf(jsonBuf, pContext); - - httpJsonToken(jsonBuf, JsonObjStt); - httpJsonPair(jsonBuf, "message", (int32_t)strlen("message"), desc, (int32_t)strlen(desc)); - httpJsonToken(jsonBuf, JsonObjEnd); - - char head[1024]; - - int32_t hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], (jsonBuf->lst - jsonBuf->buf)); - httpWriteBuf(pContext, head, hLen); - httpWriteBuf(pContext, jsonBuf->buf, (int32_t)(jsonBuf->lst - jsonBuf->buf)); -} diff --git a/src/plugins/http/src/httpGzip.c b/src/plugins/http/src/httpGzip.c deleted file mode 100644 index 6a6e995c182f9c248e84b53588ee383ada8194e1..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpGzip.c +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "zlib.h" -#include "httpGzip.h" - -typedef enum { - EHTTP_GZIP_INITING, - EHTTP_GZIP_READY, - EHTTP_GZIP_CLOSED, -} EHTTP_GZIP_STATE; - -struct ehttp_gzip_s { - ehttp_gzip_conf_t conf; - ehttp_gzip_callbacks_t callbacks; - void * arg; - z_stream * gzip; - gz_header * header; - char * chunk; - int32_t state; -}; - -static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {} - -static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { - switch (gzip->state) { - case EHTTP_GZIP_READY: { - inflateEnd(gzip->gzip); - } break; - default: - break; - } - if (gzip->gzip) { - free(gzip->gzip); - gzip->gzip = NULL; - } - if (gzip->header) { - free(gzip->header); - gzip->header = NULL; - } - if (gzip->chunk) { - free(gzip->chunk); - gzip->chunk = NULL; - } - gzip->state = EHTTP_GZIP_CLOSED; -} - -ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { - ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip)); - if (!gzip) return NULL; - - do { - gzip->conf = conf; - gzip->callbacks = callbacks; - gzip->arg = arg; - if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; - gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip)); - if (gzip->conf.get_header) { - gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header)); - } - if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; - gzip->chunk = (char *)malloc(gzip->conf.chunk_size); - if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; - gzip->gzip->zalloc = Z_NULL; - gzip->gzip->zfree = Z_NULL; - gzip->gzip->opaque = Z_NULL; - - // 863 windowBits can also be greater than 15 for optional gzip decoding. Add - // 864 32 to windowBits to enable zlib and gzip decoding with automatic header - // 865 detection, or add 16 to decode only the gzip format (the zlib format will - // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see - // 868 below), inflate() will not automatically decode concatenated gzip streams. - // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state - // 870 would need to be reset to continue decoding a subsequent gzip stream. - int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS - if (ret != Z_OK) break; - if (gzip->header) { - ret = inflateGetHeader(gzip->gzip, gzip->header); - } - if (ret != Z_OK) break; - - gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; - gzip->state = EHTTP_GZIP_READY; - return gzip; - } while (0); - - ehttp_gzip_destroy(gzip); - return NULL; -} - -ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); - -void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { - ehttp_gzip_cleanup(gzip); - - free(gzip); -} - -int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { - if (gzip->state != EHTTP_GZIP_READY) return -1; - if (len <= 0) return 0; - - gzip->gzip->next_in = (z_const Bytef*)buf; - gzip->gzip->avail_in = len; - - while (gzip->gzip->avail_in) { - int32_t ret; - if (gzip->header) { - ret = inflate(gzip->gzip, Z_BLOCK); - } else { - ret = inflate(gzip->gzip, Z_SYNC_FLUSH); - } - if (ret != Z_OK && ret != Z_STREAM_END) return -1; - - if (gzip->gzip->avail_out > 0) { - if (ret != Z_STREAM_END) continue; - } - - int32_t _len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); - - gzip->gzip->next_out[0] = '\0'; - gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, _len); - gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; - } - - return 0; -} - -int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { - if (gzip->state != EHTTP_GZIP_READY) return -1; - - gzip->gzip->next_in = NULL; - gzip->gzip->avail_in = 0; - - int32_t ret; - ret = inflate(gzip->gzip, Z_FINISH); - - if (ret != Z_STREAM_END) return -1; - - int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); - - gzip->gzip->next_out[0] = '\0'; - gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = NULL; - gzip->gzip->avail_out = 0; - - return 0; -} diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c deleted file mode 100644 index 9719d93824b50064ec1cf23677c641428434592c..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpHandle.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "httpInt.h" -#include "httpResp.h" -#include "httpContext.h" -#include "httpHandle.h" - -bool httpDecodeRequest(HttpContext* pContext) { - if (pContext->decodeMethod->fpDecode == NULL) { - return false; - } - - return (*pContext->decodeMethod->fpDecode)(pContext); -} - -/** - * Process the request from http pServer - */ -bool httpProcessData(HttpContext* pContext) { - if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) { - httpTrace("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd, - httpContextStateStr(pContext->state)); - pContext->error = true; - httpCloseContextByApp(pContext); - return false; - } - - // handle Cross-domain request - if (strcmp(pContext->parser->method, "OPTIONS") == 0) { - httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd); - httpSendOptionResp(pContext, "process options request success"); - } else { - if (!httpDecodeRequest(pContext)) { - /* - * httpCloseContextByApp has been called when parsing the error - */ - // httpCloseContextByApp(pContext); - } else { - httpClearParser(pContext->parser); - httpProcessRequest(pContext); - } - } - - return true; -} diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c deleted file mode 100644 index 86e0f2f40beffdf7d035ba3329d51bb69c2cf796..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpJson.c +++ /dev/null @@ -1,534 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "taoserror.h" -#include "tglobal.h" -#include "http.h" -#include "httpLog.h" -#include "httpJson.h" -#include "httpResp.h" -#include "httpUtil.h" - -#define MAX_NUM_STR_SZ 25 - -char JsonItmTkn = ','; -char JsonObjStt = '{'; -char JsonObjEnd = '}'; -char JsonArrStt = '['; -char JsonArrEnd = ']'; -char JsonStrStt = '\"'; -char JsonStrEnd = '\"'; -char JsonPairTkn = ':'; -char JsonNulTkn[] = "null"; -char JsonTrueTkn[] = "true"; -char JsonFalseTkn[] = "false"; - -int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz) { - int32_t len; - int32_t countWait = 0; - int32_t writeLen = 0; - - do { - if (pContext->fd > 2) { - len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); - } else { - return sz; - } - - if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), - countWait); - if (++countWait > HTTP_WRITE_RETRY_TIMES) break; - taosMsleep(HTTP_WRITE_WAIT_TIME_MS); - continue; - } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, - strerror(errno)); - break; - } else { - countWait = 0; - writeLen += len; - } - } while (writeLen < sz); - - return writeLen; -} - -int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) { - int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); - if (writeSz != sz) { - httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, sz, - writeSz, buf); - } else { - httpTrace("context:%p, fd:%d, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, sz, writeSz, buf); - } - - return writeSz; -} - -int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) { - int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); - if (writeSz != sz) { - httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, - writeSz); - } - - return writeSz; -} - -int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { - int32_t remain = 0; - char sLen[24]; - int32_t srcLen = (int32_t)(buf->lst - buf->buf); - - if (buf->pContext->fd <= 0) { - httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); - buf->pContext->fd = -1; - } - - /* - * HTTP servers often use compression to optimize transmission, for example - * with Content-Encoding: gzip or Content-Encoding: deflate. - * If both compression and chunked encoding are enabled, then the content stream is first compressed, then chunked; - * so the chunk encoding itself is not compressed, and the data in each chunk is not compressed individually. - * The remote endpoint then decodes the stream by concatenating the chunks and uncompressing the result. - */ - - if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { - if (buf->lst == buf->buf) { - httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); - return 0; // there is no data to dump. - } else { - int32_t len = sprintf(sLen, "%x\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, - buf->buf); - httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); - } - } else { - char compressBuf[JSON_BUFFER_SIZE] = {0}; - int32_t compressBufLen = JSON_BUFFER_SIZE; - int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); - if (ret == 0) { - if (compressBufLen > 0) { - int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, - buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); - httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); - } else { - httpDebug("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext, - buf->pContext->fd, isTheLast, buf->buf); - remain = 0; // there is no data to dump. - } - } else { - httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s", - buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); - remain = 0; - } - } - - httpWriteBufNoTrace(buf->pContext, "\r\n", 2); - buf->total += (int32_t)(buf->lst - buf->buf); - buf->lst = buf->buf; - memset(buf->buf, 0, (size_t)buf->size); - return remain; -} - -void httpWriteJsonBufHead(JsonBuf* buf) { - if (buf->pContext->fd <= 0) { - buf->pContext->fd = -1; - } - - char msg[1024] = {0}; - int32_t len = -1; - - if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { - len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], - httpKeepAliveStr[buf->pContext->parser->keepAlive]); - } else { - len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], - httpKeepAliveStr[buf->pContext->parser->keepAlive]); - } - - httpWriteBuf(buf->pContext, (const char*)msg, len); -} - -void httpWriteJsonBufEnd(JsonBuf* buf) { - if (buf->pContext->fd <= 0) { - httpTrace("context:%p, fd:%d, json buf fd is 0", buf->pContext, buf->pContext->fd); - buf->pContext->fd = -1; - } - - httpWriteJsonBufBody(buf, true); - httpWriteBufNoTrace(buf->pContext, "0\r\n\r\n", 5); // end of chunked resp -} - -void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) { - buf->lst = buf->buf; - buf->total = 0; - buf->size = JSON_BUFFER_SIZE; // option setting - buf->pContext = pContext; - memset(buf->lst, 0, JSON_BUFFER_SIZE); - - if (pContext->parser->acceptEncodingGzip == 1 && tsHttpEnableCompress) { - httpGzipCompressInit(buf->pContext); - } - - httpTrace("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd); -} - -void httpJsonItemToken(JsonBuf* buf) { - char c = *(buf->lst - 1); - if (c == JsonArrStt || c == JsonObjStt || c == JsonPairTkn || c == JsonItmTkn) { - return; - } - if (buf->lst > buf->buf) httpJsonToken(buf, JsonItmTkn); -} - -void httpJsonString(JsonBuf* buf, char* sVal, int32_t len) { - httpJsonItemToken(buf); - httpJsonToken(buf, JsonStrStt); - httpJsonPrint(buf, sVal, len); - httpJsonToken(buf, JsonStrEnd); -} - -void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len) { - httpJsonItemToken(buf); - httpJsonPrint(buf, sVal, len); -} - -void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int32_t maxLen) { - httpJsonItemToken(buf); - httpJsonToken(buf, JsonStrStt); - - if (sVal != NULL) { - // dispose transferred meaning byte - char* lastPos = sVal; - char* curPos = sVal; - - for (int32_t i = 0; i < maxLen; ++i) { - if (*curPos == 0) { - break; - } - - if (*curPos == '\"') { - httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); - curPos++; - lastPos = curPos; - httpJsonPrint(buf, "\\\"", 2); - } else if (*curPos == '\\') { - httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); - curPos++; - lastPos = curPos; - httpJsonPrint(buf, "\\\\", 2); - } else { - curPos++; - } - } - - if (*lastPos) { - httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos)); - } - } - - httpJsonToken(buf, JsonStrEnd); -} - -void httpJsonInt64(JsonBuf* buf, int64_t num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRId64, num); -} - -void httpJsonUInt64(JsonBuf* buf, uint64_t num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRIu64, num); -} - -void httpJsonTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) { - char ts[35] = {0}; - - int32_t fractionLen; - char* format = NULL; - time_t quot = 0; - int64_t mod = 0; - - switch (timePrecision) { - case TSDB_TIME_PRECISION_MILLI: { - mod = ((t) % 1000 + 1000) % 1000; - if (t < 0 && mod != 0) { - t -= 1000; - } - quot = t / 1000; - fractionLen = 5; - format = ".%03" PRId64; - break; - } - - case TSDB_TIME_PRECISION_MICRO: { - mod = ((t) % 1000000 + 1000000) % 1000000; - if (t < 0 && mod != 0) { - t -= 1000000; - } - quot = t / 1000000; - fractionLen = 8; - format = ".%06" PRId64; - break; - } - - case TSDB_TIME_PRECISION_NANO: { - mod = ((t) % 1000000000 + 1000000000) % 1000000000; - if (t < 0 && mod != 0) { - t -= 1000000000; - } - quot = t / 1000000000; - fractionLen = 11; - format = ".%09" PRId64; - break; - } - - default: - fractionLen = 0; - assert(false); - } - - struct tm ptm = {0}; - localtime_r(", &ptm); - int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", &ptm); - length += snprintf(ts + length, fractionLen, format, mod); - - httpJsonString(buf, ts, length); -} - -void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, int32_t timePrecision) { - char ts[40] = {0}; - struct tm* ptm; - - int32_t fractionLen; - char* format = NULL; - time_t quot = 0; - long mod = 0; - - switch (timePrecision) { - case TSDB_TIME_PRECISION_MILLI: { - mod = ((t) % 1000 + 1000) % 1000; - if (t < 0 && mod != 0) { - t -= 1000; - } - quot = t / 1000; - fractionLen = 5; - format = ".%03" PRId64; - break; - } - - case TSDB_TIME_PRECISION_MICRO: { - mod = ((t) % 1000000 + 1000000) % 1000000; - if (t < 0 && mod != 0) { - t -= 1000000; - } - quot = t / 1000000; - fractionLen = 8; - format = ".%06" PRId64; - break; - } - - case TSDB_TIME_PRECISION_NANO: { - mod = ((t) % 1000000000 + 1000000000) % 1000000000; - if (t < 0 && mod != 0) { - t -= 1000000000; - } - quot = t / 1000000000; - fractionLen = 11; - format = ".%09" PRId64; - break; - } - - default: - fractionLen = 0; - assert(false); - } - - ptm = localtime("); - int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); - length += snprintf(ts + length, fractionLen, format, mod); - length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); - - httpJsonString(buf, ts, length); -} - -void httpJsonInt(JsonBuf* buf, int32_t num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num); -} - -void httpJsonUInt(JsonBuf* buf, uint32_t num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%u", num); -} - -void httpJsonFloat(JsonBuf* buf, float num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - if (isinf(num) || isnan(num)) { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null"); - } else if (num > 1E10 || num < -1E10) { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5e", num); - } else { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.5f", num); - } -} - -void httpJsonDouble(JsonBuf* buf, double num) { - httpJsonItemToken(buf); - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); - if (isinf(num) || isnan(num)) { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "null"); - } else if (num > 1E10 || num < -1E10) { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9e", num); - } else { - buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%.9f", num); - } -} - -void httpJsonNull(JsonBuf* buf) { httpJsonString(buf, "null", 4); } - -void httpJsonBool(JsonBuf* buf, int32_t val) { - if (val == 0) - httpJsonPrint(buf, JsonFalseTkn, sizeof(JsonFalseTkn)); - else - httpJsonPrint(buf, JsonTrueTkn, sizeof(JsonTrueTkn)); -} - -void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len) { - httpJsonItemToken(buf); - httpJsonString(buf, name, len); - httpJsonToken(buf, JsonPairTkn); -} - -void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) { - httpJsonPairHead(buf, name, nameLen); - httpJsonString(buf, sVal, valLen); -} - -void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) { - httpJsonPairHead(buf, name, nameLen); - httpJsonOriginString(buf, sVal, valLen); -} - -void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonInt(buf, num); -} - -void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonInt64(buf, num); -} - -void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonBool(buf, num); -} - -void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonFloat(buf, num); -} - -void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonDouble(buf, num); -} - -void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen) { - httpJsonPairHead(buf, name, nNameLen); - httpJsonNull(buf); -} - -void httpJsonPairArray(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) { - httpJsonPairHead(buf, name, len); - httpJsonArray(buf, fnBuilder, dsHandle); -} - -void httpJsonPairObject(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) { - httpJsonPairHead(buf, name, len); - httpJsonObject(buf, fnBuilder, dsHandle); -} - -void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle) { - httpJsonItemToken(buf); - httpJsonToken(buf, JsonObjStt); - (*fnBuilder)(buf, dsHandle); - httpJsonToken(buf, JsonObjEnd); -} - -void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuilder, void* jsonHandle) { - httpJsonItemToken(buf); - httpJsonToken(buf, JsonArrStt); - (*fnBuilder)(buf, jsonHandle); - httpJsonToken(buf, JsonArrEnd); -} - -void httpJsonTestBuf(JsonBuf* buf, int32_t safety) { - if ((buf->lst - buf->buf + safety) < buf->size) return; - // buf->slot = *buf->lst; - httpWriteJsonBufBody(buf, false); -} - -void httpJsonToken(JsonBuf* buf, char c) { - httpJsonTestBuf(buf, MAX_NUM_STR_SZ); // maybe object stack - *buf->lst++ = c; -} - -void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len) { - if (len == 0 || len >= JSON_BUFFER_SIZE) { - return; - } - - if (len > buf->size) { - httpWriteJsonBufBody(buf, false); - httpJsonPrint(buf, json, len); - // buf->slot = json[len - 1]; - return; - } - httpJsonTestBuf(buf, len + 2); - memcpy(buf->lst, json, (size_t)len); - buf->lst += len; -} - -void httpJsonPairStatus(JsonBuf* buf, int32_t code) { - if (code == 0) { - httpJsonPair(buf, "status", 6, "succ", 4); - } else { - httpJsonPair(buf, "status", 6, "error", 5); - httpJsonItemToken(buf); - httpJsonPairIntVal(buf, "code", 4, code & 0XFFFF); - httpJsonItemToken(buf); - if (code == TSDB_CODE_MND_DB_NOT_SELECTED) { - httpJsonPair(buf, "desc", 4, "failed to create database", 23); - } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - httpJsonPair(buf, "desc", 4, "failed to create table", 22); - } else { - httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int32_t)strlen(tstrerror(code))); - } - } -} diff --git a/src/plugins/http/src/httpMetricsHandle.c b/src/plugins/http/src/httpMetricsHandle.c deleted file mode 100644 index dbabd48774aa077b1b4f842b4bc47dfd5de67ce4..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpMetricsHandle.c +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tfs.h" - -#include "httpMetricsHandle.h" -#include "dnode.h" -#include "httpLog.h" - -static HttpDecodeMethod metricsDecodeMethod = {"metrics", metricsProcessRequest}; - -void metricsInitHandle(HttpServer* pServer) { - httpAddMethod(pServer, &metricsDecodeMethod); -} - -bool metricsProcessRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, user:%s, process admin grant msg", pContext, pContext->fd, pContext->user); - - JsonBuf* jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) { - httpError("failed to allocate memory for metrics"); - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - return false; - } - - httpInitJsonBuf(jsonBuf, pContext); - httpWriteJsonBufHead(jsonBuf); - - httpJsonToken(jsonBuf, JsonObjStt); - { - char* keyDisks = "tags"; - httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks)); - httpJsonToken(jsonBuf, JsonArrStt); - { - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonObjStt); - char* keyTagName = "name"; - char* keyTagValue = "value"; - httpJsonPairOriginString(jsonBuf, keyTagName, (int32_t)strlen(keyTagName), "\"dnode_id\"", - (int32_t)strlen("\"dnode_id\"")); - int32_t dnodeId = dnodeGetDnodeId(); - httpJsonPairIntVal(jsonBuf, keyTagValue, (int32_t)strlen(keyTagValue), dnodeId); - httpJsonToken(jsonBuf, JsonObjEnd); - } - httpJsonToken(jsonBuf, JsonArrEnd); - } - - { - if (tsDnodeStartTime != 0) { - int64_t now = taosGetTimestampMs(); - int64_t upTime = now-tsDnodeStartTime; - char* keyUpTime = "up_time"; - httpJsonPairInt64Val(jsonBuf, keyUpTime, (int32_t)strlen(keyUpTime), upTime); - } - } - - { - int32_t cpuCores = taosGetCpuCores(); - char* keyCpuCores = "cpu_cores"; - httpJsonPairIntVal(jsonBuf, keyCpuCores, (int32_t)strlen(keyCpuCores), cpuCores); - - float sysCpuUsage = 0; - float procCpuUsage = 0; - bool succeeded = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); - if (!succeeded) { - httpError("failed to get cpu usage"); - } else { - if (sysCpuUsage <= procCpuUsage) { - sysCpuUsage = procCpuUsage + 0.1f; - } - char* keyCpuSystem = "cpu_system"; - char* keyCpuEngine = "cpu_engine"; - httpJsonPairFloatVal(jsonBuf, keyCpuSystem, (int32_t)strlen(keyCpuSystem), sysCpuUsage); - httpJsonPairFloatVal(jsonBuf, keyCpuEngine, (int32_t)strlen(keyCpuEngine), procCpuUsage); - } - } - - { - float sysMemoryUsedMB = 0; - bool succeeded = taosGetSysMemory(&sysMemoryUsedMB); - if (!succeeded) { - httpError("failed to get sys memory info"); - } else { - char* keyMemSystem = "mem_system"; - httpJsonPairFloatVal(jsonBuf, keyMemSystem, (int32_t)strlen(keyMemSystem), sysMemoryUsedMB); - } - - float procMemoryUsedMB = 0; - succeeded = taosGetProcMemory(&procMemoryUsedMB); - if (!succeeded) { - httpError("failed to get proc memory info"); - } else { - char* keyMemEngine = "mem_engine"; - httpJsonPairFloatVal(jsonBuf, keyMemEngine, (int32_t)strlen(keyMemEngine), procMemoryUsedMB); - } - } - - { - int64_t bytes = 0, rbytes = 0, tbytes = 0; - bool succeeded = taosGetCardInfo(&bytes, &rbytes, &tbytes); - if (!succeeded) { - httpError("failed to get network info"); - } else { - char* keyNetIn = "net_in"; - char* keyNetOut = "net_out"; - httpJsonPairInt64Val(jsonBuf, keyNetIn, (int32_t)strlen(keyNetIn), rbytes); - httpJsonPairInt64Val(jsonBuf, keyNetOut, (int32_t)strlen(keyNetOut), tbytes); - } - } - - { - int64_t rchars = 0; - int64_t wchars = 0; - bool succeeded = taosReadProcIO(&rchars, &wchars); - if (!succeeded) { - httpError("failed to get io info"); - } else { - char* keyIORead = "io_read"; - char* keyIOWrite = "io_write"; - httpJsonPairInt64Val(jsonBuf, keyIORead, (int32_t)strlen(keyIORead), rchars); - httpJsonPairInt64Val(jsonBuf, keyIOWrite, (int32_t)strlen(keyIOWrite), wchars); - } - } - - { - const int8_t numTiers = 3; - SFSMeta fsMeta; - STierMeta* tierMetas = calloc(numTiers, sizeof(STierMeta)); - tfsUpdateInfo(&fsMeta, tierMetas, numTiers); - { - char* keyDiskUsed = "disk_used"; - char* keyDiskTotal = "disk_total"; - httpJsonPairInt64Val(jsonBuf, keyDiskTotal, (int32_t)strlen(keyDiskTotal), fsMeta.tsize); - httpJsonPairInt64Val(jsonBuf, keyDiskUsed, (int32_t)strlen(keyDiskUsed), fsMeta.used); - char* keyDisks = "disks"; - httpJsonPairHead(jsonBuf, keyDisks, (int32_t)strlen(keyDisks)); - httpJsonToken(jsonBuf, JsonArrStt); - for (int i = 0; i < numTiers; ++i) { - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonObjStt); - char* keyDataDirLevelUsed = "datadir_used"; - char* keyDataDirLevelTotal = "datadir_total"; - httpJsonPairInt64Val(jsonBuf, keyDataDirLevelUsed, (int32_t)strlen(keyDataDirLevelUsed), tierMetas[i].used); - httpJsonPairInt64Val(jsonBuf, keyDataDirLevelTotal, (int32_t)strlen(keyDataDirLevelTotal), tierMetas[i].size); - httpJsonToken(jsonBuf, JsonObjEnd); - } - httpJsonToken(jsonBuf, JsonArrEnd); - } - free(tierMetas); - } - - { - SStatisInfo info = dnodeGetStatisInfo(); - { - char* keyReqHttp = "req_http"; - char* keyReqSelect = "req_select"; - char* keyReqInsert = "req_insert"; - httpJsonPairInt64Val(jsonBuf, keyReqHttp, (int32_t)strlen(keyReqHttp), info.httpReqNum); - httpJsonPairInt64Val(jsonBuf, keyReqSelect, (int32_t)strlen(keyReqSelect), info.queryReqNum); - httpJsonPairInt64Val(jsonBuf, keyReqInsert, (int32_t)strlen(keyReqInsert), info.submitReqNum); - } - } - - httpJsonToken(jsonBuf, JsonObjEnd); - - httpWriteJsonBufEnd(jsonBuf); - pContext->reqType = HTTP_REQTYPE_OTHERS; - httpFreeJsonBuf(pContext); - return false; -} \ No newline at end of file diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c deleted file mode 100644 index 7066f19769754e78dffeed6a40b672584c0310f1..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpParser.c +++ /dev/null @@ -1,1182 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "httpLog.h" -#include "httpContext.h" -#include "httpParser.h" -#include "httpGzip.h" -#include "httpAuth.h" - -static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len); - -static HttpStatus httpStatusCodes[] = { - {HTTP_CODE_CONTINUE, "Continue"}, - {HTTP_CODE_SWITCHING_PROTOCOL, "Switching Protocol"}, - {HTTP_CODE_PROCESSING, "Processing (WebDAV)"}, - {HTTP_CODE_EARLY_HINTS, "Early Hints"}, - {HTTP_CODE_OK, "OK"}, - {HTTP_CODE_CREATED, "Created"}, - {HTTP_CODE_ACCEPTED, "Accepted"}, - {HTTP_CODE_NON_AUTHORITATIVE_INFO, "Non-Authoritative Information"}, - {HTTP_CODE_NO_CONTENT, "No Content"}, - {HTTP_CODE_RESET_CONTENT, "Reset Content"}, - {HTTP_CODE_PARTIAL_CONTENT, "Partial Content"}, - {HTTP_CODE_MULTI_STATUS, "Multi-Status (WebDAV)"}, - {HTTP_CODE_ALREADY_REPORTED, "Already Reported (WebDAV)"}, - {HTTP_CODE_IM_USED, "IM Used (HTTP Delta encoding)"}, - {HTTP_CODE_MULTIPLE_CHOICE, "Multiple Choice"}, - {HTTP_CODE_MOVED_PERMANENTLY, "Moved Permanently"}, - {HTTP_CODE_FOUND, "Found"}, - {HTTP_CODE_SEE_OTHER, "See Other"}, - {HTTP_CODE_NOT_MODIFIED, "Not Modified"}, - {HTTP_CODE_USE_PROXY, "Use Proxy"}, - {HTTP_CODE_UNUSED, "unused"}, - {HTTP_CODE_TEMPORARY_REDIRECT, "Temporary Redirect"}, - {HTTP_CODE_PERMANENT_REDIRECT, "Permanent Redirect"}, - {HTTP_CODE_BAD_REQUEST, "Bad Request"}, - {HTTP_CODE_UNAUTHORIZED, "Unauthorized"}, - {HTTP_CODE_PAYMENT_REQUIRED, "Payment Required"}, - {HTTP_CODE_FORBIDDEN, "Forbidden"}, - {HTTP_CODE_NOT_FOUND, "Not Found"}, - {HTTP_CODE_METHOD_NOT_ALLOWED, "Method Not Allowed"}, - {HTTP_CODE_NOT_ACCEPTABLE, "Not Acceptable"}, - {HTTP_CODE_PROXY_AUTH_REQUIRED, "Proxy Authentication Required"}, - {HTTP_CODE_REQUEST_TIMEOUT, "Request Timeout"}, - {HTTP_CODE_CONFLICT, "Conflict"}, - {HTTP_CODE_GONE, "Gone"}, - {HTTP_CODE_LENGTH_REQUIRED, "Length Required"}, - {HTTP_CODE_PRECONDITION_FAILED, "Precondition Failed"}, - {HTTP_CODE_PAYLOAD_TOO_LARGE, "Payload Too Large"}, - {HTTP_CODE_URI_TOO_LARGE, "URI Too Long"}, - {HTTP_CODE_UNSUPPORTED_MEDIA_TYPE, "Unsupported Media Type"}, - {HTTP_CODE_RANGE_NOT_SATISFIABLE, "Range Not Satisfiable"}, - {HTTP_CODE_EXPECTATION_FAILED, "Expectation Failed"}, - {HTTP_CODE_IM_A_TEAPOT, "I'm a teapot"}, - {HTTP_CODE_MISDIRECTED_REQUEST, "Misdirected Request"}, - {HTTP_CODE_UNPROCESSABLE_ENTITY, "Unprocessable Entity (WebDAV)"}, - {HTTP_CODE_LOCKED, "Locked (WebDAV)"}, - {HTTP_CODE_FAILED_DEPENDENCY, "Failed Dependency (WebDAV)"}, - {HTTP_CODE_TOO_EARLY, "Too Early"}, - {HTTP_CODE_UPGRADE_REQUIRED, "Upgrade Required"}, - {HTTP_CODE_PRECONDITION_REQUIRED, "Precondition Required"}, - {HTTP_CODE_TOO_MANY_REQUESTS, "Too Many Requests"}, - {HTTP_CODE_REQ_HDR_FIELDS_TOO_LARGE,"Request Header Fields Too Large"}, - {HTTP_CODE_UNAVAIL_4_LEGAL_REASONS, "Unavailable For Legal Reasons"}, - {HTTP_CODE_INTERNAL_SERVER_ERROR, "Internal Server Error"}, - {HTTP_CODE_NOT_IMPLEMENTED, "Not Implemented"}, - {HTTP_CODE_BAD_GATEWAY, "Bad Gateway"}, - {HTTP_CODE_SERVICE_UNAVAILABLE, "Service Unavailable"}, - {HTTP_CODE_GATEWAY_TIMEOUT, "Gateway Timeout"}, - {HTTP_CODE_HTTP_VER_NOT_SUPPORTED, "HTTP Version Not Supported"}, - {HTTP_CODE_VARIANT_ALSO_NEGOTIATES, "Variant Also Negotiates"}, - {HTTP_CODE_INSUFFICIENT_STORAGE, "Insufficient Storage"}, - {HTTP_CODE_LOOP_DETECTED, "Loop Detected (WebDAV)"}, - {HTTP_CODE_NOT_EXTENDED, "Not Extended"}, - {HTTP_CODE_NETWORK_AUTH_REQUIRED, "Network Authentication Required"}, - {0, NULL} -}; - -char *httpGetStatusDesc(int32_t statusCode) { - HttpStatus *p = httpStatusCodes; - while (p->code != 0) { - if (p->code == statusCode) return p->desc; - ++p; - } - return "Unknow status code"; -} - -static void httpCleanupString(HttpString *str) { - if (str->str) { - free(str->str); - str->str = NULL; - str->pos = 0; - str->size = 0; - } -} - -static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { - char *new_str = NULL; - - if (str->size == 0) { - str->pos = 0; - str->size = len + 1; - str->str = malloc(str->size); - } else if (str->pos + len + 1 >= str->size) { - str->size += len; - str->size *= 4; - - new_str = realloc(str->str, str->size); - if (new_str == NULL && str->str) { - // if str->str was not NULL originally, - // the old allocated memory was left unchanged, - // see man 3 realloc - free(str->str); - } - - str->str = new_str; - } else { - } - - if (str->str == NULL) return -1; - - memcpy(str->str + str->pos, s, len); - str->pos += len; - str->str[str->pos] = 0; - return 0; -} - -static void httpClearString(HttpString *str) { - if (str->str) { - str->str[0] = '\0'; - str->pos = 0; - } -} - -static int32_t httpOnError(HttpParser *parser, int32_t httpCode, int32_t parseCode) { - HttpContext *pContext = parser->pContext; - if (httpCode != 0) parser->httpCode = httpCode; - if (parseCode != 0) parser->parseCode = parseCode; - - httpError("context:%p, fd:%d, parse failed, httpCode:%d parseCode:%d reason:%s", pContext, pContext->fd, httpCode, - parseCode & 0XFFFF, tstrerror(parseCode)); - return 0; -} - -static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target, char *version) { - HttpContext *pContext = pParser->pContext; - httpDebug("context:%p, fd:%d, method:%s target:%s version:%s", pContext, pContext->fd, method, target, version); - - // parse url - char *pStart = target + 1; - for (int32_t i = 0; i < HTTP_MAX_URL; i++) { - char *pSeek = strchr(pStart, '/'); - if (pSeek == NULL) { - (void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart)); - break; - } else { - (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); - } - pStart = pSeek + 1; - } - - // parse decode method - for (int32_t i = 0; i < tsHttpServer.methodScannerLen; i++) { - HttpDecodeMethod *_method = tsHttpServer.methodScanner[i]; - if (strcmp(_method->module, pParser->path[0].str) == 0) { - pContext->decodeMethod = _method; - break; - } - } - - if (pContext->decodeMethod != NULL) { - httpTrace("context:%p, fd:%d, decode method is %s", pContext, pContext->fd, pContext->decodeMethod->module); - } else { - httpError("context:%p, fd:%d, the url is not support, target:%s", pContext, pContext->fd, target); - httpOnError(pParser, 0, TSDB_CODE_HTTP_UNSUPPORT_URL); - return -1; - } - - // parse version - if (pParser->httpVersion < HTTP_VERSION_10 || pParser->httpVersion > HTTP_VERSION_12) { - httpError("context:%p, fd:%d, unsupport httpVersion %d", pContext, pContext->fd, pParser->httpVersion); - httpOnError(pParser, 0, TSDB_CODE_HTTP_INVALID_VERSION); - } else { - httpTrace("context:%p, fd:%d, httpVersion:1.%d", pContext, pContext->fd, pParser->httpVersion); - } - - return 0; -} - -static int32_t httpOnStatusLine(HttpParser *pParser, int32_t code, const char *reason) { - HttpContext *pContext = pParser->pContext; - httpError("context:%p, fd:%d, status line, code:%d reason:%s", pContext, pContext->fd, code, reason); - return 0; -} - -static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const char *val) { - HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, key:%s val:%s", pContext, pContext->fd, key, val); - - if (0 == strcasecmp(key, "Content-Length")) { - int32_t len = 0; - int32_t bytes = 0; - int32_t n = sscanf(val, "%d%n", &len, &bytes); - if (n == 1 && bytes == strlen(val)) { - parser->contentLength = len; - parser->chunkSize = len; - parser->contentLengthSpecified = 1; - httpTrace("context:%p, fd:%d, contentLength:%d chunkSize:%d contentLengthSpecified:%d", pContext, pContext->fd, - parser->contentLength, parser->chunkSize, parser->contentLengthSpecified); - return 0; - } else { - httpError("context:%p, fd:%d, failed to parser %s:%s", pContext, pContext->fd, key, val); - httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH); - return -1; - } - } - - else if (0 == strcasecmp(key, "Accept-Encoding")) { - if (strstr(val, "gzip")) { - parser->acceptEncodingGzip = 1; - httpTrace("context:%p, fd:%d, acceptEncodingGzip:%d", pContext, pContext->fd, parser->acceptEncodingGzip); - } - if (strstr(val, "chunked")) { - parser->acceptEncodingChunked = 1; - httpTrace("context:%p, fd:%d, acceptEncodingChunked:%d", pContext, pContext->fd, parser->acceptEncodingChunked); - } - return 0; - } - - else if (strncasecmp(key, "Connection", 10) == 0) { - if (strncasecmp(val, "Keep-Alive", 10) == 0) { - parser->keepAlive = HTTP_KEEPALIVE_ENABLE; - } else { - parser->keepAlive = HTTP_KEEPALIVE_DISABLE; - } - httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); - } -#if 0 - else if (0 == strcasecmp(key, "Content-Encoding")) { - if (0 == strcmp(val, "gzip")) { - parser->contentChunked = 1; - httpTrace("context:%p, fd:%d, contentChunked:%d", pContext, pContext->fd, parser->contentChunked); - } - return 0; - } -#endif - - else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { - if (strstr(val, "gzip")) { - parser->transferGzip = 1; - ehttp_gzip_conf_t conf = {0}; - ehttp_gzip_callbacks_t callbacks = {0}; - - callbacks.on_data = httpOnData; - - parser->gzip = ehttp_gzip_create_decompressor(conf, callbacks, parser); - - if (!parser->gzip) { - httpError("context:%p, fd:%d, failed to create gzip decompressor", pContext, pContext->fd); - httpOnError(parser, 0, TSDB_CODE_HTTP_CREATE_GZIP_FAILED); - return -1; - } - } - if (strstr(val, "chunked")) { - parser->transferChunked = 1; - httpTrace("context:%p, fd:%d, transferChunked:%d", pContext, pContext->fd, parser->transferChunked); - } - return 0; - } - - else if (0 == strcasecmp(key, "Authorization")) { - char t[6] = {0}; - char s[129] = {0}; - int32_t bytes = 0; - int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes); - if (n == 2 && t[0] && s[0] && bytes == strlen(val)) { - if (strcmp(t, "Basic") == 0) { - free(parser->authContent); - parser->authContent = strdup(s); - parser->authType = HTTP_BASIC_AUTH; - httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); - if (ok != 0) { - httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); - return -1; - } - return 0; - } else if (strcmp(t, "Taosd") == 0) { - free(parser->authContent); - parser->authContent = strdup(s); - parser->authType = HTTP_TAOSD_AUTH; - httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); - if (ok != 0) { - httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); - return -1; - } - return 0; - } else { - parser->authType = HTTP_INVALID_AUTH; - httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); - httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); - return -1; - } - } else { - parser->authType = HTTP_INVALID_AUTH; - httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); - httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); - return -1; - } - } - - return 0; -} - -static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { - HttpContext *pContext = parser->pContext; - HttpString *buf = &parser->body; - if (parser->parseCode != TSDB_CODE_SUCCESS) return -1; - - if (buf->size <= 0) { - buf->size = MIN(len + 2, HTTP_BUFFER_SIZE); - buf->str = malloc(buf->size); - } - - int32_t newSize = buf->pos + len + 1; - char *newStr = NULL; - if (newSize >= buf->size) { - if (buf->size >= HTTP_BUFFER_SIZE) { - httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size); - httpOnError(parser, 0, TSDB_CODE_HTTP_REQUSET_TOO_BIG); - return -1; - } - - newSize = MAX(newSize, HTTP_BUFFER_INIT); - newSize *= 4; - newSize = MIN(newSize, HTTP_BUFFER_SIZE); - newStr = realloc(buf->str, newSize); - if (newStr == NULL && buf->str) { - free(buf->str); - } - - buf->str = newStr; - buf->size = newSize; - - if (buf->str == NULL) { - httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); - httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - return -1; - } - } - - memcpy(buf->str + buf->pos, chunk, len); - buf->pos += len; - buf->str[buf->pos] = 0; - - return 0; -} - -static int32_t httpOnEnd(HttpParser *parser) { - HttpContext *pContext = parser->pContext; - parser->parsed = true; - - if (parser->parseCode != TSDB_CODE_SUCCESS) { - return -1; - } - - httpTrace("context:%p, fd:%d, parse success", pContext, pContext->fd); - return 0; -} - -static HTTP_PARSER_STATE httpTopStack(HttpParser *parser) { - HttpStack *stack = &parser->stacks; - ASSERT(stack->pos >= 1); - - return stack->stacks[stack->pos - 1]; -} - -static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) { - HttpStack *stack = &parser->stacks; - int8_t *newStacks = NULL; - if (stack->size == 0) { - stack->pos = 0; - stack->size = 32; - stack->stacks = malloc(stack->size * sizeof(int8_t)); - } else if (stack->pos + 1 > stack->size) { - stack->size *= 2; - - newStacks = realloc(stack->stacks, stack->size * sizeof(int8_t)); - if (newStacks == NULL && stack->stacks) { - free(stack->stacks); - } - - stack->stacks = newStacks; - } else { - } - - if (stack->stacks == NULL) return -1; - - stack->stacks[stack->pos] = state; - stack->pos++; - - return 0; -} - -static int32_t httpPopStack(HttpParser *parser) { - HttpStack *stack = &parser->stacks; - ASSERT(stack->pos >= 1); - stack->pos--; - return 0; -} - -static void httpClearStack(HttpStack *stack) { stack->pos = 0; } - -static int32_t httpCleanupStack(HttpStack *stack) { - free(stack->stacks); - memset(stack, 0, sizeof(HttpStack)); - - return 0; -} - -void httpInitParser(HttpParser *parser) { - HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, init parser", pContext, pContext->fd); - - parser->parsed = false; - parser->inited = 1; - parser->httpVersion = 0; - parser->acceptEncodingGzip = 0; - parser->acceptEncodingChunked = 0; - parser->contentLengthSpecified = 0; - parser->contentChunked = 0; - parser->transferGzip = 0; - parser->transferChunked = 0; - parser->keepAlive = 0; - parser->authType = 0; - parser->contentLength = 0; - parser->chunkSize = 0; - parser->receivedChunkSize = 0; - parser->receivedSize = 0; - parser->statusCode = 0; - parser->httpCode = 0; - parser->parseCode = 0; - - free(parser->method); parser->method = NULL; - free(parser->target); parser->target = NULL; - free(parser->version); parser->version = NULL; - free(parser->reasonPhrase); parser->reasonPhrase = NULL; - free(parser->key); parser->key = NULL; - free(parser->val); parser->val = NULL; - free(parser->authContent); parser->authContent = NULL; - - httpClearStack(&parser->stacks); - httpClearString(&parser->str); - httpClearString(&parser->body); - for (int32_t i = 0; i < HTTP_MAX_URL; ++i) { - httpClearString(&parser->path[i]); - } - - if (parser->gzip != NULL) { - ehttp_gzip_destroy(parser->gzip); - parser->gzip = NULL; - } - - httpPushStack(parser, HTTP_PARSER_BEGIN); -} - -HttpParser *httpCreateParser(HttpContext *pContext) { - HttpParser *parser = calloc(1, sizeof(HttpParser)); - if (!parser) return NULL; - httpTrace("context:%p, fd:%d, create parser", pContext, pContext->fd); - - parser->pContext = pContext; - return parser; -} - -void httpClearParser(HttpParser *parser) { - HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, clear parser", pContext, pContext->fd); - - pContext->parser->inited = 0; - pContext->parser->parsed = false; -} - -void httpDestroyParser(HttpParser *parser) { - if (!parser) return; - - HttpContext *pContext = parser->pContext; - httpTrace("context:%p, fd:%d, destroy parser", pContext, pContext->fd); - - free(parser->method); parser->method = NULL; - free(parser->target); parser->target = NULL; - free(parser->version); parser->version = NULL; - free(parser->reasonPhrase); parser->reasonPhrase = NULL; - free(parser->key); parser->key = NULL; - free(parser->val); parser->val = NULL; - free(parser->authContent); parser->authContent = NULL; - - httpCleanupStack(&parser->stacks); - httpCleanupString(&parser->str); - httpCleanupString(&parser->body); - for (int32_t i = 0; i < HTTP_MAX_URL; ++i) { - httpCleanupString(&parser->path[i]); - } - - if (parser->gzip != NULL) { - ehttp_gzip_destroy(parser->gzip); - parser->gzip = NULL; - } - - free(parser); -} - -#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) - -char *httpDecodeUrl(const char *enc) { - int32_t ok = 1; - HttpString str = {0}; - while (*enc) { - char *p = strchr(enc, '%'); - if (!p) break; - int32_t hex, cnt; - int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt); - if (n != 1 && cnt != 2) { - ok = 0; - break; - } - if (httpAppendString(&str, enc, (int32_t)(p - enc))) { - ok = 0; - break; - } - char c = (char)hex; - if (httpAppendString(&str, &c, 1)) { - ok = 0; - break; - } - enc = p + 3; - } - char *dec = NULL; - if (ok && *enc) { - if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { - ok = 0; - } - } - if (ok) { - dec = str.str; - str.str = NULL; - } - //httpCleanupString(&str); - return dec; -} - -static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { - HttpParser *parser = (HttpParser *)arg; - httpOnBody(parser, buf, len); -} - -static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); - break; - } - httpPopStack(parser); - httpPushStack(parser, HTTP_PARSER_REQUEST_OR_RESPONSE); - break; - } - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (parser->str.pos == 1) { - if (c == 'T' && parser->str.str[0] == 'H') { - httpPopStack(parser); - httpPushStack(parser, HTTP_PARSER_END); - httpPushStack(parser, HTTP_PARSER_HEADER); - httpPushStack(parser, HTTP_PARSER_CRLF); - httpPushStack(parser, HTTP_PARSER_REASON_PHRASE); - httpPushStack(parser, HTTP_PARSER_SP); - httpPushStack(parser, HTTP_PARSER_STATUS_CODE); - httpPushStack(parser, HTTP_PARSER_SP); - httpPushStack(parser, HTTP_PARSER_HTTP_VERSION); - *again = 1; - break; - } - httpPopStack(parser); - httpPushStack(parser, HTTP_PARSER_END); - httpPushStack(parser, HTTP_PARSER_HEADER); - httpPushStack(parser, HTTP_PARSER_CRLF); - httpPushStack(parser, HTTP_PARSER_HTTP_VERSION); - httpPushStack(parser, HTTP_PARSER_SP); - httpPushStack(parser, HTTP_PARSER_TARGET); - httpPushStack(parser, HTTP_PARSER_SP); - httpPushStack(parser, HTTP_PARSER_METHOD); - *again = 1; - break; - } - - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); - break; - } - break; - } - parser->method = strdup(parser->str.str); - if (!parser->method) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_METHOD_FAILED); - break; - } else { - httpTrace("context:%p, fd:%d, httpMethod:%s", pContext, pContext->fd, parser->method); - } - httpClearString(&parser->str); - httpPopStack(parser); - *again = 1; - } while (0); - return ok; -} - -static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (!isspace(c)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_TARGET_FAILED); - break; - } - break; - } - parser->target = httpDecodeUrl(parser->str.str); - if (!parser->target) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_TARGET_FAILED); - break; - } - httpClearString(&parser->str); - httpPopStack(parser); - *again = 1; - } while (0); - return ok; -} - -static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - const char *prefix = "HTTP/1."; - int32_t len = (int32_t)strlen(prefix); - if (parser->str.pos < len) { - if (prefix[parser->str.pos] != c) { - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); - break; - } - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); - break; - } - break; - } - - if (c != '0' && c != '1' && c != '2') { - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); - break; - } - - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); - break; - } - - if (c == '0') - parser->httpVersion = HTTP_VERSION_10; - else if (c == '1') - parser->httpVersion = HTTP_VERSION_11; - else if (c == '2') - parser->httpVersion = HTTP_VERSION_12; - else { - } - - parser->version = strdup(parser->str.str); - if (!parser->version) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_VERSION_FAILED); - break; - } - - if (parser->method) { - ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); - /* - if (parser->target) { - free(parser->target); - parser->target = NULL; - } - */ - } - - httpClearString(&parser->str); - httpPopStack(parser); - } while (0); - return ok; -} - -static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (c == ' ') { - httpPopStack(parser); - break; - } - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_SP_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnOptionalSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - int32_t ok = 0; - if (c != ' ') { - *again = 1; - httpPopStack(parser); - } - return ok; -} - -static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (isdigit(c)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_STATUS_FAILED); - break; - } - if (parser->str.pos < 3) break; - - sscanf(parser->str.str, "%d", &parser->statusCode); - httpClearString(&parser->str); - httpPopStack(parser); - break; - } - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_STATUS_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (c == '\r') { - parser->reasonPhrase = strdup(parser->str.str); - if (!parser->reasonPhrase) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_PHRASE_FAILED); - break; - } - ok = httpOnStatusLine(parser, parser->statusCode, parser->reasonPhrase); - httpClearString(&parser->str); - httpPopStack(parser); - *again = 1; - break; - } - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_PHRASE_FAILED); - break; - } - } while (0); - return ok; -} - -static int32_t httpParserPostProcess(HttpParser *parser) { - HttpContext *pContext = parser->pContext; - if (parser->gzip) { - if (ehttp_gzip_finish(parser->gzip)) { - httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_FINISH_GZIP_FAILED); - return -1; - } - } - httpOnEnd(parser); - return 0; -} - -static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - const char *s = "\r\n"; - int32_t len = (int32_t)strlen(s); - if (s[parser->str.pos] != c) { - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_CRLF_FAILED); - break; - } - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_CRLF_FAILED); - break; - } - if (parser->str.pos == len) { - httpClearString(&parser->str); - httpPopStack(parser); - if (httpTopStack(parser) == HTTP_PARSER_END) { - ok = httpParserPostProcess(parser); - } - } - break; - } while (0); - return ok; -} - -static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (c == '\r') { - httpPopStack(parser); - if (parser->transferChunked) { - httpPushStack(parser, HTTP_PARSER_CHUNK_SIZE); - httpPushStack(parser, HTTP_PARSER_CRLF); - } else { - if (parser->contentLength > 0) { - httpPushStack(parser, HTTP_PARSER_CHUNK); - } - httpPushStack(parser, HTTP_PARSER_CRLF); - } - *again = 1; - break; - } - if (c != ' ' && c != '\t' && c != ':') { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_HEADER_FAILED); - break; - } - httpPushStack(parser, HTTP_PARSER_CRLF); - httpPushStack(parser, HTTP_PARSER_HEADER_VAL); - httpPushStack(parser, HTTP_PARSER_OPTIONAL_SP); - httpPushStack(parser, HTTP_PARSER_HEADER_KEY); - break; - } - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_HEADER_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); - break; - } - break; - } - if (c == ':') { - parser->key = strdup(parser->str.str); - if (!parser->key) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); - break; - } - httpClearString(&parser->str); - httpPopStack(parser); - break; - } - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - parser->parseCode = TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED); - break; - } - break; - } - const char *val = parser->str.str; - ok = httpOnParseHeaderField(parser, parser->key, val); - free(parser->key); - parser->key = NULL; - val = NULL; - if (ok == -1) break; - httpClearString(&parser->str); - httpPopStack(parser); - *again = 1; - } while (0); - return ok; -} - -static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - int32_t bytes; - int32_t len; - int32_t n; - do { - if (isxdigit(c)) { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED); - break; - } - break; - } - if (c == '\r') { - n = sscanf(parser->str.str, "%x%n", &len, &bytes); - if (n == 1 && bytes == strlen(parser->str.str) && len >= 0) { - if (len == 0) { - if (parser->contentLengthSpecified == 0 || parser->receivedSize == parser->contentLength) { - httpClearString(&parser->str); - httpPopStack(parser); - httpPushStack(parser, HTTP_PARSER_CRLF); - httpPushStack(parser, HTTP_PARSER_CRLF); - *again = 1; - break; - } - } else { - if (parser->contentLengthSpecified == 0 || parser->receivedSize + len <= parser->contentLength) { - parser->chunkSize = len; - httpClearString(&parser->str); - httpPopStack(parser); - httpPushStack(parser, HTTP_PARSER_CHUNK_SIZE); - httpPushStack(parser, HTTP_PARSER_CRLF); - httpPushStack(parser, HTTP_PARSER_CHUNK); - httpPushStack(parser, HTTP_PARSER_CRLF); - *again = 1; - break; - } - } - } - } - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_BAD_REQUEST, TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED); - } while (0); - return ok; -} - -static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - if (httpAppendString(&parser->str, &c, 1)) { - httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_CHUNK_FAILED); - break; - } - ++parser->receivedSize; - ++parser->receivedChunkSize; - if (parser->receivedChunkSize < parser->chunkSize) break; - - if (parser->gzip) { - if (ehttp_gzip_write(parser->gzip, parser->str.str, parser->str.pos)) { - httpError("context:%p, fd:%d, gzip failed", pContext, pContext->fd); - ok = -1; - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_CHUNK_FAILED); - break; - } - } else { - httpOnBody(parser, parser->str.str, parser->str.pos); - } - parser->receivedChunkSize = 0; - httpClearString(&parser->str); - httpPopStack(parser); - if (httpTopStack(parser) == HTTP_PARSER_END) { - ok = httpParserPostProcess(parser); - } - } while (0); - return ok; -} - -static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; - do { - ok = -1; - httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); - httpOnError(parser, HTTP_CODE_INSUFFICIENT_STORAGE, TSDB_CODE_HTTP_PARSE_END_FAILED); - } while (0); - return ok; -} - -static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { - HttpContext * pContext = parser->pContext; - int32_t ok = 0; - HTTP_PARSER_STATE state = httpTopStack(parser); - do { - if (state == HTTP_PARSER_BEGIN) { - ok = httpParserOnBegin(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_REQUEST_OR_RESPONSE) { - ok = httpParserOnRquestOrResponse(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_METHOD) { - ok = httpParserOnMethod(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_TARGET) { - ok = httpParserOnTarget(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_HTTP_VERSION) { - ok = httpParserOnVersion(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_SP) { - ok = httpParserOnSp(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_OPTIONAL_SP) { - ok = httpParserOnOptionalSp(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_STATUS_CODE) { - ok = httpParserOnStatusCode(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_REASON_PHRASE) { - ok = httpParserOnReasonPhrase(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_CRLF) { - ok = httpParserOnCrlf(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_HEADER) { - ok = httpParserOnHeader(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_HEADER_KEY) { - ok = httpParserOnHeaderKey(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_HEADER_VAL) { - ok = httpParserOnHeaderVal(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_CHUNK_SIZE) { - ok = httpParserOnChunkSize(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_CHUNK) { - ok = httpParserOnChunk(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_END) { - ok = httpParserOnEnd(parser, state, c, again); - break; - } - if (state == HTTP_PARSER_ERROR) { - ok = -2; - break; - } - - ok = -1; - httpError("context:%p, fd:%d, unknown parse state:%d", pContext, pContext->fd, state); - httpOnError(parser, HTTP_CODE_INTERNAL_SERVER_ERROR, TSDB_CODE_HTTP_PARSE_INVALID_STATE); - } while (0); - - if (ok == -1) { - httpError("context:%p, fd:%d, failed to parse, state:%d", pContext, pContext->fd, state); - httpPushStack(parser, HTTP_PARSER_ERROR); - } - - if (ok == -2) { - ok = -1; - httpError("context:%p, fd:%d, failed to parse, invalid state", pContext, pContext->fd); - httpOnError(parser, HTTP_CODE_INTERNAL_SERVER_ERROR, TSDB_CODE_HTTP_PARSE_ERROR_STATE); - } - - return ok; -} - -int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { - HttpContext *pContext = parser->pContext; - const char * p = buf; - int32_t ret = 0; - int32_t i = 0; - - while (i < len) { - int32_t again = 0; - ret = httpParseChar(parser, *p, &again); - if (ret != 0) { - httpError("context:%p, fd:%d, parse failed, ret:%d i:%d len:%d buf:%s", pContext, pContext->fd, ret, i, len, buf); - break; - } - if (again) continue; - ++p; - ++i; - } - - return ret; -} diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c deleted file mode 100644 index 917179078d4f33662b2c150e65626c825eb2a1b0..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpQueue.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "httpQueue.h" -#include "../../../../include/client/taos.h" -#include "httpAuth.h" -#include "httpContext.h" -#include "httpInt.h" -#include "httpResp.h" -#include "httpSession.h" -#include "httpSql.h" -#include "os.h" -#include "tnote.h" -#include "tqueue.h" -#include "tsclient.h" - -typedef struct { - pthread_t thread; - int32_t workerId; -} SHttpWorker; - -typedef struct { - int32_t num; - SHttpWorker *httpWorker; -} SHttpWorkerPool; - -typedef struct { - void * param; - void * result; - int32_t code; - int32_t rows; - FHttpResultFp fp; -} SHttpResult; - -static SHttpWorkerPool tsHttpPool; -static taos_qset tsHttpQset; -static taos_queue tsHttpQueue; - -void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { - if (tsHttpQueue != NULL) { - SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult)); - pMsg->param = param; - pMsg->result = result; - pMsg->code = code; - pMsg->rows = rows; - pMsg->fp = fp; - taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); - } else { - taos_stop_query(result); - taos_free_result(result); - //(*fp)(param, result, code, rows); - } -} - -static void *httpProcessResultQueue(void *param) { - SHttpResult *pMsg; - int32_t type; - void * unUsed; - - setThreadName("httpResultQ"); - - while (1) { - if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) { - httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset); - break; - } - - httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result, - pMsg->code, pMsg->rows); - (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows); - taosFreeQitem(pMsg); - } - - return NULL; -} - -static bool httpAllocateResultQueue() { - tsHttpQueue = taosOpenQueue(); - if (tsHttpQueue == NULL) return false; - - taosAddIntoQset(tsHttpQset, tsHttpQueue, NULL); - - for (int32_t i = 0; i < tsHttpPool.num; ++i) { - SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - pWorker->workerId = i; - - pthread_attr_t thAttr; - pthread_attr_init(&thAttr); - pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&pWorker->thread, &thAttr, httpProcessResultQueue, pWorker) != 0) { - httpError("failed to create thread to process http result queue, reason:%s", strerror(errno)); - } - - pthread_attr_destroy(&thAttr); - httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); - } - - httpInfo("http result queue is opened"); - return true; -} - -static void httpFreeResultQueue() { - taosCloseQueue(tsHttpQueue); - tsHttpQueue = NULL; -} - -bool httpInitResultQueue() { - tsHttpQset = taosOpenQset(); - - tsHttpPool.num = tsHttpMaxThreads; - tsHttpPool.httpWorker = (SHttpWorker *)calloc(sizeof(SHttpWorker), tsHttpPool.num); - - if (tsHttpPool.httpWorker == NULL) return -1; - for (int32_t i = 0; i < tsHttpPool.num; ++i) { - SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - pWorker->workerId = i; - } - - return httpAllocateResultQueue(); -} - -void httpCleanupResultQueue() { - httpFreeResultQueue(); - - for (int32_t i = 0; i < tsHttpPool.num; ++i) { - SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - taosQsetThreadResume(tsHttpQset); - } - } - - for (int32_t i = 0; i < tsHttpPool.num; ++i) { - SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (taosCheckPthreadValid(pWorker->thread)) { - pthread_join(pWorker->thread, NULL); - } - } - - taosCloseQset(tsHttpQset); - free(tsHttpPool.httpWorker); - - httpInfo("http result queue is closed"); -} diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c deleted file mode 100644 index 1d05b455cb5c66e4f492140e1f337210da04caef..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpResp.c +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "httpLog.h" -#include "httpResp.h" -#include "httpJson.h" -#include "httpContext.h" - -const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"}; - -const char *httpVersionStr[] = {"HTTP/1.0", "HTTP/1.1", "HTTP/1.2"}; - -const char *httpRespTemplate[] = { - // HTTP_RESPONSE_JSON_OK - // HTTP_RESPONSE_JSON_ERROR - "{\"status\":\"succ\",\"code\":%d,\"desc\":\"%s\"}", - "{\"status\":\"error\",\"code\":%d,\"desc\":\"%s\"}", - // HTTP_RESPONSE_OK - // HTTP_RESPONSE_ERROR - "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n", - "%s %d %s\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n", - // HTTP_RESPONSE_CHUNKED_UN_COMPRESS, HTTP_RESPONSE_CHUNKED_COMPRESS - "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nTransfer-Encoding: chunked\r\n\r\n", - "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Encoding: gzip\r\nTransfer-Encoding: chunked\r\n\r\n", - // HTTP_RESPONSE_OPTIONS - "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\nAccess-Control-Allow-Methods: *\r\nAccess-Control-Max-Age: 3600\r\nAccess-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, authorization\r\n\r\n", - // HTTP_RESPONSE_GRAFANA - "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" -}; - -static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, const char *desc) { - httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc); - - char head[512] = {0}; - char body[512] = {0}; - - int8_t httpVersion = 0; - int8_t keepAlive = 0; - if (pContext->parser != NULL) { - httpVersion = pContext->parser->httpVersion; - keepAlive = pContext->parser->keepAlive; - } - - int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[httpVersion], httpCode, - httpCodeStr, httpKeepAliveStr[keepAlive], bodyLen); - - httpWriteBuf(pContext, head, headLen); - httpWriteBuf(pContext, body, bodyLen); - httpCloseContextByApp(pContext); -} - -void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { - int32_t httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR; - if (errNo == TSDB_CODE_SUCCESS) - httpCode = HTTP_CODE_OK; - else if (errNo == TSDB_CODE_HTTP_SERVER_OFFLINE) - httpCode = HTTP_CODE_NOT_FOUND; - else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_INVALID_URL) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY) - httpCode = HTTP_CODE_INSUFFICIENT_STORAGE; - else if (errNo == TSDB_CODE_HTTP_REQUSET_TOO_BIG) - httpCode = HTTP_CODE_PAYLOAD_TOO_LARGE; - else if (errNo == TSDB_CODE_HTTP_NO_AUTH_INFO) - httpCode = HTTP_CODE_UNAUTHORIZED; - else if (errNo == TSDB_CODE_HTTP_NO_MSG_INPUT) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_NO_SQL_INPUT) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_NO_EXEC_USEDB) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_SESSION_FULL) - httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR; - else if (errNo == TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR) - httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR; - else if (errNo == TSDB_CODE_HTTP_INVALID_MULTI_REQUEST) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_CREATE_GZIP_FAILED) - httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR; - else if (errNo == TSDB_CODE_HTTP_FINISH_GZIP_FAILED) - httpCode = HTTP_CODE_INTERNAL_SERVER_ERROR; - else if (errNo == TSDB_CODE_HTTP_INVALID_VERSION) - httpCode = HTTP_CODE_HTTP_VER_NOT_SUPPORTED; - else if (errNo == TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH) - httpCode = HTTP_CODE_LENGTH_REQUIRED; - else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_TYPE) - httpCode = HTTP_CODE_UNAUTHORIZED; - else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_FORMAT) - httpCode = HTTP_CODE_UNAUTHORIZED; - else if (errNo == TSDB_CODE_HTTP_INVALID_BASIC_AUTH) - httpCode = HTTP_CODE_UNAUTHORIZED; - else if (errNo == TSDB_CODE_HTTP_INVALID_TAOSD_AUTH) - httpCode = HTTP_CODE_UNAUTHORIZED; - else if (errNo == TSDB_CODE_HTTP_PARSE_METHOD_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_TARGET_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_VERSION_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_SP_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_STATUS_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_PHRASE_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_CRLF_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_END_FAILED) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_INVALID_STATE) - httpCode = HTTP_CODE_BAD_REQUEST; - else if (errNo == TSDB_CODE_HTTP_PARSE_ERROR_STATE) - httpCode = HTTP_CODE_BAD_REQUEST; - else - httpCode = HTTP_CODE_BAD_REQUEST; - - if (pContext->parser && pContext->parser->httpCode != 0) { - httpCode = pContext->parser->httpCode; - } - - pContext->error = true; - - char *httpCodeStr = httpGetStatusDesc(httpCode); - httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo & 0XFFFF, tstrerror(errNo)); -} - -void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { - int32_t httpCode = HTTP_CODE_BAD_REQUEST; - char temp[512] = {0}; - int32_t len = sprintf(temp, "invalid SQL: %s", errMsg); - - for (int32_t i = 0; i < len; ++i) { - if (temp[i] == '\"') { - temp[i] = '\''; - } else if (temp[i] == '\n') { - temp[i] = ' '; - } else { - } - } - - httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_OPERATION & 0XFFFF, temp); -} - -void httpSendSuccResp(HttpContext *pContext, char *desc) { - char head[1024] = {0}; - char body[1024] = {0}; - - int8_t httpVersion = 0; - int8_t keepAlive = 0; - if (pContext->parser != NULL) { - httpVersion = pContext->parser->httpVersion; - keepAlive = pContext->parser->keepAlive; - } - - int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[httpVersion], - httpKeepAliveStr[keepAlive], bodyLen); - - httpWriteBuf(pContext, head, headLen); - httpWriteBuf(pContext, body, bodyLen); - httpCloseContextByApp(pContext); -} - -void httpSendOptionResp(HttpContext *pContext, char *desc) { - char head[1024] = {0}; - char body[1024] = {0}; - - int8_t httpVersion = 0; - int8_t keepAlive = 0; - if (pContext->parser != NULL) { - httpVersion = pContext->parser->httpVersion; - keepAlive = pContext->parser->keepAlive; - } - - int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[httpVersion], - httpKeepAliveStr[keepAlive], bodyLen); - - httpWriteBuf(pContext, head, headLen); - httpWriteBuf(pContext, body, bodyLen); - httpCloseContextByApp(pContext); -} diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c deleted file mode 100644 index 24e4f90244cf52799fde1a45054875a4bf3d1850..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpRestHandle.c +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "httpLog.h" -#include "httpRestHandle.h" -#include "httpRestJson.h" -#include "tglobal.h" - -static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest}; -static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest}; -static HttpEncodeMethod restEncodeSqlTimestampMethod = { - .startJsonFp = restStartSqlJson, - .stopJsonFp = restStopSqlJson, - .buildQueryJsonFp = restBuildSqlTimestampJson, - .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, - .initJsonFp = NULL, - .cleanJsonFp = NULL, - .checkFinishedFp = NULL, - .setNextCmdFp = NULL -}; - -static HttpEncodeMethod restEncodeSqlLocalTimeStringMethod = { - .startJsonFp = restStartSqlJson, - .stopJsonFp = restStopSqlJson, - .buildQueryJsonFp = restBuildSqlLocalTimeStringJson, - .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, - .initJsonFp = NULL, - .cleanJsonFp = NULL, - .checkFinishedFp = NULL, - .setNextCmdFp = NULL -}; - -static HttpEncodeMethod restEncodeSqlUtcTimeStringMethod = { - .startJsonFp = restStartSqlJson, - .stopJsonFp = restStopSqlJson, - .buildQueryJsonFp = restBuildSqlUtcTimeStringJson, - .buildAffectRowJsonFp = restBuildSqlAffectRowsJson, - .initJsonFp = NULL, - .cleanJsonFp = NULL, - .checkFinishedFp = NULL, - .setNextCmdFp = NULL -}; - -void restInitHandle(HttpServer* pServer) { - httpAddMethod(pServer, &restDecodeMethod); - httpAddMethod(pServer, &restDecodeMethod2); -} - -bool restGetUserFromUrl(HttpContext* pContext) { - HttpParser* pParser = pContext->parser; - if (pParser->path[REST_USER_USEDB_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_USEDB_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->user, pParser->path[REST_USER_USEDB_URL_POS].str, TSDB_USER_LEN); - return true; -} - -bool restGetPassFromUrl(HttpContext* pContext) { - HttpParser* pParser = pContext->parser; - if (pParser->path[REST_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, HTTP_PASSWORD_LEN); - return true; -} - -bool restProcessLoginRequest(HttpContext* pContext) { - httpDebug("context:%p, fd:%d, user:%s, process restful login msg", pContext, pContext->fd, pContext->user); - pContext->reqType = HTTP_REQTYPE_LOGIN; - return true; -} - -bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { - httpDebug("context:%p, fd:%d, user:%s, process restful sql msg", pContext, pContext->fd, pContext->user); - - char* sql = pContext->parser->body.str; - if (sql == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT); - return false; - } - - /* - * for async test - * - if (httpCheckUsedbSql(sql)) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_EXEC_USEDB); - return false; - } - */ - - HttpSqlCmd* cmd = &(pContext->singleCmd); - cmd->nativSql = sql; - - /* find if there is db_name in url */ - pContext->db[0] = '\0'; - - HttpString *path = &pContext->parser->path[REST_USER_USEDB_URL_POS]; - if (tsHttpDbNameMandatory) { - if (path->pos == 0) { - httpError("context:%p, fd:%d, user:%s, database name is mandatory", pContext, pContext->fd, pContext->user); - httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL); - return false; - } - } - - if (path->pos > 0 && !(strlen(sql) > 4 && (sql[0] == 'u' || sql[0] == 'U') && - (sql[1] == 's' || sql[1] == 'S') && (sql[2] == 'e' || sql[2] == 'E') && sql[3] == ' ')) - { - snprintf(pContext->db, /*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN, "%s", path->str); - } - - pContext->reqType = HTTP_REQTYPE_SINGLE_SQL; - if (timestampFmt == REST_TIMESTAMP_FMT_LOCAL_STRING) { - pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod; - } else if (timestampFmt == REST_TIMESTAMP_FMT_TIMESTAMP) { - pContext->encodeMethod = &restEncodeSqlTimestampMethod; - } else if (timestampFmt == REST_TIMESTAMP_FMT_UTC_STRING) { - pContext->encodeMethod = &restEncodeSqlUtcTimeStringMethod; - } - - return true; -} - -#define REST_FUNC_URL_POS 2 -#define REST_OUTP_URL_POS 3 -#define REST_AGGR_URL_POS 4 -#define REST_BUFF_URL_POS 5 - -#define HTTP_FUNC_LEN 32 -#define HTTP_OUTP_LEN 16 -#define HTTP_AGGR_LEN 2 -#define HTTP_BUFF_LEN 32 - -static int udfSaveFile(const char *fname, const char *content, long len) { - int fd = open(fname, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755); - if (fd < 0) - return -1; - if (taosWrite(fd, (void *)content, len) < 0) - return -1; - - return 0; -} - -static bool restProcessUdfRequest(HttpContext* pContext) { - HttpParser* pParser = pContext->parser; - if (pParser->path[REST_FUNC_URL_POS].pos >= HTTP_FUNC_LEN || pParser->path[REST_FUNC_URL_POS].pos <= 0) { - return false; - } - - if (pParser->path[REST_OUTP_URL_POS].pos >= HTTP_OUTP_LEN || pParser->path[REST_OUTP_URL_POS].pos <= 0) { - return false; - } - - if (pParser->path[REST_AGGR_URL_POS].pos >= HTTP_AGGR_LEN || pParser->path[REST_AGGR_URL_POS].pos <= 0) { - return false; - } - - if (pParser->path[REST_BUFF_URL_POS].pos >= HTTP_BUFF_LEN || pParser->path[REST_BUFF_URL_POS].pos <= 0) { - return false; - } - - char* sql = pContext->parser->body.str; - int len = pContext->parser->body.pos; - if (sql == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT); - return false; - } - - char udfDir[256] = {0}; - char buf[64] = "udf-"; - char funcName[64] = {0}; - int aggr = 0; - char outputType[16] = {0}; - char buffSize[32] = {0}; - - tstrncpy(funcName, pParser->path[REST_FUNC_URL_POS].str, HTTP_FUNC_LEN); - tstrncpy(buf + 4, funcName, HTTP_FUNC_LEN); - - if (pParser->path[REST_AGGR_URL_POS].str[0] != '0') { - aggr = 1; - } - - tstrncpy(outputType, pParser->path[REST_OUTP_URL_POS].str, HTTP_OUTP_LEN); - tstrncpy(buffSize, pParser->path[REST_BUFF_URL_POS].str, HTTP_BUFF_LEN); - - taosGetTmpfilePath(funcName, udfDir); - - udfSaveFile(udfDir, sql, len); - - tfree(sql); - pContext->parser->body.str = malloc(1024); - sql = pContext->parser->body.str; - int sqlLen = sprintf(sql, "create %s function %s as \"%s\" outputtype %s bufsize %s", - aggr == 1 ? "aggregate" : " ", funcName, udfDir, outputType, buffSize); - - pContext->parser->body.pos = sqlLen; - pContext->parser->body.size = sqlLen + 1; - - HttpSqlCmd* cmd = &(pContext->singleCmd); - cmd->nativSql = sql; - - pContext->reqType = HTTP_REQTYPE_SINGLE_SQL; - pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod; - - return true; -} - -bool restProcessRequest(struct HttpContext* pContext) { - if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) { - restGetUserFromUrl(pContext); - restGetPassFromUrl(pContext); - } - - if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); - return false; - } - - if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sql")) { - return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_LOCAL_STRING); - } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sqlt")) { - return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_TIMESTAMP); - } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "sqlutc")) { - return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_UTC_STRING); - } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) { - return restProcessLoginRequest(pContext); - } else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "udf")) { - return restProcessUdfRequest(pContext); - } else { - } - - httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL); - return false; -} diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c deleted file mode 100644 index 13596b0e8a4ea4d183cc4bf75917fd08a9dd7290..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpRestJson.c +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "tsclient.h" -#include "httpLog.h" -#include "httpJson.h" -#include "httpRestHandle.h" -#include "httpRestJson.h" - -void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // data row array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - httpJsonItemToken(jsonBuf); - httpJsonInt(jsonBuf, affect_rows); - - // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - - cmd->numOfRows = 1; -} - -void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - TAOS_FIELD *fields = taos_fetch_fields(result); - int32_t num_fields = taos_num_fields(result); - - httpInitJsonBuf(jsonBuf, pContext); - httpWriteJsonBufHead(jsonBuf); - - // object begin - httpJsonToken(jsonBuf, JsonObjStt); - - // status, and data - httpJsonItemToken(jsonBuf); - httpJsonPair(jsonBuf, REST_JSON_STATUS, REST_JSON_STATUS_LEN, REST_JSON_SUCCESS, REST_JSON_SUCCESS_LEN); - - // head begin - httpJsonItemToken(jsonBuf); - httpJsonPairHead(jsonBuf, REST_JSON_HEAD, REST_JSON_HEAD_LEN); - // head array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - SSqlObj *pObj = (SSqlObj *) result; - bool isAlterSql = (pObj->sqlstr == NULL) ? false : httpCheckAlterSql(pObj->sqlstr); - - if (num_fields == 0) { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); - } else { - if (isAlterSql == true) { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); - } else { - for (int32_t i = 0; i < num_fields; ++i) { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); - } - } - } - - // head array end - httpJsonToken(jsonBuf, JsonArrEnd); - - // column_meta begin - httpJsonItemToken(jsonBuf); - httpJsonPairHead(jsonBuf, REST_JSON_HEAD_INFO, REST_JSON_HEAD_INFO_LEN); - // column_meta array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - if (num_fields == 0) { - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); - httpJsonItemToken(jsonBuf); - httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT); - httpJsonItemToken(jsonBuf); - httpJsonInt(jsonBuf, 4); - - httpJsonToken(jsonBuf, JsonArrEnd); - } else { - for (int32_t i = 0; i < num_fields; ++i) { - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - if (isAlterSql == true) { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); - } else { - httpJsonItemToken(jsonBuf); - httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); - } - - httpJsonItemToken(jsonBuf); - httpJsonInt(jsonBuf, fields[i].type); - httpJsonItemToken(jsonBuf); - httpJsonInt(jsonBuf, fields[i].bytes); - - httpJsonToken(jsonBuf, JsonArrEnd); - } - } - - // column_meta array end - httpJsonToken(jsonBuf, JsonArrEnd); - - // data begin - httpJsonItemToken(jsonBuf); - httpJsonPairHead(jsonBuf, REST_JSON_DATA, REST_JSON_DATA_LEN); - // data array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); -} - -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, - int32_t timestampFormat) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return false; - - int32_t num_fields = taos_num_fields(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - - for (int32_t k = 0; k < numOfRows; ++k) { - TAOS_ROW row = taos_fetch_row(result); - if (row == NULL) { - continue; - } - int32_t *length = taos_fetch_lengths(result); - - // data row array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); - - for (int32_t i = 0; i < num_fields; i++) { - httpJsonItemToken(jsonBuf); - - if (row[i] == NULL) { - httpJsonOriginString(jsonBuf, "null", 4); - continue; - } - - switch (fields[i].type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - httpJsonInt(jsonBuf, *((int8_t *)row[i])); - break; - case TSDB_DATA_TYPE_SMALLINT: - httpJsonInt(jsonBuf, *((int16_t *)row[i])); - break; - case TSDB_DATA_TYPE_INT: - httpJsonInt(jsonBuf, *((int32_t *)row[i])); - break; - case TSDB_DATA_TYPE_BIGINT: - httpJsonInt64(jsonBuf, *((int64_t *)row[i])); - break; - case TSDB_DATA_TYPE_UTINYINT: - httpJsonUInt(jsonBuf, *((uint8_t *)row[i])); - break; - case TSDB_DATA_TYPE_USMALLINT: - httpJsonUInt(jsonBuf, *((uint16_t *)row[i])); - break; - case TSDB_DATA_TYPE_UINT: - httpJsonUInt(jsonBuf, *((uint32_t *)row[i])); - break; - case TSDB_DATA_TYPE_UBIGINT: - httpJsonUInt64(jsonBuf, *((uint64_t *)row[i])); - break; - case TSDB_DATA_TYPE_FLOAT: - httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i])); - break; - case TSDB_DATA_TYPE_DOUBLE: - httpJsonDouble(jsonBuf, GET_DOUBLE_VAL(row[i])); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { - httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result)); - } else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) { - httpJsonInt64(jsonBuf, *((int64_t *)row[i])); - } else { - httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result)); - } - break; - default: - break; - } - } - - // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows++; - - if (pContext->fd <= 0) { - httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); - return false; - } - - if (cmd->numOfRows >= tsRestRowLimit) { - httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext, - pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit); - return false; - } - } - - httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows); - return true; -} - -bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); -} - -bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); -} - -bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); -} - -void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // data array end - httpJsonToken(jsonBuf, JsonArrEnd); - - // rows - httpJsonItemToken(jsonBuf); - httpJsonPairHead(jsonBuf, REST_JSON_ROWS, REST_JSON_ROWS_LEN); - httpJsonInt64(jsonBuf, cmd->numOfRows); - - // object end - httpJsonToken(jsonBuf, JsonObjEnd); - - httpWriteJsonBufEnd(jsonBuf); -} diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c deleted file mode 100644 index 13a0835c3960333c6d12aa443025de5fb95d565e..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpServer.c +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "tsocket.h" -#include "tutil.h" -#include "ttimer.h" -#include "tglobal.h" -#include "httpInt.h" -#include "httpContext.h" -#include "httpResp.h" -#include "httpUtil.h" - -static bool httpReadData(HttpContext *pContext); - -#ifdef __APPLE__ -static int sv_dummy = 0; -#endif - -static void httpStopThread(HttpThread *pThread) { - pThread->stop = true; - - // signal the thread to stop, try graceful method first, - // and use pthread_cancel when failed -#ifdef __APPLE__ - int sv[2]; - sv[0] = sv[1] = -1; - int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv); - do { - if (r) break; - struct epoll_event ev = {0}; - ev.events = EPOLLIN; - ev.data.ptr = &sv_dummy; - pThread->stop = true; - r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); - if (r) break; - if (1 != send(sv[1], "1", 1, 0)) { - r = -1; - break; - } - } while (0); - if (r) { - pthread_cancel(pThread->thread); - } -#else - struct epoll_event event = {.events = EPOLLIN}; - eventfd_t fd = eventfd(1, 0); - if (fd == -1) { - httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", - pThread->label, strerror(errno)); - pThread->stop = true; - pthread_cancel(pThread->thread); - } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", - pThread->label, strerror(errno)); - pthread_cancel(pThread->thread); - } -#endif // __APPLE__ - - pthread_join(pThread->thread, NULL); - -#ifdef __APPLE__ - if (sv[0] != -1) { - close(sv[0]); - sv[0] = -1; - } - if (sv[1] != -1) { - close(sv[1]); - sv[1] = -1; - } -#else // __APPLE__ - if (fd != -1) { - taosCloseSocket(fd); - } -#endif // __APPLE__ - - EpollClose(pThread->pollFd); - pthread_mutex_destroy(&(pThread->threadMutex)); -} - -void httpCleanUpConnect() { - HttpServer *pServer = &tsHttpServer; - if (pServer->pThreads == NULL) return; - - if (taosCheckPthreadValid(pServer->thread)) { - pthread_join(pServer->thread, NULL); - } - - for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - HttpThread *pThread = pServer->pThreads + i; - if (pThread != NULL) { - httpStopThread(pThread); - } - } - - httpDebug("http server:%s is cleaned up", pServer->label); -} - -static void httpProcessHttpData(void *param) { - HttpServer * pServer = &tsHttpServer; - HttpThread * pThread = (HttpThread *)param; - HttpContext *pContext; - int32_t fdNum; - - taosSetMaskSIGPIPE(); - setThreadName("httpData"); - - while (1) { - struct epoll_event events[HTTP_MAX_EVENTS]; - //-1 means uncertainty, 0-nowait, 1-wait 1 ms, set it from -1 to 1 - fdNum = epoll_wait(pThread->pollFd, events, HTTP_MAX_EVENTS, TAOS_EPOLL_WAIT_TIME); - if (pThread->stop) { - httpDebug("%p, http thread get stop event, exiting...", pThread); - break; - } - if (fdNum <= 0) continue; - - for (int32_t i = 0; i < fdNum; ++i) { -#ifdef __APPLE__ - if (events[i].data.ptr == &sv_dummy) { - // no need to drain the recv buffer of sv[0] - // since there's only one time to send at most 1 byte to sv[0] - // btw, pThread->stop shall be already set, thus never reached here - httpDebug("if you see this line, there's internal logic error"); - continue; - } -#endif // __APPLE__ - pContext = httpGetContext(events[i].data.ptr); - if (pContext == NULL) { - httpError("context:%p, is already released, close connect", events[i].data.ptr); - // epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - // taosClose(events[i].data.fd); - continue; - } - - if (events[i].events & EPOLLPRI) { - httpDebug("context:%p, fd:%d, state:%s, EPOLLPRI events occured, accessed:%d, close connect", pContext, - pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpCloseContextByServer(pContext); - continue; - } - - if (events[i].events & EPOLLRDHUP) { - httpDebug("context:%p, fd:%d, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", pContext, - pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpCloseContextByServer(pContext); - continue; - } - - if (events[i].events & EPOLLERR) { - httpDebug("context:%p, fd:%d, state:%s, EPOLLERR events occured, accessed:%d, close connect", pContext, - pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpCloseContextByServer(pContext); - continue; - } - - if (events[i].events & EPOLLHUP) { - httpDebug("context:%p, fd:%d, state:%s, EPOLLHUP events occured, accessed:%d, close connect", pContext, - pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpCloseContextByServer(pContext); - continue; - } - - if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) { - httpDebug("context:%p, fd:%d, state:%s, not in ready state, ignore read events", pContext, pContext->fd, - httpContextStateStr(pContext->state)); - httpReleaseContext(pContext/*, true*/); - continue; - } - - if (pServer->status != HTTP_SERVER_RUNNING) { - httpDebug("context:%p, fd:%d, state:%s, server is not running, accessed:%d, close connect", pContext, - pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes); - httpSendErrorResp(pContext, TSDB_CODE_HTTP_SERVER_OFFLINE); - httpNotifyContextClose(pContext); - } else { - if (httpReadData(pContext)) { - (*(pThread->processData))(pContext); - atomic_fetch_add_32(&pServer->requestNum, 1); - } - } - } - } -} - -static void *httpAcceptHttpConnection(void *arg) { - SOCKET connFd = -1; - struct sockaddr_in clientAddr; - int32_t threadId = 0; - HttpServer * pServer = &tsHttpServer; - HttpThread * pThread = NULL; - HttpContext * pContext = NULL; - int32_t totalFds = 0; - - taosSetMaskSIGPIPE(); - setThreadName("httpAcceptConn"); - - pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); - - if (pServer->fd < 0) { - httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label, - taosIpStr(pServer->serverIp), pServer->serverPort, strerror(errno)); - return NULL; - } else { - httpInfo("http server init success at %u", pServer->serverPort); - pServer->status = HTTP_SERVER_RUNNING; - } - - while (1) { - socklen_t addrlen = sizeof(clientAddr); - connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); - if (pServer->stop) { - httpDebug("http server:%s socket stop, exiting...", pServer->label); - break; - } - - if (connFd == -1) { - if (errno == EINVAL) { - httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); - break; - } - httpError("http server:%s, accept connect failure, errno:%d reason:%s", pServer->label, errno, strerror(errno)); - continue; - } - - totalFds = 1; - for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - totalFds += pServer->pThreads[i].numOfContexts; - } - -#if 0 - if (totalFds > tsHttpCacheSessions * 100) { - httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd, - taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions); - taosCloseSocket(connFd); - continue; - } -#endif - - taosKeepTcpAlive(connFd); - taosSetNonblocking(connFd, 1); - - // pick up the thread to handle this connection - pThread = pServer->pThreads + threadId; - - pContext = httpCreateContext(connFd); - if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, - taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - taosCloseSocket(connFd); - continue; - } - - pContext->pThread = pThread; - sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - - struct epoll_event event; -#ifndef _TD_NINGSI_60 - event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; -#else - event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP | EPOLLRDHUP; -#endif - event.data.ptr = pContext; - if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, - pContext->ipstr, pThread->label, strerror(errno)); - taosCloseSocket(pContext->fd); - httpReleaseContext(pContext/*, true*/); - continue; - } - - // notify the data process, add into the FdObj list - atomic_add_fetch_32(&pThread->numOfContexts, 1); - httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalContext:%d, accept a new connection", pContext, - connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds); - - // pick up next thread for next connection - threadId++; - threadId = threadId % pServer->numOfThreads; - } - - taosCloseSocket(pServer->fd); - return NULL; -} - -bool httpInitConnect() { - HttpServer *pServer = &tsHttpServer; - pServer->pThreads = calloc(pServer->numOfThreads, sizeof(HttpThread)); - if (pServer->pThreads == NULL) { - httpError("init error no enough memory"); - return false; - } - - HttpThread *pThread = pServer->pThreads; - for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - sprintf(pThread->label, "%s%d", pServer->label, i); - pThread->processData = pServer->processData; - pThread->threadId = i; - - if (pthread_mutex_init(&(pThread->threadMutex), NULL) < 0) { - httpError("http thread:%s, failed to init HTTP process data mutex, reason:%s", pThread->label, strerror(errno)); - return false; - } - - pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter - if (pThread->pollFd <= 0) { - httpError("http thread:%s, failed to create HTTP epoll", pThread->label); - pthread_mutex_destroy(&(pThread->threadMutex)); - return false; - } - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { - httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, - strerror(errno)); - pthread_mutex_destroy(&(pThread->threadMutex)); - return false; - } - pthread_attr_destroy(&thattr); - - httpDebug("http thread:%p:%s, initialized", pThread, pThread->label); - pThread++; - } - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pServer->thread), &thattr, (void *)httpAcceptHttpConnection, (void *)(pServer)) != 0) { - httpError("http server:%s, failed to create Http accept thread, reason:%s", pServer->label, strerror(errno)); - httpCleanUpConnect(); - return false; - } - pthread_attr_destroy(&thattr); - - httpDebug("http server:%s, initialized, ip:%s:%u, numOfThreads:%d", pServer->label, taosIpStr(pServer->serverIp), - pServer->serverPort, pServer->numOfThreads); - return true; -} - -static bool httpReadData(HttpContext *pContext) { - HttpParser *pParser = pContext->parser; - if (!pParser->inited) { - httpInitParser(pParser); - } - - if (pParser->parsed) { - httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); - return false; - } - - pContext->accessTimes++; - pContext->lastAccessTime = taosGetTimestampSec(); - char buf[HTTP_STEP_SIZE + 1] = {0}; - - while (1) { - int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE); - if (nread > 0) { - buf[nread] = '\0'; - httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf); - int32_t ok = httpParseBuf(pParser, buf, nread); - - if (ok) { - httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok, - pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } - - if (pParser->parseCode) { - httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode); - httpSendErrorResp(pContext, pParser->parseCode); - httpNotifyContextClose(pContext); - return false; - } - - if (!pParser->parsed) { - httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd); - continue; - } else { - httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos); - return true; - } - } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { - httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno); - continue; // later again - } else { - httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno); - taosCloseSocket(pContext->fd); - httpReleaseContext(pContext/*, false */); - return false; - } - } else { - httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread); - taosCloseSocket(pContext->fd); - httpReleaseContext(pContext/*, false */); - return false; - } - } -} diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c deleted file mode 100644 index 17c12d36a39748bc7d15b019c2d43106524acaa6..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpSession.c +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "httpSession.h" -#include "../../../../include/client/taos.h" -#include "httpContext.h" -#include "httpInt.h" -#include "os.h" -#include "taoserror.h" -#include "tcache.h" -#include "tglobal.h" - -void httpCreateSession(HttpContext *pContext, void *taos) { - HttpServer *server = &tsHttpServer; - httpReleaseSession(pContext); - - pthread_mutex_lock(&server->serverMutex); - - HttpSession session; - memset(&session, 0, sizeof(HttpSession)); - session.taos = taos; - session.refCount = 1; - int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - - pContext->session = - taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); - // void *temp = pContext->session; - // taosCacheRelease(server->sessionCache, (void **)&temp, false); - - if (pContext->session == NULL) { - httpError("context:%p, fd:%d, user:%s, error:%s", pContext, pContext->fd, pContext->user, - tstrerror(TSDB_CODE_HTTP_SESSION_FULL)); - taos_close(taos); - pthread_mutex_unlock(&server->serverMutex); - return; - } - - httpDebug("context:%p, fd:%d, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd, - pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); - pthread_mutex_unlock(&server->serverMutex); -} - -static void httpFetchSessionImp(HttpContext *pContext) { - HttpServer *server = &tsHttpServer; - pthread_mutex_lock(&server->serverMutex); - - char sessionId[HTTP_SESSION_ID_LEN]; - int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - - pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); - if (pContext->session != NULL) { - atomic_add_fetch_32(&pContext->session->refCount, 1); - httpDebug("context:%p, fd:%d, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd, - pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount); - } else { - httpDebug("context:%p, fd:%d, user:%s, session not found", pContext, pContext->fd, pContext->user); - } - - pthread_mutex_unlock(&server->serverMutex); -} - -void httpGetSession(HttpContext *pContext) { - if (pContext->session == NULL) { - httpFetchSessionImp(pContext); - } else { - char sessionId[HTTP_SESSION_ID_LEN]; - snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - httpReleaseSession(pContext); - httpFetchSessionImp(pContext); - } -} - -void httpReleaseSession(HttpContext *pContext) { - if (pContext == NULL || pContext->session == NULL) return; - - int32_t refCount = atomic_sub_fetch_32(&pContext->session->refCount, 1); - assert(refCount >= 0); - httpDebug("context:%p, release session:%p:%p, sessionRef:%d", pContext, pContext->session, pContext->session->taos, - pContext->session->refCount); - - taosCacheRelease(tsHttpServer.sessionCache, (void **)&pContext->session, false); - pContext->session = NULL; -} - -static void httpDestroySession(void *data) { - HttpSession *session = data; - httpDebug("session:%p:%p, is destroyed, sessionRef:%d", session, session->taos, session->refCount); - - if (session->taos != NULL) { - taos_close(session->taos); - session->taos = NULL; - } -} - -void httpCleanUpSessions() { - if (tsHttpServer.sessionCache != NULL) { - SCacheObj *cache = tsHttpServer.sessionCache; - httpInfo("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); - taosCacheCleanup(tsHttpServer.sessionCache); - tsHttpServer.sessionCache = NULL; - } -} - -bool httpInitSessions() { - tsHttpServer.sessionCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, 5, false, httpDestroySession, "rests"); - if (tsHttpServer.sessionCache == NULL) { - httpError("failed to init session cache"); - return false; - } - - return true; -} diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c deleted file mode 100644 index 28127f582aa6e0f4e85e80e4cd5c9612139ffd15..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpSql.c +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "httpSql.h" -#include "../../../../include/client/taos.h" -#include "httpAuth.h" -#include "httpContext.h" -#include "httpInt.h" -#include "httpQueue.h" -#include "httpResp.h" -#include "httpSession.h" -#include "os.h" -#include "taoserror.h" -#include "tnote.h" -#include "tsclient.h" - -void httpProcessMultiSql(HttpContext *pContext); - -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); - -void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { - HttpContext *pContext = (HttpContext *)param; - if (pContext == NULL) return; - - HttpSqlCmds * multiCmds = pContext->multiCmds; - HttpEncodeMethod *encode = pContext->encodeMethod; - - HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; - char * sql = httpGetCmdsString(pContext, singleCmd->sql); - - bool isContinue = false; - - if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) { - isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows); - } - } - - if (isContinue) { - // retrieve next batch of rows - httpDebug("context:%p, fd:%d, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", pContext, - pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql); - taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, param); - } else { - httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, numOfRows, sql); - - if (code < 0) { - httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, tstrerror(code), sql); - } - - taos_free_result(result); - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { - (encode->stopJsonFp)(pContext, singleCmd); - } - multiCmds->pos++; - httpProcessMultiSql(pContext); - } -} - -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { - int32_t code = taos_errno(result); - httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); -} - -void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { - HttpContext *pContext = (HttpContext *)param; - if (pContext == NULL) return; - - HttpSqlCmds * multiCmds = pContext->multiCmds; - HttpEncodeMethod *encode = pContext->encodeMethod; - - HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; - char * sql = httpGetCmdsString(pContext, singleCmd->sql); - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - httpWarn("context:%p, fd:%d, user:%s, process pos:%d, code:%s:inprogress, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, tstrerror(code), sql); - return; - } - - if (code != TSDB_CODE_SUCCESS) { - if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) { - singleCmd->code = code; - httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos + 1, tstrerror(code), sql); - } else { - singleCmd->code = code; - httpError("context:%p, fd:%d, user:%s, process pos:%d, error code:%s, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, tstrerror(code), sql); - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) { - if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result); - if (encode->stopJsonFp) (encode->stopJsonFp)(pContext, singleCmd); - } - } - multiCmds->pos++; - httpProcessMultiSql(pContext); - - taos_free_result(result); - return; - } - - bool isUpdate = tscIsUpdateQuery(result); - if (isUpdate) { - // not select or show commands - int32_t affectRows = taos_affected_rows(result); - httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, affectRows, sql); - - singleCmd->code = 0; - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) { - (encode->startJsonFp)(pContext, singleCmd, result); - } - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildAffectRowJsonFp) { - (encode->buildAffectRowJsonFp)(pContext, singleCmd, affectRows); - } - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { - (encode->stopJsonFp)(pContext, singleCmd); - } - - if (encode->setNextCmdFp) { - (encode->setNextCmdFp)(pContext, singleCmd, code); - } else { - multiCmds->pos++; - } - - taos_free_result(result); - httpProcessMultiSql(pContext); - } else { - httpDebug("context:%p, fd:%d, user:%s, process pos:%d, start retrieve, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, sql); - - if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) { - (encode->startJsonFp)(pContext, singleCmd, result); - } - taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, pContext); - } -} - -void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { - int32_t code = taos_errno(result); - int32_t affectRows = taos_affected_rows(result); - httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp); -} - -void httpProcessMultiSql(HttpContext *pContext) { - HttpSqlCmds * multiCmds = pContext->multiCmds; - HttpEncodeMethod *encode = pContext->encodeMethod; - - if (multiCmds->pos >= multiCmds->size) { - httpDebug("context:%p, fd:%d, user:%s, process pos:%d, size:%d, stop mulit-querys", pContext, pContext->fd, - pContext->user, multiCmds->pos, multiCmds->size); - if (encode->cleanJsonFp) { - (encode->cleanJsonFp)(pContext); - } - httpCloseContextByApp(pContext); - return; - } - - HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->pos; - - char *sql = httpGetCmdsString(pContext, cmd->sql); - httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user, - multiCmds->pos, sql); - nPrintHttp("%s", sql); - taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext); -} - -void httpProcessMultiSqlCmd(HttpContext *pContext) { - if (pContext == NULL) return; - - HttpSqlCmds *multiCmds = pContext->multiCmds; - if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_MULTI_REQUEST); - return; - } - - httpDebug("context:%p, fd:%d, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd, pContext->user, - multiCmds->pos, multiCmds->size); - HttpEncodeMethod *encode = pContext->encodeMethod; - if (encode->initJsonFp) { - (encode->initJsonFp)(pContext); - } - - httpProcessMultiSql(pContext); -} - -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); - -void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { - HttpContext *pContext = (HttpContext *)param; - if (pContext == NULL) return; - - HttpEncodeMethod *encode = pContext->encodeMethod; - - bool isContinue = false; - - if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { - if (encode->buildQueryJsonFp) { - isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows); - } - } - - if (isContinue) { - // retrieve next batch of rows - httpDebug("context:%p, fd:%d, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, - numOfRows); - taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, param); - } else { - httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, - numOfRows); - - if (code < 0) { - httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user, - tstrerror(code)); - } - - taos_free_result(result); - - if (encode->stopJsonFp) { - (encode->stopJsonFp)(pContext, &pContext->singleCmd); - } - - httpCloseContextByApp(pContext); - } -} - -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { - int32_t code = taos_errno(result); - httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); -} - -void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { - HttpContext *pContext = (HttpContext *)param; - if (pContext == NULL) return; - - HttpEncodeMethod *encode = pContext->encodeMethod; - - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), result); - return; - } - - if (code != TSDB_CODE_SUCCESS) { - SSqlObj *pObj = (SSqlObj *)result; - if (code == TSDB_CODE_TSC_INVALID_OPERATION) { - terrno = code; - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, - pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); - httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); - } else { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, - tstrerror(code), pObj); - httpSendErrorResp(pContext, code); - } - taos_free_result(result); - return; - } - - bool isUpdate = tscIsUpdateQuery(result); - if (isUpdate) { - // not select or show commands - int32_t affectRows = taos_affected_rows(result); - assert(affectRows == affectRowsInput); - - httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, - pContext->user, affectRows, result); - - if (encode->startJsonFp) { - (encode->startJsonFp)(pContext, &pContext->singleCmd, result); - } - - if (encode->buildAffectRowJsonFp) { - (encode->buildAffectRowJsonFp)(pContext, &pContext->singleCmd, affectRows); - } - - if (encode->stopJsonFp) { - (encode->stopJsonFp)(pContext, &pContext->singleCmd); - } - - taos_free_result(result); - httpCloseContextByApp(pContext); - } else { - httpDebug("context:%p, fd:%d, user:%s, start retrieve", pContext, pContext->fd, pContext->user); - - if (encode->startJsonFp) { - (encode->startJsonFp)(pContext, &pContext->singleCmd, result); - } - - taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, pContext); - } -} - -void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { - int32_t code = taos_errno(result); - int32_t affectRows = taos_affected_rows(result); - httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp); -} - -void httpProcessSingleSqlCmd(HttpContext *pContext) { - HttpSqlCmd * cmd = &pContext->singleCmd; - char * sql = cmd->nativSql; - HttpSession *pSession = pContext->session; - - if (sql == NULL || sql[0] == 0) { - httpError("context:%p, fd:%d, user:%s, error:no sql input", pContext, pContext->fd, pContext->user); - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT); - return; - } - - httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql); - nPrintHttp("%s", sql); - taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext); -} - -void httpProcessLoginCmd(HttpContext *pContext) { - char token[128] = {0}; - if (httpGenTaosdAuthToken(pContext, token, 128) != 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR); - } else { - httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user, - token); - httpSendSuccResp(pContext, token); - } -} - -void httpProcessHeartBeatCmd(HttpContext *pContext) { - HttpEncodeMethod *encode = pContext->encodeMethod; - if (encode->startJsonFp) { - (encode->startJsonFp)(pContext, &pContext->singleCmd, NULL); - } - if (encode->stopJsonFp) { - (encode->stopJsonFp)(pContext, &pContext->singleCmd); - } - httpCloseContextByApp(pContext); -} - -void httpExecCmd(HttpContext *pContext) { - switch (pContext->reqType) { - case HTTP_REQTYPE_LOGIN: - httpProcessLoginCmd(pContext); - break; - case HTTP_REQTYPE_SINGLE_SQL: - httpProcessSingleSqlCmd(pContext); - break; - case HTTP_REQTYPE_MULTI_SQL: - httpProcessMultiSqlCmd(pContext); - break; - case HTTP_REQTYPE_HEARTBEAT: - httpProcessHeartBeatCmd(pContext); - break; - case HTTP_REQTYPE_OTHERS: - httpCloseContextByApp(pContext); - break; - default: - httpCloseContextByApp(pContext); - break; - } - - memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); -} - -void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { - HttpContext *pContext = param; - taos_free_result(result); - - if (pContext == NULL) return; - - if (code < 0) { - httpError("context:%p, fd:%d, user:%s, login error, code:%s", pContext, pContext->fd, pContext->user, - tstrerror(code)); - httpSendErrorResp(pContext, code); - return; - } - - httpDebug("context:%p, fd:%d, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd, pContext->user, - pContext->taos); - if (pContext->taos == NULL) { - httpError("context:%p, fd:%d, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->user); - httpSendErrorResp(pContext, TSDB_CODE_HTTP_LOGIN_FAILED); - return; - } - - httpCreateSession(pContext, pContext->taos); - - if (pContext->session == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL); - } else { - httpExecCmd(pContext); - } -} - -void httpProcessRequest(HttpContext *pContext) { - httpGetSession(pContext); - - if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) { - taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext, - &(pContext->taos)); - httpDebug("context:%p, fd:%d, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, pContext->user, - pContext->taos); - - if (pContext->taos != NULL) { - STscObj *pObj = pContext->taos; - pObj->from = TAOS_REQ_FROM_HTTP; - } - } else { - httpExecCmd(pContext); - } -} - -int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql) -{ - char *pos; - - if (oldSql == NULL || newSql == NULL) { - return TSDB_CODE_SUCCESS; - } - - /* bad sql clause */ - pos = strstr(oldSql, "%%"); - if (pos) { - httpError("bad sql:%s", oldSql); - return TSDB_CODE_HTTP_REQUEST_JSON_ERROR; - } - - pos = strchr(oldSql, '%'); - if (pos == NULL) { - httpDebug("sql:%s", oldSql); - *newSql = oldSql; - return TSDB_CODE_SUCCESS; - } - - *newSql = (char *) calloc(1, (strlen(oldSql) << 1) + 1); - if (newSql == NULL) { - httpError("failed to allocate for new sql, old sql:%s", oldSql); - return TSDB_CODE_HTTP_NO_ENOUGH_MEMORY; - } - - char *src = oldSql; - char *dst = *newSql; - size_t sqlLen = strlen(src); - - while (1) { - memcpy(dst, src, pos - src + 1); - dst += pos - src + 1; - *dst++ = '%'; - - if (pos + 1 >= oldSql + sqlLen) { - break; - } - - src = ++pos; - pos = strchr(pos, '%'); - if (pos == NULL) { - memcpy(dst, src, strlen(src)); - break; - } - } - - return TSDB_CODE_SUCCESS; -} - -void httpCheckFreeEscapedSql(char *oldSql, char *newSql) -{ - if (oldSql && newSql) { - if (oldSql != newSql) { - free(newSql); - } - } -} - diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c deleted file mode 100644 index 878a3251c030610b5866fa148c11a93465becd31..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpSystem.c +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "../../../../include/client/taos.h" -#include "httpContext.h" -#include "httpGcHandle.h" -#include "httpHandle.h" -#include "httpInt.h" -#include "httpMetricsHandle.h" -#include "httpQueue.h" -#include "httpResp.h" -#include "httpRestHandle.h" -#include "httpServer.h" -#include "httpSession.h" -#include "httpTgHandle.h" -#include "os.h" -#include "tadmin.h" -#include "tglobal.h" -#include "tsocket.h" -#include "ttimer.h" - -#ifndef _ADMIN -void adminInitHandle(HttpServer* pServer) {} -void opInitHandle(HttpServer* pServer) {} -#endif - -HttpServer tsHttpServer; - -int32_t httpInitSystem() { - strcpy(tsHttpServer.label, "rest"); - tsHttpServer.serverIp = 0; - tsHttpServer.serverPort = tsHttpPort; - tsHttpServer.numOfThreads = tsHttpMaxThreads; - tsHttpServer.processData = httpProcessData; - - pthread_mutex_init(&tsHttpServer.serverMutex, NULL); - - restInitHandle(&tsHttpServer); - adminInitHandle(&tsHttpServer); - gcInitHandle(&tsHttpServer); - tgInitHandle(&tsHttpServer); - opInitHandle(&tsHttpServer); - metricsInitHandle(&tsHttpServer); - return 0; -} - -int32_t httpStartSystem() { - httpInfo("start http server ..."); - - if (tsHttpServer.status != HTTP_SERVER_INIT) { - httpError("http server is already started"); - return -1; - } - - if (!httpInitResultQueue()) { - httpError("http init result queue failed"); - return -1; - } - - if (!httpInitContexts()) { - httpError("http init contexts failed"); - return -1; - } - - if (!httpInitSessions()) { - httpError("http init session failed"); - return -1; - } - - if (!httpInitConnect()) { - httpError("http init server failed"); - return -1; - } - - return 0; -} - -void httpStopSystem() { - tsHttpServer.status = HTTP_SERVER_CLOSING; - tsHttpServer.stop = 1; -#ifdef WINDOWS - closesocket(tsHttpServer.fd); -#elif __APPLE__ - if (tsHttpServer.fd!=-1) { - close(tsHttpServer.fd); - tsHttpServer.fd = -1; - } -#else - shutdown(tsHttpServer.fd, SHUT_RD); -#endif - tgCleanupHandle(); -} - -void httpCleanUpSystem() { - httpInfo("http server cleanup"); - httpStopSystem(); - - httpCleanUpConnect(); - httpCleanupContexts(); - httpCleanUpSessions(); - httpCleanupResultQueue(); - - pthread_mutex_destroy(&tsHttpServer.serverMutex); - tfree(tsHttpServer.pThreads); - tsHttpServer.pThreads = NULL; - - tsHttpServer.status = HTTP_SERVER_CLOSED; -} - -int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c deleted file mode 100644 index 32516b9fd1b5126c4e60a547bb1568d4d063d683..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpTgHandle.c +++ /dev/null @@ -1,917 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tglobal.h" -#include "taosdef.h" -#include "taosmsg.h" -#include "httpInt.h" -#include "httpTgHandle.h" -#include "httpTgJson.h" -#include "cJSON.h" - -/* - * taos.telegraf.cfg formats like - { - "metrics": [ - { - "name" : "system", - "tbname" : "system_uptime", - "fields": [ - "uptime" - ] - }, - { - "name": "system", - "tbname" : "system_uptime_format", - "fields": [ - "uptime_format" - ] - }, - { - "name": "swap", - "tbname" : "swap_in", - "fields": [ - "in" - ] - }, - { - "name": "cpu", - "tbname" : "cpu_usage", - "fields": [ - "usage_active", - "usage_guest" - ] - } - ] - } - */ - -#define TG_MAX_SORT_TAG_SIZE 20 - -static HttpDecodeMethod tgDecodeMethod = {"telegraf", tgProcessRquest}; -static HttpEncodeMethod tgQueryMethod = { - .startJsonFp = tgStartQueryJson, - .stopJsonFp = tgStopQueryJson, - .buildQueryJsonFp = NULL, - .buildAffectRowJsonFp = tgBuildSqlAffectRowsJson, - .initJsonFp = tgInitQueryJson, - .cleanJsonFp = tgCleanQueryJson, - .checkFinishedFp = tgCheckFinished, - .setNextCmdFp = tgSetNextCmd -}; - -static const char DEFAULT_TELEGRAF_CFG[] = - "{\"metrics\":[" - "{\"name\":\"system\",\"tbname\":\"system_uptime\",\"fields\":[\"uptime\"]}," - "{\"name\":\"system\",\"tbname\":\"system_uptime_format\",\"fields\":[\"uptime_format\"]}," - "{\"name\":\"swap\",\"tbname\":\"swap_in\",\"fields\":[\"in\"]}," - "{\"name\":\"cpu\",\"tbname\":\"cpu_usage\",\"fields\":[\"usage_guest\"]}" - "]}"; - -typedef struct { - char * name; - char * tbName; - char ** fields; - int32_t fieldNum; -} STgSchema; - -typedef struct { - STgSchema *schemas; - int32_t size; - int32_t pos; -} STgSchemas; - -static STgSchemas tgSchemas = {0}; - -void tgFreeSchema(STgSchema *schema) { - if (schema->name != NULL) { - free(schema->name); - schema->name = NULL; - } - if (schema->tbName != NULL) { - free(schema->tbName); - schema->tbName = NULL; - } - if (schema->fields != NULL) { - for (int32_t f = 0; f < schema->fieldNum; ++f) { - if (schema->fields[f] != NULL) { - free(schema->fields[f]); - schema->fields[f] = NULL; - } - } - free(schema->fields); - schema->fields = NULL; - schema->fieldNum = 0; - } -} - -void tgFreeSchemas() { - if (tgSchemas.schemas != NULL) { - for (int32_t s = 0; s < tgSchemas.size; ++s) { - tgFreeSchema(&tgSchemas.schemas[s]); - } - free(tgSchemas.schemas); - tgSchemas.size = 0; - tgSchemas.schemas = NULL; - } -} - -void tgInitSchemas(int32_t size) { - tgFreeSchemas(); - tgSchemas.schemas = calloc(sizeof(STgSchema), size); - tgSchemas.size = 0; -} - -void tgParseSchemaMetric(cJSON *metric) { - STgSchema schema = {0}; - bool parsedOk = true; - - // name - cJSON *name = cJSON_GetObjectItem(metric, "name"); - if (name == NULL) { - parsedOk = false; - goto ParseEnd; - } - if (name->type != cJSON_String) { - parsedOk = false; - goto ParseEnd; - } - if (name->valuestring == NULL) { - parsedOk = false; - goto ParseEnd; - } - int32_t nameLen = (int32_t)strlen(name->valuestring); - if (nameLen == 0) { - parsedOk = false; - goto ParseEnd; - } - - schema.name = calloc(nameLen + 1, 1); - strcpy(schema.name, name->valuestring); - - // tbname - cJSON *tbname = cJSON_GetObjectItem(metric, "tbname"); - if (tbname == NULL) { - parsedOk = false; - goto ParseEnd; - } - if (tbname->type != cJSON_String) { - parsedOk = false; - goto ParseEnd; - } - if (tbname->valuestring == NULL) { - parsedOk = false; - goto ParseEnd; - } - int32_t tbnameLen = (int32_t)strlen(tbname->valuestring); - if (tbnameLen == 0) { - parsedOk = false; - goto ParseEnd; - } - - schema.tbName = calloc(tbnameLen + 1, 1); - strcpy(schema.tbName, tbname->valuestring); - - // fields - cJSON *fields = cJSON_GetObjectItem(metric, "fields"); - if (fields == NULL) { - goto ParseEnd; - } - int32_t fieldSize = cJSON_GetArraySize(fields); - if (fieldSize <= 0 || fieldSize > TSDB_MAX_COLUMNS) { - goto ParseEnd; - } - - if (fieldSize > 0) { - schema.fields = calloc(sizeof(STgSchema), (size_t)fieldSize); - schema.fieldNum = fieldSize; - for (int32_t i = 0; i < fieldSize; i++) { - cJSON *field = cJSON_GetArrayItem(fields, i); - if (field == NULL) { - parsedOk = false; - goto ParseEnd; - } - if (field->valuestring == NULL) { - parsedOk = false; - goto ParseEnd; - } - nameLen = (int32_t)strlen(field->valuestring); - if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) { - parsedOk = false; - goto ParseEnd; - } - schema.fields[i] = calloc(nameLen + 1, 1); - strcpy(schema.fields[i], field->valuestring); - } - } - -ParseEnd: - if (parsedOk) { - tgSchemas.schemas[tgSchemas.size++] = schema; - } else { - tgFreeSchema(&schema); - } -} - -int32_t tgParseSchema(const char *content, char *fileName) { - cJSON *root = cJSON_Parse(content); - if (root == NULL) { - httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); - return -1; - } - int32_t size = 0; - cJSON * metrics = cJSON_GetObjectItem(root, "metrics"); - if (metrics != NULL) { - size = cJSON_GetArraySize(metrics); - if (size <= 0) { - httpError("failed to parse telegraf schema file:%s, metrics size is 0", fileName); - cJSON_Delete(root); - return -1; - } - - tgInitSchemas(size); - for (int32_t i = 0; i < size; i++) { - cJSON *metric = cJSON_GetArrayItem(metrics, i); - if (metric != NULL) { - tgParseSchemaMetric(metric); - } - } - } else { - size = 1; - tgInitSchemas(size); - tgParseSchemaMetric(root); - } - - cJSON_Delete(root); - return size; -} - -int32_t tgReadSchema(char *fileName) { - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - return -1; - } - - httpInfo("open telegraf schema file:%s success", fileName); - fseek(fp, 0, SEEK_END); - int32_t contentSize = (int32_t)ftell(fp); - if (contentSize <= 0) { - fclose(fp); - return 0; - } - - rewind(fp); - char * content = (char *)calloc(contentSize + 1, 1); - int32_t result = (int32_t)fread(content, 1, contentSize, fp); - - if (result != contentSize) { - httpError("failed to read telegraf schema file:%s", fileName); - fclose(fp); - free(content); - return 0; - } - - content[contentSize] = 0; - int32_t schemaNum = tgParseSchema(content, fileName); - - free(content); - fclose(fp); - httpInfo("parse telegraf schema file:%s, schema size:%d", fileName, schemaNum); - - return schemaNum; -} - -void tgInitHandle(HttpServer *pServer) { - char fileName[TSDB_FILENAME_LEN * 2] = {0}; - sprintf(fileName, "%s/taos.telegraf.cfg", configDir); - if (tgReadSchema(fileName) <= 0) { - tgFreeSchemas(); - if (tgParseSchema(DEFAULT_TELEGRAF_CFG, "default") <= 0) { - tgFreeSchemas(); - } - } - - httpAddMethod(pServer, &tgDecodeMethod); -} - -void tgCleanupHandle() { tgFreeSchemas(); } - -bool tgGetUserFromUrl(HttpContext *pContext) { - HttpParser *pParser = pContext->parser; - if (pParser->path[TG_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].str, sizeof(pContext->user)); - return true; -} - -bool tgGetPassFromUrl(HttpContext *pContext) { - HttpParser *pParser = pContext->parser; - if (pParser->path[TG_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { - return false; - } - - tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].str, sizeof(pContext->pass)); - return true; -} - -char *tgGetDbFromUrl(HttpContext *pContext) { - HttpParser *pParser = pContext->parser; - if (pParser->path[TG_DB_URL_POS].pos <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_NOT_INPUT); - return NULL; - } - - if (pParser->path[TG_DB_URL_POS].pos >= TSDB_DB_NAME_LEN) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_TOO_LONG); - return NULL; - } - - return pParser->path[TG_DB_URL_POS].str; -} - -char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { - for (int32_t s = 0; s < tgSchemas.size; ++s) { - STgSchema *schema = &tgSchemas.schemas[s]; - if (strcasecmp(schema->name, stname) != 0) { - continue; - } - - bool schemaMatched = true; - for (int32_t f = 0; f < schema->fieldNum; ++f) { - char *fieldName = schema->fields[f]; - bool fieldMatched = false; - - for (int32_t i = 0; i < fieldsSize; i++) { - cJSON *field = cJSON_GetArrayItem(fields, i); - if (strcasecmp(field->string, fieldName) == 0) { - fieldMatched = true; - break; - } - } - - if (!fieldMatched) { - schemaMatched = false; - break; - } - } - - if (schemaMatched) { - return schema->tbName; - } - } - - return stname; -} - -/* - * parse single metric - { - "fields": { - "field_1": 30, - "field_2": 4, - "field_N": 59, - "n_images": 660 - }, - "name": "docker", - "tags": { - "host": "raynor" - }, - "timestamp": 1458229140 - } - */ -bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { - // metric name - cJSON *name = cJSON_GetObjectItem(metric, "name"); - if (name == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NULL); - return false; - } - if (name->type != cJSON_String) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_TYPE); - return false; - } - if (name->valuestring == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL); - return false; - } - int32_t nameLen = (int32_t)strlen(name->valuestring); - if (nameLen == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL); - return false; - } - if (nameLen >= TSDB_TABLE_NAME_LEN - 8) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_LONG); - return false; - } - - // timestamp - cJSON *timestamp = cJSON_GetObjectItem(metric, "timestamp"); - if (timestamp == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_NULL); - return false; - } - if (timestamp->type != cJSON_Number) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE); - return false; - } - if (timestamp->valueint <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL); - return false; - } - - // tags - cJSON *tags = cJSON_GetObjectItem(metric, "tags"); - if (tags == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_NULL); - return false; - } - - int32_t tagsSize = cJSON_GetArraySize(tags); - if (tagsSize <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_0); - return false; - } - - if (tagsSize > TG_MAX_SORT_TAG_SIZE) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG); - return false; - } - - cJSON *host = NULL; - - for (int32_t i = 0; i < tagsSize; i++) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NULL); - return false; - } - if (tag->string == NULL || strlen(tag->string) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_NULL); - return false; - } - - /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ - if (0) { - if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); - return false; - } - } - - if (tag->type != cJSON_Number && tag->type != cJSON_String) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE); - return false; - } - - if (tag->type == cJSON_String) { - if (tag->valuestring == NULL || strlen(tag->valuestring) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_NULL); - return false; - } - } - - if (strcasecmp(tag->string, "host") == 0) { - host = tag; - } - } - - if (host == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_NULL); - return false; - } - - if (host->type != cJSON_String) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_HOST_NOT_STRING); - return false; - } - - if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_SIZE); - return false; - } - - // fields - cJSON *fields = cJSON_GetObjectItem(metric, "fields"); - if (fields == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_NULL); - return false; - } - - int32_t fieldsSize = cJSON_GetArraySize(fields); - if (fieldsSize <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_0); - return false; - } - - if (fieldsSize > (TSDB_MAX_COLUMNS - TSDB_MAX_TAGS - 1)) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG); - return false; - } - - for (int32_t i = 0; i < fieldsSize; i++) { - cJSON *field = cJSON_GetArrayItem(fields, i); - if (field == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NULL); - return false; - } - if (field->string == NULL || strlen(field->string) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_NULL); - return false; - } - /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ - if (0) { - if (strlen(field->string) >= TSDB_COL_NAME_LEN) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); - return false; - } - } - if (field->type != cJSON_Number && field->type != cJSON_String) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE); - return false; - } - if (field->type == cJSON_String) { - if (field->valuestring == NULL || strlen(field->valuestring) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL); - return false; - } - } - } - - // assembling cmds - HttpSqlCmd *stable_cmd = httpNewSqlCmd(pContext); - if (stable_cmd == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - return false; - } - stable_cmd->cmdType = HTTP_CMD_TYPE_CREATE_STBALE; - stable_cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN; - - HttpSqlCmd *table_cmd = httpNewSqlCmd(pContext); - if (table_cmd == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - return false; - } - table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; - - // order by tag name - cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int32_t orderTagsLen = 0; - for (int32_t i = 0; i < tagsSize; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - orderedTags[orderTagsLen++] = tag; - for (int32_t j = orderTagsLen - 1; j >= 1; --j) { - cJSON *tag1 = orderedTags[j]; - cJSON *tag2 = orderedTags[j - 1]; - if (strcasecmp(tag1->string, "host") == 0 || strcmp(tag1->string, tag2->string) < 0) { - orderedTags[j] = tag2; - orderedTags[j - 1] = tag1; - } - } - } - orderTagsLen = orderTagsLen < TSDB_MAX_TAGS ? orderTagsLen : TSDB_MAX_TAGS; - - table_cmd->tagNum = stable_cmd->tagNum = (int8_t)orderTagsLen; - table_cmd->timestamp = stable_cmd->timestamp = httpAddToSqlCmdBuffer(pContext, "%" PRId64, timestamp->valueint); - - // stable name - char *stname = tgGetStableName(name->valuestring, fields, fieldsSize); - table_cmd->metric = stable_cmd->metric = httpAddToSqlCmdBuffer(pContext, "%s", stname); - if (tsTelegrafUseFieldNum == 0) { - table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname); - } else { - table_cmd->stable = stable_cmd->stable = - httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); - } - table_cmd->stable = stable_cmd->stable = - httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); - - // stable tag for detail - for (int32_t i = 0; i < orderTagsLen; ++i) { - cJSON *tag = orderedTags[i]; - - char *tagStr = NULL; - int32_t retCode = httpCheckAllocEscapeSql(tag->string, &tagStr); - if (retCode != TSDB_CODE_SUCCESS) { - httpSendErrorResp(pContext, retCode); - - return false; - } - - stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tagStr); - - httpCheckFreeEscapedSql(tag->string, tagStr); - - if (tag->type == cJSON_String) - stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "'%s'", tag->valuestring); - else if (tag->type == cJSON_Number) - stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "%" PRId64, tag->valueint); - else if (tag->type == cJSON_True) - stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "1"); - else if (tag->type == cJSON_False) - stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "0"); - else - stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "NULL"); - } - - // table name - if (tsTelegrafUseFieldNum == 0) { - table_cmd->table = stable_cmd->table = - httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); - } else { - table_cmd->table = stable_cmd->table = - httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); - } - for (int32_t i = 0; i < orderTagsLen; ++i) { - cJSON *tag = orderedTags[i]; - if (tag == host) continue; - if (tag->type == cJSON_String) - httpAddToSqlCmdBufferNoTerminal(pContext, "_%s", tag->valuestring); - else if (tag->type == cJSON_Number) - httpAddToSqlCmdBufferNoTerminal(pContext, "_%" PRId64, tag->valueint); - else if (tag->type == cJSON_False) - httpAddToSqlCmdBufferNoTerminal(pContext, "_0"); - else if (tag->type == cJSON_True) - httpAddToSqlCmdBufferNoTerminal(pContext, "_1"); - else - httpAddToSqlCmdBufferNoTerminal(pContext, "_n"); - } - httpAddToSqlCmdBuffer(pContext, ""); - - table_cmd->table = stable_cmd->table = - httpShrinkTableName(pContext, table_cmd->table, httpGetCmdsString(pContext, table_cmd->table)); - - // assembling create stable sql - stable_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "create table if not exists %s.%s(ts timestamp", db, - httpGetCmdsString(pContext, table_cmd->stable)); - for (int32_t i = 0; i < fieldsSize; ++i) { - cJSON *field = cJSON_GetArrayItem(fields, i); - char * field_type = "double"; - if (field->type == cJSON_String) - field_type = "binary(32)"; - else if (field->type == cJSON_False || field->type == cJSON_True) - field_type = "tinyint"; - else { - } - - char *field_name = field->string; - httpAddToSqlCmdBufferNoTerminal(pContext, ",f_%s %s", field_name, field_type); - } - httpAddToSqlCmdBufferNoTerminal(pContext, ") tags("); - - for (int32_t i = 0; i < orderTagsLen; ++i) { - cJSON *tag = orderedTags[i]; - char * tag_type = "bigint"; - if (tag->type == cJSON_String) - tag_type = "binary(32)"; - else if (tag->type == cJSON_False || tag->type == cJSON_True) - tag_type = "tinyint"; - else { - } - - char *tag_name = tag->string; - if (i != orderTagsLen - 1) - httpAddToSqlCmdBufferNoTerminal(pContext, "t_%s %s,", tag_name, tag_type); - else - httpAddToSqlCmdBuffer(pContext, "t_%s %s)", tag_name, tag_type); - } - - // assembling insert sql - table_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "import into %s.%s using %s.%s tags(", db, - httpGetCmdsString(pContext, table_cmd->table), db, - httpGetCmdsString(pContext, table_cmd->stable)); - for (int32_t i = 0; i < orderTagsLen; ++i) { - cJSON *tag = orderedTags[i]; - if (i != orderTagsLen - 1) { - if (tag->type == cJSON_Number) - httpAddToSqlCmdBufferNoTerminal(pContext, "%" PRId64 ",", tag->valueint); - else if (tag->type == cJSON_String) - httpAddToSqlCmdBufferNoTerminal(pContext, "'%s',", tag->valuestring); - else if (tag->type == cJSON_False) - httpAddToSqlCmdBufferNoTerminal(pContext, "0,"); - else if (tag->type == cJSON_True) - httpAddToSqlCmdBufferNoTerminal(pContext, "1,"); - else { - httpAddToSqlCmdBufferNoTerminal(pContext, "NULL,"); - } - } else { - if (tag->type == cJSON_Number) - httpAddToSqlCmdBufferNoTerminal(pContext, "%" PRId64 ")", tag->valueint); - else if (tag->type == cJSON_String) - httpAddToSqlCmdBufferNoTerminal(pContext, "'%s')", tag->valuestring); - else if (tag->type == cJSON_False) - httpAddToSqlCmdBufferNoTerminal(pContext, "0)"); - else if (tag->type == cJSON_True) - httpAddToSqlCmdBufferNoTerminal(pContext, "1)"); - else { - httpAddToSqlCmdBufferNoTerminal(pContext, "NULL)"); - } - } - } - - httpAddToSqlCmdBufferNoTerminal(pContext, " values(%" PRId64 ",", timestamp->valueint); - for (int32_t i = 0; i < fieldsSize; ++i) { - cJSON *field = cJSON_GetArrayItem(fields, i); - if (i != fieldsSize - 1) { - if (field->type == cJSON_Number) - httpAddToSqlCmdBufferNoTerminal(pContext, "%lf,", field->valuedouble); - else if (field->type == cJSON_String) - httpAddToSqlCmdBufferNoTerminal(pContext, "'%s',", field->valuestring); - else if (field->type == cJSON_False) - httpAddToSqlCmdBufferNoTerminal(pContext, "0,"); - else if (field->type == cJSON_True) - httpAddToSqlCmdBufferNoTerminal(pContext, "1,"); - else { - httpAddToSqlCmdBufferNoTerminal(pContext, "NULL,"); - } - } else { - if (field->type == cJSON_Number) - httpAddToSqlCmdBuffer(pContext, "%lf)", field->valuedouble); - else if (field->type == cJSON_String) - httpAddToSqlCmdBuffer(pContext, "'%s')", field->valuestring); - else if (field->type == cJSON_False) - httpAddToSqlCmdBuffer(pContext, "0)"); - else if (field->type == cJSON_True) - httpAddToSqlCmdBuffer(pContext, "1)"); - else { - httpAddToSqlCmdBuffer(pContext, "NULL)"); - } - } - } - - return true; -} - -/** - * request from telegraf 1.7.0 - * single request: - { - "fields": { - "field_1": 30, - "field_2": 4, - "field_N": 59, - "n_images": 660 - }, - "name": "docker", - "tags": { - "host": "raynor" - }, - "timestamp": 1458229140 - } - * multiple request: - { - "metrics": [ - { - "fields": { - "field_1": 30, - "field_2": 4, - "field_N": 59, - "n_images": 660 - }, - "name": "docker", - "tags": { - "host": "raynor" - }, - "timestamp": 1458229140 - }, - { - "fields": { - "field_1": 30, - "field_2": 4, - "field_N": 59, - "n_images": 660 - }, - "name": "docker", - "tags": { - "host": "raynor" - },orderTagsLen - "timestamp": 1458229140 - } - ] - } - */ -bool tgProcessQueryRequest(HttpContext *pContext, char *db) { - httpDebug("context:%p, fd:%d, process telegraf query msg", pContext, pContext->fd); - - char *filter = pContext->parser->body.str; - if (filter == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT); - return false; - } - - cJSON *root = cJSON_Parse(filter); - if (root == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON); - return false; - } - - cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); - if (metrics != NULL) { - int32_t size = cJSON_GetArraySize(metrics); - httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size); - if (size <= 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_NULL); - cJSON_Delete(root); - return false; - } - - int32_t cmdSize = size * 2 + 1; - if (cmdSize > HTTP_MAX_CMD_SIZE) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_SIZE); - cJSON_Delete(root); - return false; - } - - if (!httpMallocMultiCmds(pContext, cmdSize, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - - HttpSqlCmd *cmd = httpNewSqlCmd(pContext); - if (cmd == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - cmd->cmdType = HTTP_CMD_TYPE_CREATE_DB; - cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN; - cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db); - - for (int32_t i = 0; i < size; i++) { - cJSON *metric = cJSON_GetArrayItem(metrics, i); - if (metric != NULL) { - if (!tgProcessSingleMetric(pContext, metric, db)) { - cJSON_Delete(root); - return false; - } - } - } - } else { - httpDebug("context:%p, fd:%d, single metric", pContext, pContext->fd); - - if (!httpMallocMultiCmds(pContext, 3, HTTP_BUFFER_SIZE)) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - - HttpSqlCmd *cmd = httpNewSqlCmd(pContext); - if (cmd == NULL) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - cJSON_Delete(root); - return false; - } - cmd->cmdType = HTTP_CMD_TYPE_CREATE_DB; - cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN; - cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db); - - if (!tgProcessSingleMetric(pContext, root, db)) { - cJSON_Delete(root); - return false; - } - } - - cJSON_Delete(root); - - pContext->reqType = HTTP_REQTYPE_MULTI_SQL; - pContext->encodeMethod = &tgQueryMethod; - pContext->multiCmds->pos = 2; - - return true; -} - -bool tgProcessRquest(struct HttpContext *pContext) { - if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO); - return false; - } - - char *db = tgGetDbFromUrl(pContext); - if (db == NULL) { - return false; - } - - return tgProcessQueryRequest(pContext, db); -} diff --git a/src/plugins/http/src/httpTgJson.c b/src/plugins/http/src/httpTgJson.c deleted file mode 100644 index 603092f09df96ae9bc9be2c0c4445d3d3f795ea1..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpTgJson.c +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "httpLog.h" -#include "httpJson.h" -#include "httpResp.h" -#include "httpTgHandle.h" -#include "httpTgJson.h" - -void tgInitQueryJson(HttpContext *pContext) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - httpInitJsonBuf(jsonBuf, pContext); - httpWriteJsonBufHead(jsonBuf); - - // array begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonObjStt); - - httpJsonPairHead(jsonBuf, "metrics", 7); - - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonArrStt); -} - -void tgCleanQueryJson(HttpContext *pContext) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // array end - httpJsonToken(jsonBuf, JsonArrEnd); - httpJsonToken(jsonBuf, JsonObjEnd); - - httpWriteJsonBufEnd(jsonBuf); -} - -void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // object begin - httpJsonItemToken(jsonBuf); - httpJsonToken(jsonBuf, JsonObjStt); - - // data - httpJsonItemToken(jsonBuf); - httpJsonPair(jsonBuf, "metric", 6, httpGetCmdsString(pContext, cmd->stable), - (int32_t)strlen(httpGetCmdsString(pContext, cmd->metric))); - - httpJsonItemToken(jsonBuf); - httpJsonPair(jsonBuf, "stable", 6, httpGetCmdsString(pContext, cmd->stable), - (int32_t)strlen(httpGetCmdsString(pContext, cmd->stable))); - - httpJsonItemToken(jsonBuf); - httpJsonPair(jsonBuf, "table", 5, httpGetCmdsString(pContext, cmd->table), - (int32_t)strlen(httpGetCmdsString(pContext, cmd->table))); - - httpJsonItemToken(jsonBuf); - httpJsonPair(jsonBuf, "timestamp", 9, httpGetCmdsString(pContext, cmd->timestamp), - (int32_t)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way -} - -void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // data - httpJsonItemToken(jsonBuf); - httpJsonPairStatus(jsonBuf, cmd->code); - - // object end - httpJsonToken(jsonBuf, JsonObjEnd); -} - -void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) { - JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); - if (jsonBuf == NULL) return; - - // data - httpJsonPairIntVal(jsonBuf, "affected_rows", 13, affect_rows); -} - -bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - httpDebug("context:%p, fd:%d, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext, - pContext->fd, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); - - if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { - if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { - if (code == TSDB_CODE_MND_DB_NOT_SELECTED || code == TSDB_CODE_MND_INVALID_DB) { - cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - if (multiCmds->cmds[0].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { - multiCmds->pos = (int16_t)-1; - httpDebug("context:%p, fd:%d, import failed, try create database", pContext, pContext->fd); - return false; - } - } else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - if (multiCmds->cmds[multiCmds->pos - 1].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { - multiCmds->pos = (int16_t)(multiCmds->pos - 2); - httpDebug("context:%p, fd:%d, import failed, try create stable", pContext, pContext->fd); - return false; - } - } else { - } - } else { - } - } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) { - cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpDebug("context:%p, fd:%d, code:%s, create database failed", pContext, pContext->fd, tstrerror(code)); - } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) { - cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpDebug("context:%p, fd:%d, code:%s, create stable failed", pContext, pContext->fd, tstrerror(code)); - } else { - } - - return true; -} - -void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - httpDebug("context:%p, fd:%d, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, - cmd->tagNum); - - if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { - multiCmds->pos = (int16_t)(multiCmds->pos + 2); - } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) { - multiCmds->pos++; - } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) { - multiCmds->pos++; - } else { - multiCmds->pos++; - } -} diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c deleted file mode 100644 index 12f3a6efd8168d9badcf62f3322e27c1dec769a7..0000000000000000000000000000000000000000 --- a/src/plugins/http/src/httpUtil.c +++ /dev/null @@ -1,497 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "httpUtil.h" -#include "../../../../include/client/taos.h" -#include "httpInt.h" -#include "httpResp.h" -#include "httpSql.h" -#include "os.h" -#include "tmd5.h" -#include "ttoken.h" - -bool httpCheckUsedbSql(char *sql) { - if (strstr(sql, "use ") != NULL) { - return true; - } - return false; -} - -bool httpCheckAlterSql(char *sql) { - int32_t index = 0; - - do { - SStrToken t0 = tStrGetToken(sql, &index, false); - if (t0.type != TK_LP) { - return t0.type == TK_ALTER; - } - } while (1); -} - -void httpTimeToString(int32_t t, char *buf, int32_t buflen) { - memset(buf, 0, (size_t)buflen); - char ts[32] = {0}; - - struct tm *ptm; - time_t tt = t / 1000; - ptm = localtime(&tt); - strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03d", ts, t % 1000); -} - -int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { - HttpSqlCmds *cmd = pContext->multiCmds; - if (cmd->buffer == NULL) return -1; - - int32_t remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - char * buffer = cmd->buffer + cmd->bufferPos; - int32_t len = 0; - - va_list argpointer; - va_start(argpointer, format); - len += vsnprintf(buffer, (size_t)remainLength, format, argpointer); - va_end(argpointer); - - if (cmd->bufferPos + len + 1 >= cmd->bufferSize) { - return -1; - } - - cmd->buffer[cmd->bufferPos + len] = 0; - cmd->bufferPos = cmd->bufferPos + len + 1; - - remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - return (int32_t)(buffer - cmd->buffer); -} - -int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...) { - HttpSqlCmds *cmd = pContext->multiCmds; - if (cmd->buffer == NULL) return -1; - - int32_t remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - char * buffer = cmd->buffer + cmd->bufferPos; - int32_t len = 0; - - va_list argpointer; - va_start(argpointer, format); - len += vsnprintf(buffer, (size_t)remainLength, format, argpointer); - va_end(argpointer); - - if (cmd->bufferPos + len + 1 >= cmd->bufferSize) { - return -1; - } - - cmd->bufferPos = cmd->bufferPos + len; - - remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - return (int32_t)(buffer - cmd->buffer); -} - -int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) { - HttpSqlCmds *cmd = pContext->multiCmds; - if (cmd->buffer == NULL) return -1; - - int32_t remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - char *buffer = cmd->buffer + cmd->bufferPos; - *buffer = 0; - cmd->bufferPos = cmd->bufferPos + 1; - - remainLength = cmd->bufferSize - cmd->bufferPos; - if (remainLength < 4096) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - return (int32_t)(buffer - cmd->buffer); -} - -int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize) { - HttpSqlCmds *cmd = pContext->multiCmds; - if (cmd->buffer == NULL) return -1; - - if (cmd->bufferPos + mallocSize >= cmd->bufferSize) { - if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1; - } - - char *buffer = cmd->buffer + cmd->bufferPos; - memset(cmd->buffer + cmd->bufferPos, 0, (size_t)mallocSize); - cmd->bufferPos = cmd->bufferPos + mallocSize; - - return (int32_t)(buffer - cmd->buffer); -} - -bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize) { - if (cmdSize > HTTP_MAX_CMD_SIZE) { - httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, - cmdSize, HTTP_MAX_CMD_SIZE); - return false; - } - - if (pContext->multiCmds == NULL) { - pContext->multiCmds = (HttpSqlCmds *)malloc(sizeof(HttpSqlCmds)); - if (pContext->multiCmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->user); - return false; - } - memset(pContext->multiCmds, 0, sizeof(HttpSqlCmds)); - } - - HttpSqlCmds *multiCmds = pContext->multiCmds; - if (multiCmds->cmds == NULL || cmdSize > multiCmds->maxSize) { - free(multiCmds->cmds); - multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); - if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); - return false; - } - multiCmds->maxSize = (int16_t)cmdSize; - } - - if (multiCmds->buffer == NULL || bufferSize > multiCmds->bufferSize) { - free(multiCmds->buffer); - multiCmds->buffer = (char *)malloc((size_t)bufferSize); - if (multiCmds->buffer == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, - bufferSize); - return false; - } - multiCmds->bufferSize = bufferSize; - } - - multiCmds->pos = 0; - multiCmds->size = 0; - multiCmds->bufferPos = 0; - memset(multiCmds->cmds, 0, (size_t)multiCmds->maxSize * sizeof(HttpSqlCmd)); - - return true; -} - -bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - - if (cmdSize <= 0 || cmdSize > HTTP_MAX_CMD_SIZE) { - httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user, - cmdSize, HTTP_MAX_CMD_SIZE); - return false; - } - - HttpSqlCmd *new_cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd)); - if (new_cmds == NULL && multiCmds->cmds) { - free(multiCmds->cmds); - } - multiCmds->cmds = new_cmds; - if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); - return false; - } - memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd)); - multiCmds->maxSize = (int16_t)cmdSize; - - return true; -} - -bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - - if (bufferSize <= 0 || bufferSize > HTTP_MAX_BUFFER_SIZE) { - httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd, - pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE); - return false; - } - - char *new_buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize); - if (new_buffer == NULL && multiCmds->buffer) { - free(multiCmds->buffer); - } - multiCmds->buffer = new_buffer; - if (multiCmds->buffer == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize); - return false; - } - memset(multiCmds->buffer + multiCmds->bufferSize, 0, (size_t)(bufferSize - multiCmds->bufferSize)); - multiCmds->bufferSize = bufferSize; - - return true; -} - -void httpFreeMultiCmds(HttpContext *pContext) { - if (pContext->multiCmds != NULL) { - if (pContext->multiCmds->buffer != NULL) free(pContext->multiCmds->buffer); - if (pContext->multiCmds->cmds != NULL) free(pContext->multiCmds->cmds); - free(pContext->multiCmds); - pContext->multiCmds = NULL; - } -} - -JsonBuf *httpMallocJsonBuf(HttpContext *pContext) { - if (pContext->jsonBuf == NULL) { - pContext->jsonBuf = (JsonBuf *)malloc(sizeof(JsonBuf)); - if (pContext->jsonBuf == NULL) { - return NULL; - } - - memset(pContext->jsonBuf, 0, sizeof(JsonBuf)); - } - - if (!pContext->jsonBuf->pContext) { - pContext->jsonBuf->pContext = pContext; - } - - return pContext->jsonBuf; -} - -void httpFreeJsonBuf(HttpContext *pContext) { - if (pContext->jsonBuf != NULL) { - free(pContext->jsonBuf); - pContext->jsonBuf = 0; - } -} - -bool httpCompareMethod(HttpDecodeMethod *pSrc, HttpDecodeMethod *pCmp) { - if (strcmp(pSrc->module, pCmp->module) != 0) { - return false; - } - return true; -} - -void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod) { - int32_t pos = 0; - for (pos = 0; pos < pServer->methodScannerLen; ++pos) { - if (httpCompareMethod(pServer->methodScanner[pos], pMethod)) { - break; - } - } - - if (pos == pServer->methodScannerLen && pServer->methodScannerLen < HTTP_METHOD_SCANNER_SIZE) { - pServer->methodScanner[pos] = pMethod; - pServer->methodScannerLen++; - } -} - -HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - if (multiCmds->size >= multiCmds->maxSize) { - if (!httpReMallocMultiCmdsSize(pContext, 2 * multiCmds->maxSize)) return NULL; - } - - HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->size++; - cmd->cmdType = HTTP_CMD_TYPE_UN_SPECIFIED; - cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_WITH_RETURN; - cmd->cmdState = HTTP_CMD_STATE_NOT_RUN_YET; - - return cmd; -} - -HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - if (multiCmds->size == 0) return NULL; - if (multiCmds->size > multiCmds->maxSize) return NULL; - - return multiCmds->cmds + multiCmds->size - 1; -} - -int32_t httpNextSqlCmdPos(HttpContext *pContext) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - return multiCmds->size; -} - -void httpTrimTableName(char *name) { - for (int32_t i = 0; name[i] != 0; i++) { - if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'') - name[i] = '_'; - if (i == TSDB_TABLE_NAME_LEN) { - name[i] = 0; - break; - } - } -} - -int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name) { - int32_t len = 0; - for (int32_t i = 0; name[i] != 0; i++) { - if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'' || - name[i] == '\"') - name[i] = '_'; - len++; - } - - if (len < TSDB_TABLE_NAME_LEN - 1) { - return pos; - } - - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)name, (uint32_t)len); - MD5Final(&context); - - int32_t table_name = httpAddToSqlCmdBuffer( - pContext, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - - if (table_name != -1) { - httpGetCmdsString(pContext, table_name)[0] = 't'; - } - - return table_name; -} - -char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { - HttpSqlCmds *multiCmds = pContext->multiCmds; - if (pos < 0 || pos >= multiCmds->bufferSize) { - return ""; - } - - return multiCmds->buffer + pos; -} - -int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int32_t err = 0; - z_stream gzipStream = {0}; - - static char dummyHead[2] = { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, - }; - - gzipStream.zalloc = (alloc_func)0; - gzipStream.zfree = (free_func)0; - gzipStream.opaque = (voidpf)0; - gzipStream.next_in = (Bytef *)srcData; - gzipStream.avail_in = 0; - gzipStream.next_out = (Bytef *)destData; - if (inflateInit2(&gzipStream, 47) != Z_OK) { - return -1; - } - - while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) { - gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1 - if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) { - break; - } - - if (err != Z_OK) { - if (err == Z_DATA_ERROR) { - gzipStream.next_in = (Bytef *)dummyHead; - gzipStream.avail_in = sizeof(dummyHead); - if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) { - return -2; - } - } else { - return -3; - } - } - } - - if (inflateEnd(&gzipStream) != Z_OK) { - return -4; - } - *nDestData = (int32_t)gzipStream.total_out; - - return 0; -} - -int32_t httpGzipCompressInit(HttpContext *pContext) { - pContext->gzipStream.zalloc = (alloc_func)0; - pContext->gzipStream.zfree = (free_func)0; - pContext->gzipStream.opaque = (voidpf)0; - if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != - Z_OK) { - return -1; - } - - return 0; -} - -int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, - bool isTheLast) { - int32_t err = 0; - int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out); - pContext->gzipStream.next_in = (Bytef *)srcData; - pContext->gzipStream.avail_in = (uLong)nSrcData; - pContext->gzipStream.next_out = (Bytef *)destData; - pContext->gzipStream.avail_out = (uLong)(*nDestData); - - while (pContext->gzipStream.avail_in != 0) { - if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { - return -1; - } - - int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen); - if (cacheLen >= *nDestData) { - return -2; - } - } - - if (pContext->gzipStream.avail_in != 0) { - return pContext->gzipStream.avail_in; - } - - if (isTheLast) { - for (;;) { - if ((err = deflate(&pContext->gzipStream, Z_FINISH)) == Z_STREAM_END) { - break; - } - if (err != Z_OK) { - return -3; - } - } - - if (deflateEnd(&pContext->gzipStream) != Z_OK) { - return -4; - } - } - - *nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen; - return 0; -} - -bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) { - HttpParser *pParser = pContext->parser; - - if (pos < 0 || pos >= HTTP_MAX_URL) { - return false; - } - - if (pParser->path[pos].pos <= 0) { - return false; - } - - if (strcmp(pParser->path[pos].str, cmp) != 0) { - return false; - } - - return true; -} diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt deleted file mode 100644 index 081512138505ab7e7a54a8bbe770aa293adec0be..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates) -AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - ADD_LIBRARY(mqtt ${SRC}) - TARGET_LINK_LIBRARIES(mqtt cJson mqttc) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(mqtt taos_static) - ELSE () - TARGET_LINK_LIBRARIES(mqtt taos) - ENDIF () -ENDIF () - -IF (TD_DARWIN) - ADD_LIBRARY(mqtt ${SRC}) - TARGET_LINK_LIBRARIES(mqtt cJson mqttc) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(mqtt taos_static) - ELSE () - TARGET_LINK_LIBRARIES(mqtt taos) - ENDIF () -ENDIF () - diff --git a/src/plugins/mqtt/inc/mqttInit.h b/src/plugins/mqtt/inc/mqttInit.h deleted file mode 100644 index 81a9a39a2ce04c66934c4bd25ed67bdb51c77ba3..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/inc/mqttInit.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MQTT_INIT_H -#define TDENGINE_MQTT_INIT_H -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @file - * A simple subscriber program that performs automatic reconnections. - */ -#include "mqtt.h" - -#define QOS 1 -#define TIMEOUT 10000L -#define MQTT_SEND_BUF_SIZE 102400 -#define MQTT_RECV_BUF_SIZE 102400 - -/** - * @brief A structure that I will use to keep track of some data needed - * to setup the connection to the broker. - * - * An instance of this struct will be created in my \c main(). Then, whenever - * \ref mqttReconnectClient is called, this instance will be passed. - */ -typedef struct SMqttReconnectState { - uint8_t* sendbuf; - size_t sendbufsz; - uint8_t* recvbuf; - size_t recvbufsz; -} SMqttReconnectState; - -/** - * @brief My reconnect callback. It will reestablish the connection whenever - * an error occurs. - */ -void mqttReconnectClient(struct mqtt_client* client, void** reconnect_state_vptr); - -/** - * @brief The function will be called whenever a PUBLISH message is received. - */ -void mqttPublishCallback(void** unused, struct mqtt_response_publish* published); - -/** - * @brief The client's refresher. This function triggers back-end routines to - * handle ingress/egress traffic to the broker. - * - * @note All this function needs to do is call \ref __mqtt_recv and - * \ref __mqtt_send every so often. I've picked 100 ms meaning that - * client ingress/egress traffic will be handled every 100 ms. - */ -void* mqttClientRefresher(void* client); - -/** - * @brief Safelty closes the \p sockfd and cancels the \p client_daemon before \c exit. - */ -void mqttCleanupRes(int status, int sockfd, pthread_t* client_daemon); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/plugins/mqtt/inc/mqttLog.h b/src/plugins/mqtt/inc/mqttLog.h deleted file mode 100644 index e186b8111220bd83f2f6833d84d1fdc19f744520..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/inc/mqttLog.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MQTT_LOG_H -#define TDENGINE_MQTT_LOG_H - -#include "tlog.h" - -extern int32_t mqttDebugFlag; - -#define mqttFatal(...) { if (mqttDebugFlag & DEBUG_FATAL) { taosPrintLog("MQT FATAL ", 255, __VA_ARGS__); }} -#define mqttError(...) { if (mqttDebugFlag & DEBUG_ERROR) { taosPrintLog("MQT ERROR ", 255, __VA_ARGS__); }} -#define mqttWarn(...) { if (mqttDebugFlag & DEBUG_WARN) { taosPrintLog("MQT WARN ", 255, __VA_ARGS__); }} -#define mqttInfo(...) { if (mqttDebugFlag & DEBUG_INFO) { taosPrintLog("MQT ", 255, __VA_ARGS__); }} -#define mqttDebug(...) { if (mqttDebugFlag & DEBUG_DEBUG) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }} -#define mqttTrace(...) { if (mqttDebugFlag & DEBUG_TRACE) { taosPrintLog("MQT ", mqttDebugFlag, __VA_ARGS__); }} - -#endif diff --git a/src/plugins/mqtt/inc/mqttPayload.h b/src/plugins/mqtt/inc/mqttPayload.h deleted file mode 100644 index 12a714afac52d1814f3cbf82ade610ec8af07953..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/inc/mqttPayload.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MQTT_PLYLOAD_H -#define TDENGINE_MQTT_PLYLOAD_H - -#ifdef __cplusplus -extern "C" { -#endif - -char* mqttConverJsonToSql(char* json, int maxSize); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/plugins/mqtt/src/mqttPayload.c b/src/plugins/mqtt/src/mqttPayload.c deleted file mode 100644 index 1af8b02fad7492da5664628b534b890732acbe3a..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/src/mqttPayload.c +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "cJSON.h" -#include "mqttLog.h" -#include "mqttPayload.h" - -// subscribe message like this - -/* -/test { - "timestamp": 1599121290, - "gateway": { - "name": "AcuLink 810 Gateway", - "model": "AcuLink810-868", - "serial": "S8P20200207" - }, - "device": { - "name": "Acuvim L V3 .221", - "model": "Acuvim-L-V3", - "serial": "221", - "online": true, - "readings": [ - { - "param": "Freq_Hz", - "value": "59.977539", - "unit": "Hz" - }, - { - "param": "Va_V", - "value": "122.002907", - "unit": "V" - }, - { - "param": "DI4", - "value": "5.000000", - "unit": "" - } - ] - } -} -*/ - -// send msg cmd -// mosquitto_pub -h test.mosquitto.org -t "/test" -m '{"timestamp": 1599121290,"gateway": {"name": "AcuLink 810 Gateway","model": "AcuLink810-868","serial": "S8P20200207"},"device": {"name": "Acuvim L V3 .221","model": "Acuvim-L-V3","serial": "221","online": true,"readings": [{"param": "Freq_Hz","value": "59.977539","unit": "Hz"},{"param": "Va_V","value": "122.002907","unit": "V"},{"param": "DI4","value": "5.000000","unit": ""}]}}' - -/* - * This is an example, this function needs to be implemented in order to parse the json file into a sql statement - * Note that you need to create a super table and database before writing data - * In this case: - * create database mqttdb; - * create table mqttdb.devices(ts timestamp, value bigint) tags(name binary(32), model binary(32), serial binary(16), param binary(16), unit binary(16)); - */ - -char* mqttConverJsonToSql(char* json, int maxSize) { - // const int32_t maxSize = 10240; - maxSize *= 5; - char* sql = malloc(maxSize); - - cJSON* root = cJSON_Parse(json); - if (root == NULL) { - mqttError("failed to parse msg, invalid json format"); - goto MQTT_PARSE_OVER; - } - - cJSON* timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (!timestamp || timestamp->type != cJSON_Number) { - mqttError("failed to parse msg, timestamp not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* device = cJSON_GetObjectItem(root, "device"); - if (!device) { - mqttError("failed to parse msg, device not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* name = cJSON_GetObjectItem(device, "name"); - if (!name || name->type != cJSON_String) { - mqttError("failed to parse msg, name not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* model = cJSON_GetObjectItem(device, "model"); - if (!model || model->type != cJSON_String) { - mqttError("failed to parse msg, model not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* serial = cJSON_GetObjectItem(device, "serial"); - if (!serial || serial->type != cJSON_String) { - mqttError("failed to parse msg, serial not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* readings = cJSON_GetObjectItem(device, "readings"); - if (!readings || readings->type != cJSON_Array) { - mqttError("failed to parse msg, readings not found"); - goto MQTT_PARSE_OVER; - } - - int count = cJSON_GetArraySize(readings); - if (count <= 0) { - mqttError("failed to parse msg, readings size smaller than 0"); - goto MQTT_PARSE_OVER; - } - - int len = snprintf(sql, maxSize, "insert into"); - - for (int i = 0; i < count; ++i) { - cJSON* reading = cJSON_GetArrayItem(readings, i); - if (reading == NULL) continue; - - cJSON* param = cJSON_GetObjectItem(reading, "param"); - if (!param || param->type != cJSON_String) { - mqttError("failed to parse msg, param not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* value = cJSON_GetObjectItem(reading, "value"); - if (!value || value->type != cJSON_String) { - mqttError("failed to parse msg, value not found"); - goto MQTT_PARSE_OVER; - } - - cJSON* unit = cJSON_GetObjectItem(reading, "unit"); - if (!unit || unit->type != cJSON_String) { - mqttError("failed to parse msg, unit not found"); - goto MQTT_PARSE_OVER; - } - - len += snprintf(sql + len, maxSize - len, - " mqttdb.serial_%s_%s using mqttdb.devices tags('%s', '%s', '%s', '%s', '%s') values(%" PRId64 ", %s)", - serial->valuestring, param->valuestring, name->valuestring, model->valuestring, serial->valuestring, - param->valuestring, unit->valuestring, timestamp->valueint * 1000, value->valuestring); - } - - cJSON_free(root); - return sql; - -MQTT_PARSE_OVER: - cJSON_free(root); - free(sql); - return NULL; -} \ No newline at end of file diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c deleted file mode 100644 index d0c33d43a83b7546ced3a3fb80cb370b4d99112d..0000000000000000000000000000000000000000 --- a/src/plugins/mqtt/src/mqttSystem.c +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "../../../../include/client/taos.h" -#include "mqtt.h" -#include "mqttInit.h" -#include "mqttLog.h" -#include "mqttPayload.h" -#include "os.h" -#include "posix_sockets.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tmqtt.h" - -struct SMqttReconnectState tsMqttStatus = {0}; -struct mqtt_client tsMqttClient = {0}; -static pthread_t tsMqttClientDaemonThread = {0}; -static void* tsMqttConnect = NULL; -static bool tsMqttIsRuning = false; - -int32_t mqttInitSystem() { return 0; } - -int32_t mqttStartSystem() { - tsMqttStatus.sendbufsz = MQTT_SEND_BUF_SIZE; - tsMqttStatus.recvbufsz = MQTT_RECV_BUF_SIZE; - tsMqttStatus.sendbuf = malloc(MQTT_SEND_BUF_SIZE); - tsMqttStatus.recvbuf = malloc(MQTT_RECV_BUF_SIZE); - tsMqttIsRuning = true; - - mqtt_init_reconnect(&tsMqttClient, mqttReconnectClient, &tsMqttStatus, mqttPublishCallback); - if (pthread_create(&tsMqttClientDaemonThread, NULL, mqttClientRefresher, &tsMqttClient)) { - mqttError("mqtt failed to start daemon."); - mqttCleanupRes(EXIT_FAILURE, -1, NULL); - return -1; - } - - mqttInfo("mqtt listening for topic:%s messages", tsMqttTopic); - return 0; -} - -void mqttStopSystem() { - if (tsMqttIsRuning) { - tsMqttIsRuning = false; - tsMqttClient.error = MQTT_ERROR_SOCKET_ERROR; - - taosMsleep(300); - mqttCleanupRes(EXIT_SUCCESS, tsMqttClient.socketfd, &tsMqttClientDaemonThread); - - mqttInfo("mqtt is stopped"); - } -} - -void mqttCleanUpSystem() { - mqttStopSystem(); - mqttInfo("mqtt is cleaned up"); -} - -void mqttPublishCallback(void** unused, struct mqtt_response_publish* published) { - const char* content = published->application_message; - mqttDebug("receive mqtt message, size:%d", (int)published->application_message_size); - - if (tsMqttConnect == NULL) { - tsMqttConnect = taos_connect(NULL, "_root", tsInternalPass, "", 0); - if (tsMqttConnect == NULL) { - mqttError("failed to connect to tdengine, reason:%s", tstrerror(terrno)); - return; - } else { - mqttInfo("successfully connected to the tdengine"); - } - } - - mqttTrace("receive mqtt message, content:%s", content); - - char* sql = mqttConverJsonToSql((char*)content, (int)published->application_message_size); - if (sql != NULL) { - void* res = taos_query(tsMqttConnect, sql); - int code = taos_errno(res); - if (code != 0) { - mqttError("failed to exec sql, reason:%s sql:%s", tstrerror(code), sql); - } else { - mqttTrace("successfully to exec sql:%s", sql); - } - taos_free_result(res); - } else { - mqttError("failed to parse mqtt message"); - } -} - -void* mqttClientRefresher(void* client) { - setThreadName("mqttCliRefresh"); - - while (tsMqttIsRuning) { - mqtt_sync((struct mqtt_client*)client); - taosMsleep(100); - } - - mqttDebug("mqtt quit refresher"); - return NULL; -} - -void mqttCleanupRes(int status, int sockfd, pthread_t* client_daemon) { - mqttInfo("clean up mqtt module"); - if (sockfd != -1) { - close(sockfd); - } - - if (client_daemon != NULL) { - pthread_cancel(*client_daemon); - } -} - -void mqttReconnectClient(struct mqtt_client* client, void** unused) { - mqttInfo("mqtt tries to connect to the mqtt server"); - - if (client->error != MQTT_ERROR_INITIAL_RECONNECT) { - close(client->socketfd); - } - - if (client->error != MQTT_ERROR_INITIAL_RECONNECT) { - mqttError("mqtt client was in error state %s", mqtt_error_str(client->error)); - } - - int sockfd = open_nb_socket(tsMqttHostName, tsMqttPort); - if (sockfd < 0) { - mqttError("mqtt client failed to open socket %s:%s", tsMqttHostName, tsMqttPort); - //mqttCleanupRes(EXIT_FAILURE, sockfd, NULL); - return; - } - - mqtt_reinit(client, sockfd, tsMqttStatus.sendbuf, tsMqttStatus.sendbufsz, tsMqttStatus.recvbuf, tsMqttStatus.recvbufsz); - mqtt_connect(client, tsMqttClientId, NULL, NULL, 0, tsMqttUser, tsMqttPass, MQTT_CONNECT_CLEAN_SESSION, 400); - mqtt_subscribe(client, tsMqttTopic, 0); -} diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt deleted file mode 100644 index 2cd84c7c3fff63a702d99d8b2dc45303f17528ef..0000000000000000000000000000000000000000 --- a/src/sync/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) - -LIST(REMOVE_ITEM SRC src/syncArbitrator.c) -ADD_LIBRARY(sync ${SRC}) -TARGET_LINK_LIBRARIES(sync tutil pthread common) - -LIST(APPEND BIN_SRC src/syncArbitrator.c) -LIST(APPEND BIN_SRC src/syncTcp.c) -ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) -TARGET_LINK_LIBRARIES(tarbitrator sync common os tutil) - -#ADD_SUBDIRECTORY(test) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h deleted file mode 100644 index 411e706fb11a7052f5ed3ad162bd3201e212e552..0000000000000000000000000000000000000000 --- a/src/sync/inc/syncInt.h +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_INT_H -#define TDENGINE_SYNC_INT_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "syncMsg.h" -#include "twal.h" - -#define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }} -#define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }} -#define sWarn(...) { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); }} -#define sInfo(...) { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} -#define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} -#define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} - -#define SYNC_TCP_THREADS 2 -#define SYNC_MAX_NUM 2 - -#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) -#define SYNC_RECV_BUFFER_SIZE (5*1024*1024) - -#define SYNC_MAX_FWDS 4096 -#define SYNC_FWD_TIMER 300 -#define SYNC_ROLE_TIMER 15000 // ms -#define SYNC_CHECK_INTERVAL 1000 // ms -#define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms - -#define nodeRole pNode->peerInfo[pNode->selfIndex]->role -#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version -#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus - -typedef struct { - char * buffer; - int32_t bufferSize; - char * offset; - int32_t forwards; - int32_t code; -} SRecvBuffer; - -typedef struct { - uint64_t version; - void *mhandle; - int8_t acks; - int8_t nacks; - int8_t confirmed; - int32_t code; - int64_t time; -} SFwdInfo; - -typedef struct { - int32_t first; - int32_t last; - int32_t fwds; // number of forwards - SFwdInfo fwdInfo[]; -} SSyncFwds; - -typedef struct SsyncPeer { - int32_t nodeId; - uint32_t ip; - uint16_t port; - int8_t role; - int8_t sstatus; // sync status - char fqdn[TSDB_FQDN_LEN]; // peer ip string - char id[TSDB_EP_LEN + 32]; // peer vgId + end point - uint64_t version; - uint64_t sversion; // track the peer version in retrieve process - uint64_t lastFileVer; // track the file version while retrieve - uint64_t lastWalVer; // track the wal version while retrieve - SOCKET syncFd; - SOCKET peerFd; // forward FD - int32_t numOfRetrieves; // number of retrieves tried - int32_t fileChanged; // a flag to indicate file is changed during retrieving process - int32_t refCount; - int8_t isArb; - int64_t rid; - void * timer; - void * pConn; - struct SSyncNode *pSyncNode; -} SSyncPeer; - -typedef struct SSyncNode { - char path[TSDB_FILENAME_LEN]; - int8_t replica; - int8_t quorum; - int8_t selfIndex; - uint32_t vgId; - int32_t refCount; - int64_t rid; - SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator - SSyncPeer * pMaster; - SRecvBuffer *pRecv; - SSyncFwds * pSyncFwds; // saved forward info if quorum >1 - void * pFwdTimer; - void * pRoleTimer; - void * pTsdb; - FGetWalInfo getWalInfoFp; - FWriteToCache writeToCacheFp; - FConfirmForward confirmForward; - FNotifyRole notifyRoleFp; - FNotifyFlowCtrl notifyFlowCtrlFp; - FStartSyncFile startSyncFileFp; - FStopSyncFile stopSyncFileFp; - FGetVersion getVersionFp; - FSendFile sendFileFp; - FRecvFile recvFileFp; - pthread_mutex_t mutex; -} SSyncNode; - -// sync module global -extern int32_t tsSyncNum; -extern char tsNodeFqdn[TSDB_FQDN_LEN]; -extern char * syncStatus[]; - -void * syncRetrieveData(void *param); -void * syncRestoreData(void *param); -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer); -SSyncPeer *syncAcquirePeer(int64_t rid); -void syncReleasePeer(SSyncPeer *pPeer); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_VNODEPEER_H diff --git a/src/sync/inc/syncMain.h b/src/sync/inc/syncMain.h deleted file mode 100644 index d4ddb127334ea967cdc43dcc6a57dd4cb8c0ec3b..0000000000000000000000000000000000000000 --- a/src/sync/inc/syncMain.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_PLUGINS_SYNC_H -#define TDENGINE_PLUGINS_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -int32_t syncTest1(); -int32_t syncTest2(); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/sync/inc/syncMsg.h b/src/sync/inc/syncMsg.h deleted file mode 100644 index 85ac9c78affa5282d5ca703caffc1bc5c24461bb..0000000000000000000000000000000000000000 --- a/src/sync/inc/syncMsg.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_MSG_H -#define TDENGINE_SYNC_MSG_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "tsync.h" - -typedef enum { - TAOS_SMSG_START = 0, - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_SYNC_DATA_RSP = 2, - TAOS_SMSG_SYNC_FWD = 3, - TAOS_SMSG_SYNC_FWD_RSP = 4, - TAOS_SMSG_SYNC_REQ = 5, - TAOS_SMSG_SYNC_REQ_RSP = 6, - TAOS_SMSG_SYNC_MUST = 7, - TAOS_SMSG_SYNC_MUST_RSP = 8, - TAOS_SMSG_STATUS = 9, - TAOS_SMSG_STATUS_RSP = 10, - TAOS_SMSG_SETUP = 11, - TAOS_SMSG_SETUP_RSP = 12, - TAOS_SMSG_SYNC_FILE = 13, - TAOS_SMSG_SYNC_FILE_RSP = 14, - TAOS_SMSG_TEST = 15, - TAOS_SMSG_END = 16 -} ESyncMsgType; - -typedef enum { - SYNC_STATUS_BROADCAST, - SYNC_STATUS_BROADCAST_RSP, - SYNC_STATUS_SETUP_CONN, - SYNC_STATUS_SETUP_CONN_RSP, - SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP, - SYNC_STATUS_CHECK_ROLE, - SYNC_STATUS_CHECK_ROLE_RSP -} ESyncStatusType; - -#pragma pack(push, 1) - -typedef struct { - int8_t type; // msg type - int8_t protocol; // protocol version - uint16_t signature; // fixed value - int32_t code; // - int32_t cId; // cluster Id - int32_t vgId; // vg ID - int32_t len; // content length, does not include head - uint32_t cksum; -} SSyncHead; - -typedef struct { - SSyncHead head; - uint16_t port; - uint16_t tranId; - int32_t sourceId; // only for arbitrator - char fqdn[TSDB_FQDN_LEN]; -} SSyncMsg; - -typedef struct { - SSyncHead head; - int8_t sync; - int8_t reserved; - uint16_t tranId; - int8_t reserverd[4]; -} SSyncRsp; - -typedef struct { - int8_t role; - uint64_t version; -} SPeerStatus; - -typedef struct { - SSyncHead head; - int8_t role; - int8_t ack; - int8_t type; - int8_t reserved[3]; - uint16_t tranId; - uint64_t version; - SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; -} SPeersStatus; - -typedef struct { - SSyncHead head; - uint64_t fversion; -} SFileVersion; - -typedef struct { - SSyncHead head; - int8_t ack; -} SFileAck; - -typedef struct { - SSyncHead head; - uint64_t version; - int32_t code; -} SFwdRsp; - -#pragma pack(pop) - -#define SYNC_PROTOCOL_VERSION 1 -#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) - -extern char *statusType[]; - -uint16_t syncGenTranId(); -int32_t syncCheckHead(SSyncHead *pHead); - -void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); -void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); -void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); -void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); - -void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); -void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_VNODEPEER_H diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h deleted file mode 100644 index e2e5234d3996608a6ed9ce952fcb08aabca7e8b5..0000000000000000000000000000000000000000 --- a/src/sync/inc/syncTcp.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_TCP_POOL_H -#define TDENGINE_SYNC_TCP_POOL_H - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t numOfThreads; - uint32_t serverIp; - int16_t port; - int32_t bufferSize; - void (*processBrokenLink)(int64_t handleId, int32_t closedByApp); - int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(SOCKET fd, uint32_t ip); -} SPoolInfo; - -void *syncOpenTcpThreadPool(SPoolInfo *pInfo); -void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); -void syncFreeTcpConn(void *); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TCP_POOL_H - diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c deleted file mode 100644 index 08ebe7ae3e14dade398c793e3489508e21b58f20..0000000000000000000000000000000000000000 --- a/src/sync/src/syncArbitrator.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include "os.h" -#include "hash.h" -#include "tlog.h" -#include "tutil.h" -#include "ttimer.h" -#include "tsocket.h" -#include "tglobal.h" -#include "taoserror.h" -#include "twal.h" -#include "tsync.h" -#include "syncInt.h" -#include "syncTcp.h" - -extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd); -static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp); -static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); -static tsem_t tsArbSem; -static void * tsArbTcpPool; - -typedef struct { - char id[TSDB_EP_LEN + 24]; - SOCKET nodeFd; - void * pConn; -} SNodeConn; - -int32_t main(int32_t argc, char *argv[]) { - char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; - - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - tsArbitratorPort = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - debugFlag = atoi(argv[++i]); - } else if (strcmp(argv[i], "-g") == 0 && i < argc - 1) { - if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue; - tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath)); - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-p port]: arbitrator server port number, default is:%d\n", tsServerPort + TSDB_PORT_ARBITRATOR); - printf(" [-d debugFlag]: debug flag, option 131 | 135 | 143, default:0\n"); - printf(" [-g logFilePath]: log file pathe, default:/arbitrator.log\n"); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - sDebugFlag = debugFlag; - - if (tsem_init(&tsArbSem, 0, 0) != 0) { - printf("failed to create exit semphore\n"); - exit(EXIT_FAILURE); - } - - /* Set termination handler. */ - taosSetSignal(SIGTERM, arbSignalHandler); - taosSetSignal(SIGINT, arbSignalHandler); - taosSetSignal(SIGHUP, arbSignalHandler); - taosSetSignal(SIGABRT, arbSignalHandler); - - tsAsyncLog = 0; - strcat(arbLogPath, "/arbitrator.log"); - taosInitLog(arbLogPath, 1000000, 10); - - taosGetFqdn(tsNodeFqdn); - - SPoolInfo info; - info.numOfThreads = 1; - info.serverIp = 0; - info.port = tsArbitratorPort; - info.bufferSize = SYNC_MAX_SIZE; - info.processBrokenLink = arbProcessBrokenLink; - info.processIncomingMsg = arbProcessPeerMsg; - info.processIncomingConn = arbProcessIncommingConnection; - tsArbTcpPool = syncOpenTcpThreadPool(&info); - - if (tsArbTcpPool == NULL) { - sDebug("failed to open TCP thread pool, exit..."); - return -1; - } - - sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); - - tsem_wait(&tsArbSem); - - syncCloseTcpThreadPool(tsArbTcpPool); - sInfo("TAOS arbitrator is shut down"); - - closelog(); - return 0; -} - -static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { - char ipstr[24]; - tinet_ntoa(ipstr, sourceIp); - sDebug("peer TCP connection from ip:%s", ipstr); - - SSyncMsg msg; - if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); - taosCloseSocket(connFd); - return; - } - - if (msg.head.type == TAOS_SMSG_TEST) { - syncProcessTestMsg(&msg, connFd); - return; - } - - SNodeConn *pNode = calloc(sizeof(SNodeConn), 1); - if (pNode == NULL) { - sError("failed to allocate memory since %s", strerror(errno)); - taosCloseSocket(connFd); - return; - } - - msg.fqdn[TSDB_FQDN_LEN - 1] = 0; - snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", msg.sourceId, msg.fqdn, msg.port); - if (msg.head.vgId) { - sDebug("%s, vgId in head is not zero, close the connection", pNode->id); - tfree(pNode); - taosCloseSocket(connFd); - return; - } - - sDebug("%s, arbitrator request is accepted", pNode->id); - pNode->nodeFd = connFd; - pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd); - - return; -} - -static void arbProcessBrokenLink(int64_t rid, int32_t closedByApp) { - SNodeConn *pNode = (SNodeConn *)rid; - - sDebug("%s, TCP link is broken since %s, closedByApp:%d", pNode->id, strerror(errno), closedByApp); - tfree(pNode); -} - -static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { - SNodeConn *pNode = (SNodeConn *)rid; - SSyncHead head; - int32_t bytes = 0; - char * cont = (char *)buffer; - - int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(SSyncHead)); - if (hlen != sizeof(SSyncHead)) { - sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen); - return -1; - } - - bytes = taosReadMsg(pNode->nodeFd, cont, head.len); - if (bytes != head.len) { - sDebug("%s, failed to read, bytes:%d len:%d", pNode->id, bytes, head.len); - return -1; - } - - sDebug("%s, msg is received, len:%d", pNode->id, head.len); - return 0; -} - -static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) { - taosIgnSignal(SIGTERM); - taosIgnSignal(SIGINT); - taosIgnSignal(SIGABRT); - taosIgnSignal(SIGHUP); - - sInfo("shut down signal is %d", signum); - - // inform main thread to exit - tsem_post(&tsArbSem); -} diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c deleted file mode 100644 index 68bafb09ca82a7a55b8eb3cd94c24138f5ef3a6c..0000000000000000000000000000000000000000 --- a/src/sync/src/syncMain.c +++ /dev/null @@ -1,1519 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "hash.h" -#include "tlog.h" -#include "tutil.h" -#include "ttimer.h" -#include "tref.h" -#include "tsocket.h" -#include "tglobal.h" -#include "taoserror.h" -#include "tqueue.h" -#include "twal.h" -#include "tsync.h" -#include "syncTcp.h" -#include "syncInt.h" - -int32_t tsSyncNum = 0; // number of sync in process in whole system -char tsNodeFqdn[TSDB_FQDN_LEN] = {0}; - -static void * tsTcpPool = NULL; -static void * tsSyncTmrCtrl = NULL; -static void * tsVgIdHash = NULL; -static int32_t tsNodeRefId = -1; -static int32_t tsPeerRefId = -1; - -// local functions -static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); -static void syncRecoverFromMaster(SSyncPeer *pPeer); -static void syncCheckPeerConnection(void *param, void *tmrId); -static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(int64_t rid, int32_t closedByApp); -static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); -static void syncRemovePeer(SSyncPeer *pPeer); -static void syncAddArbitrator(SSyncNode *pNode); -static void syncFreeNode(void *); -static void syncFreePeer(void *); -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); -static void syncMonitorFwdInfos(void *param, void *tmrId); -static void syncMonitorNodeRole(void *param, void *tmrId); -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); -static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); -static void syncRestartPeer(SSyncPeer *pPeer); -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype, bool force); - -static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); -static void syncStartCheckPeerConn(SSyncPeer *pPeer); -static void syncStopCheckPeerConn(SSyncPeer *pPeer); -static SSyncNode *syncAcquireNode(int64_t rid); -static void syncReleaseNode(SSyncNode *pNode); - -char* syncRole[] = { - "offline", - "unsynced", - "syncing", - "slave", - "master" -}; - -char *syncStatus[] = { - "init", - "start", - "file", - "cache", - "invalid" -}; - -int32_t syncInit() { - SPoolInfo info = {0}; - - info.numOfThreads = SYNC_TCP_THREADS; - info.serverIp = 0; - info.port = tsSyncPort; - info.bufferSize = SYNC_MAX_SIZE; - info.processBrokenLink = syncProcessBrokenLink; - info.processIncomingMsg = syncProcessPeerMsg; - info.processIncomingConn = syncProcessIncommingConnection; - - tsTcpPool = syncOpenTcpThreadPool(&info); - if (tsTcpPool == NULL) { - sError("failed to init tcpPool"); - syncCleanUp(); - return -1; - } - - tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); - if (tsSyncTmrCtrl == NULL) { - sError("failed to init tmrCtrl"); - syncCleanUp(); - return -1; - } - - tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (tsVgIdHash == NULL) { - sError("failed to init vgIdHash"); - syncCleanUp(); - return -1; - } - - tsNodeRefId = taosOpenRef(200, syncFreeNode); - if (tsNodeRefId < 0) { - sError("failed to init node ref"); - syncCleanUp(); - return -1; - } - - tsPeerRefId = taosOpenRef(1000, syncFreePeer); - if (tsPeerRefId < 0) { - sError("failed to init peer ref"); - syncCleanUp(); - return -1; - } - - tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn)); - sInfo("sync module initialized successfully"); - - return 0; -} - -void syncCleanUp() { - if (tsTcpPool != NULL) { - syncCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; - } - - if (tsSyncTmrCtrl != NULL) { - taosTmrCleanUp(tsSyncTmrCtrl); - tsSyncTmrCtrl = NULL; - } - - if (tsVgIdHash) { - taosHashCleanup(tsVgIdHash); - tsVgIdHash = NULL; - } - - if (tsNodeRefId != -1) { - taosCloseRef(tsNodeRefId); - tsNodeRefId = -1; - } - - if (tsPeerRefId != -1) { - taosCloseRef(tsPeerRefId); - tsPeerRefId = -1; - } - - sInfo("sync module is cleaned up"); -} - -int64_t syncStart(const SSyncInfo *pInfo) { - const SSyncCfg *pCfg = &pInfo->syncCfg; - - SSyncNode *pNode = calloc(sizeof(SSyncNode), 1); - if (pNode == NULL) { - sError("no memory to allocate syncNode"); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); - pthread_mutex_init(&pNode->mutex, NULL); - - pNode->getWalInfoFp = pInfo->getWalInfoFp; - pNode->writeToCacheFp = pInfo->writeToCacheFp; - pNode->notifyRoleFp = pInfo->notifyRoleFp; - pNode->confirmForward = pInfo->confirmForward; - pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp; - pNode->startSyncFileFp = pInfo->startSyncFileFp; - pNode->stopSyncFileFp = pInfo->stopSyncFileFp; - pNode->getVersionFp = pInfo->getVersionFp; - pNode->sendFileFp = pInfo->sendFileFp; - pNode->recvFileFp = pInfo->recvFileFp; - - pNode->selfIndex = -1; - pNode->vgId = pInfo->vgId; - pNode->replica = pCfg->replica; - pNode->quorum = pCfg->quorum; - pNode->pTsdb = pInfo->pTsdb; - if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - - pNode->refCount = 1; - pNode->rid = taosAddRef(tsNodeRefId, pNode); - if (pNode->rid < 0) { - syncFreeNode(pNode); - return -1; - } - - for (int32_t index = 0; index < pCfg->replica; ++index) { - const SNodeInfo *pNodeInfo = pCfg->nodeInfo + index; - pNode->peerInfo[index] = syncAddPeer(pNode, pNodeInfo); - if (pNode->peerInfo[index] == NULL) { - sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, - pNodeInfo->nodeFqdn, pNodeInfo->nodePort); - syncStop(pNode->rid); - exit(1); - } - - if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { - pNode->selfIndex = index; - } - } - - if (pNode->selfIndex < 0) { - sError("vgId:%d, this node is not configured", pNode->vgId); - terrno = TSDB_CODE_SYN_INVALID_CONFIG; - syncStop(pNode->rid); - return -1; - } - - nodeVersion = pInfo->version; // set the initial version - nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER; - sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, - syncRole[nodeRole]); - - pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + SYNC_MAX_FWDS * sizeof(SFwdInfo), 1); - if (pNode->pSyncFwds == NULL) { - sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); - terrno = TAOS_SYSTEM_ERROR(errno); - syncStop(pNode->rid); - return -1; - } - - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - if (pNode->pFwdTimer == NULL) { - sError("vgId:%d, failed to allocate fwd timer", pNode->vgId); - syncStop(pNode->rid); - return -1; - } - - pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - if (pNode->pRoleTimer == NULL) { - sError("vgId:%d, failed to allocate role timer", pNode->vgId); - syncStop(pNode->rid); - return -1; - } - - syncAddArbitrator(pNode); - taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *)); - - if (pNode->notifyRoleFp) { - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } - - syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStartCheckPeerConn(pNode->peerInfo[index]); - } - - return pNode->rid; -} - -void syncStop(int64_t rid) { - SSyncPeer *pPeer; - - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return; - - sInfo("vgId:%d, cleanup sync", pNode->vgId); - - pthread_mutex_lock(&pNode->mutex); - - if (tsVgIdHash) taosHashRemove(tsVgIdHash, &pNode->vgId, sizeof(int32_t)); - if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); - if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer); - - for (int32_t index = 0; index < pNode->replica; ++index) { - pPeer = pNode->peerInfo[index]; - if (pPeer) syncRemovePeer(pPeer); - } - - pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - if (pPeer) syncRemovePeer(pPeer); - - pthread_mutex_unlock(&pNode->mutex); - - syncReleaseNode(pNode); - taosRemoveRef(tsNodeRefId, rid); -} - -int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { - int32_t i, j; - - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; - - sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, - pNode->replica); - - pthread_mutex_lock(&pNode->mutex); - - syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStopCheckPeerConn(pNode->peerInfo[index]); - } - - for (i = 0; i < pNode->replica; ++i) { - for (j = 0; j < pNewCfg->replica; ++j) { - if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && - (pNode->peerInfo[i]->port == pNewCfg->nodeInfo[j].nodePort)) - break; - } - - if (j >= pNewCfg->replica) { - syncRemovePeer(pNode->peerInfo[i]); - pNode->peerInfo[i] = NULL; - } - } - - SSyncPeer *newPeers[TAOS_SYNC_MAX_REPLICA]; - for (i = 0; i < pNewCfg->replica; ++i) { - const SNodeInfo *pNewNode = &pNewCfg->nodeInfo[i]; - - for (j = 0; j < pNode->replica; ++j) { - if (pNode->peerInfo[j] && (strcmp(pNode->peerInfo[j]->fqdn, pNewNode->nodeFqdn) == 0) && - (pNode->peerInfo[j]->port == pNewNode->nodePort)) - break; - } - - if (j >= pNode->replica) { - newPeers[i] = syncAddPeer(pNode, pNewNode); - } else { - newPeers[i] = pNode->peerInfo[j]; - } - - if (newPeers[i] == NULL) { - sError("vgId:%d, failed to reconfig", pNode->vgId); - return TSDB_CODE_SYN_INVALID_CONFIG; - } - - if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) { - pNode->selfIndex = i; - } - } - - pNode->replica = pNewCfg->replica; - pNode->quorum = pNewCfg->quorum; - if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - memcpy(pNode->peerInfo, newPeers, sizeof(SSyncPeer *) * pNewCfg->replica); - - for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i) { - pNode->peerInfo[i] = NULL; - } - - syncAddArbitrator(pNode); - - if (pNewCfg->replica <= 1) { - sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId); - nodeRole = TAOS_SYNC_ROLE_MASTER; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } - - syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb - for (int32_t index = 0; index < pNode->replica; ++index) { - syncStartCheckPeerConn(pNode->peerInfo[index]); - } - - pthread_mutex_unlock(&pNode->mutex); - - sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); - syncBroadcastStatus(pNode); - - syncReleaseNode(pNode); - return 0; -} - -int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype, bool force) { - if (rid <= 0) return 0; - - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return 0; - - int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype, force); - - syncReleaseNode(pNode); - return code; -} - -void syncConfirmForward(int64_t rid, uint64_t _version, int32_t code, bool force) { - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return; - - SSyncPeer *pPeer = pNode->pMaster; - if (pPeer && (pNode->quorum > 1 || force)) { - SFwdRsp rsp; - syncBuildSyncFwdRsp(&rsp, pNode->vgId, _version, code); - - if (taosWriteMsg(pPeer->peerFd, &rsp, sizeof(SFwdRsp)) == sizeof(SFwdRsp)) { - sTrace("%s, forward-rsp is sent, code:0x%x hver:%" PRIu64, pPeer->id, code, _version); - } else { - sDebug("%s, failed to send forward-rsp, restart", pPeer->id); - syncRestartConnection(pPeer); - } - } - - syncReleaseNode(pNode); -} - -void syncRecover(int64_t rid) { - SSyncPeer *pPeer; - - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return; - - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - - pthread_mutex_lock(&pNode->mutex); - - nodeVersion = 0; - - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == pNode->selfIndex) continue; - - pPeer = pNode->peerInfo[i]; - if (pPeer->peerFd >= 0) { - syncRestartConnection(pPeer); - } - } - - pthread_mutex_unlock(&pNode->mutex); - - syncReleaseNode(pNode); -} - -int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return -1; - - pNodesRole->selfIndex = pNode->selfIndex; - for (int32_t i = 0; i < pNode->replica; ++i) { - pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; - pNodesRole->role[i] = pNode->peerInfo[i]->role; - } - - syncReleaseNode(pNode); - return 0; -} - -static void syncAddArbitrator(SSyncNode *pNode) { - SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - - // if not configured, return right away - if (tsArbitrator[0] == 0) { - if (pPeer) syncRemovePeer(pPeer); - pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL; - return; - } - - SNodeInfo nodeInfo; - nodeInfo.nodeId = 0; - int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); - if (-1 == ret) { - nodeInfo.nodePort = tsArbitratorPort; - } - - if (pPeer) { - if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) { - return; - } else { - syncRemovePeer(pPeer); - pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL; - } - } - - pPeer = syncAddPeer(pNode, &nodeInfo); - if (pPeer != NULL) { - pPeer->isArb = 1; - sInfo("%s, is added as arbitrator", pPeer->id); - } - - pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = pPeer; -} - -static void syncFreeNode(void *param) { - SSyncNode *pNode = param; - sDebug("vgId:%d, node is freed, refCount:%d", pNode->vgId, pNode->refCount); - - pthread_mutex_destroy(&pNode->mutex); - tfree(pNode->pRecv); - tfree(pNode->pSyncFwds); - tfree(pNode); -} - -SSyncNode *syncAcquireNode(int64_t rid) { - SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid); - if (pNode == NULL) { - sDebug("failed to acquire node from refId:%" PRId64, rid); - } else { - int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1); - sTrace("vgId:%d, acquire node refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount); - } - - return pNode; -} - -void syncReleaseNode(SSyncNode *pNode) { - int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); - sTrace("vgId:%d, release node refId:%" PRId64 ", refCount:%d", pNode->vgId, pNode->rid, refCount); - - taosReleaseRef(tsNodeRefId, pNode->rid); -} - -static void syncFreePeer(void *param) { - SSyncPeer *pPeer = param; - sDebug("%s, peer is freed, refCount:%d", pPeer->id, pPeer->refCount); - - syncReleaseNode(pPeer->pSyncNode); - tfree(pPeer); -} - -SSyncPeer *syncAcquirePeer(int64_t rid) { - SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid); - if (pPeer == NULL) { - sDebug("failed to acquire peer from refId:%" PRId64, rid); - } else { - int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1); - sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount); - } - - return pPeer; -} - -void syncReleasePeer(SSyncPeer *pPeer) { - int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); - sTrace("%s, release peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount); - - taosReleaseRef(tsPeerRefId, pPeer->rid); -} - -static void syncClosePeerConn(SSyncPeer *pPeer) { - sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); - - taosTmrStopA(&pPeer->timer); - taosCloseSocket(pPeer->syncFd); - if (pPeer->peerFd >= 0) { - pPeer->peerFd = -1; - void *pConn = pPeer->pConn; - if (pConn != NULL) { - syncFreeTcpConn(pPeer->pConn); - pPeer->pConn = NULL; - } - } -} - -static void syncRemovePeer(SSyncPeer *pPeer) { - sInfo("%s, it is removed", pPeer->id); - - //pPeer->ip = 0; - pPeer->fqdn[0] = '\0'; - syncClosePeerConn(pPeer); - //taosRemoveRef(tsPeerRefId, pPeer->rid); - syncReleasePeer(pPeer); -} - -static void syncStartCheckPeerConn(SSyncPeer *pPeer) { - if (pPeer == NULL) return; - SSyncNode *pNode = pPeer->pSyncNode; - - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { - int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - - sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - } -} - -static void syncStopCheckPeerConn(SSyncPeer *pPeer) { - if (pPeer == NULL) return; - - taosTmrStopA(&pPeer->timer); - sDebug("%s, stop check peer connection", pPeer->id); -} - -uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer) { - uint32_t ip = taosGetIpv4FromFqdn(pPeer->fqdn); - if (ip == 0xFFFFFFFF) { - sError("failed to resolve peer fqdn:%s since %s", pPeer->fqdn, strerror(errno)); - terrno = TSDB_CODE_RPC_FQDN_ERROR; - return 0; - } - - return ip; -} - -static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - /*uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); - if (ip == 0xFFFFFFFF) { - sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); - terrno = TSDB_CODE_RPC_FQDN_ERROR; - return NULL; - } - */ - SSyncPeer *pPeer = calloc(1, sizeof(SSyncPeer)); - if (pPeer == NULL) return NULL; - - pPeer->nodeId = pInfo->nodeId; - tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn)); - //pPeer->ip = ip; - pPeer->port = pInfo->nodePort; - pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; - snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId); - - pPeer->peerFd = -1; - pPeer->syncFd = -1; - pPeer->role = TAOS_SYNC_ROLE_OFFLINE; - pPeer->pSyncNode = pNode; - pPeer->refCount = 1; - pPeer->rid = taosAddRef(tsPeerRefId, pPeer); - - sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid); - - (void)syncAcquireNode(pNode->rid); - return pPeer; -} - -void syncBroadcastStatus(SSyncNode *pNode) { - for (int32_t index = 0; index < pNode->replica; ++index) { - if (index == pNode->selfIndex) continue; - SSyncPeer *pPeer = pNode->peerInfo[index]; - syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId()); - } -} - -static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int32_t index = 0; index < pNode->replica; ++index) { - pNode->peerInfo[index]->numOfRetrieves = 0; - } - - if (pNode->notifyFlowCtrlFp) { - (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); - } -} - -static void syncChooseMaster(SSyncNode *pNode) { - SSyncPeer *pPeer; - int32_t onlineNum = 0; - int32_t index = -1; - int32_t replica = pNode->replica; - - for (int32_t i = 0; i < pNode->replica; ++i) { - if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - } - } - - if (onlineNum == pNode->replica) { - // if all peers are online, peer with highest version shall be master - index = 0; - for (int32_t i = 1; i < pNode->replica; ++i) { - if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) { - index = i; - } - } - sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); - } else { - sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); - } - - // add arbitrator connection - SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - if (pArb) { - if (pArb->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - replica = pNode->replica + 1; - sDebug("vgId:%d, arb:%s is used while choose master", pNode->vgId, pArb->id); - } else { - sError("vgId:%d, arb:%s is not used while choose master for its offline", pNode->vgId, pArb->id); - } - } - - if (index < 0 && onlineNum > replica / 2.0) { - // over half of nodes are online - for (int32_t i = 0; i < pNode->replica; ++i) { - // slave with highest version shall be master - pPeer = pNode->peerInfo[i]; - if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) { - if (index < 0 || pPeer->version > pNode->peerInfo[index]->version) { - index = i; - } - } - } - - if (index >= 0) { - sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, - pNode->peerInfo[index]->id, index, onlineNum, replica); - } - } - - if (index >= 0) { - if (index == pNode->selfIndex) { - sInfo("vgId:%d, start to work as master", pNode->vgId); - nodeRole = TAOS_SYNC_ROLE_MASTER; - - // Wait for other nodes to receive status to avoid version inconsistency - taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER); - - syncResetFlowCtrl(pNode); - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } else { - pPeer = pNode->peerInfo[index]; - sInfo("%s, it shall work as master", pPeer->id); - } - } else { - sDebug("vgId:%d, failed to choose master", pNode->vgId); - } -} - -static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { - int32_t onlineNum = 0, arbOnlineNum = 0; - int32_t masterIndex = -1; - int32_t replica = pNode->replica; - - for (int32_t index = 0; index < pNode->replica; ++index) { - if (pNode->peerInfo[index]->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - } - } - - // add arbitrator connection - SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; - if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) { - onlineNum++; - ++arbOnlineNum; - replica = pNode->replica + 1; - } - - if (onlineNum <= replica * 0.5) { - if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && ((replica > 2 && onlineNum - arbOnlineNum > 1) || pNode->replica < 3)) { - sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); - masterIndex = pNode->selfIndex; - } else { - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); - } - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } - } else { - for (int32_t index = 0; index < pNode->replica; ++index) { - SSyncPeer *pTemp = pNode->peerInfo[index]; - if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; - if (masterIndex < 0) { - masterIndex = index; - sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, index); - } else { // multiple masters, it shall not happen - if (masterIndex == pNode->selfIndex) { - sError("%s, peer is master, work as slave instead", pTemp->id); - nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } else { - sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, index); - } - } - } - } - - SSyncPeer *pMaster = (masterIndex >= 0) ? pNode->peerInfo[masterIndex] : NULL; - return pMaster; -} - -static int32_t syncValidateMaster(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - int32_t code = 0; - - if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { - sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - code = -1; - - for (int32_t index = 0; index < pNode->replica; ++index) { - if (index == pNode->selfIndex) continue; - syncRestartPeer(pNode->peerInfo[index]); - } - } - - return code; -} - -static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t newPeerRole) { - SSyncNode *pNode = pPeer->pSyncNode; - int8_t oldPeerRole = pPeer->role; - int8_t oldSelfRole = nodeRole; - int8_t syncRequired = 0; - - pPeer->role = newPeerRole; - sDebug("%s, peer role:%s change to %s", pPeer->id, syncRole[oldPeerRole], syncRole[newPeerRole]); - - SSyncPeer *pMaster = syncCheckMaster(pNode); - - if (pMaster) { - // master is there - pNode->pMaster = pMaster; - sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version); - - if (syncValidateMaster(pPeer) < 0) return; - - if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { - if (nodeVersion < pMaster->version) { - sDebug("%s, is master, sync required, self sver:%" PRIu64, pMaster->id, nodeVersion); - syncRequired = 1; - } else { - sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); - nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - } - } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { - sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); - } - } else { - // master not there, if all peer's state and version are consistent, choose the master - int32_t consistent = 0; - int32_t index = 0; - if (peersStatus != NULL) { - for (index = 0; index < pNode->replica; ++index) { - SSyncPeer *pTemp = pNode->peerInfo[index]; - if (pTemp->role != peersStatus[index].role) break; - if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[index].version)) break; - } - - if (index >= pNode->replica) consistent = 1; - } else { - if (pNode->replica == 2) consistent = 1; - } - - if (consistent) { - sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica); - syncChooseMaster(pNode); - } else { - sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica); - } - } - - if (syncRequired) { - syncRecoverFromMaster(pMaster); - } - - if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) { - sDebug("vgId:%d, roles changed, broadcast status, replica:%d", pNode->vgId, pNode->replica); - syncBroadcastStatus(pNode); - } - - if (nodeRole != TAOS_SYNC_ROLE_MASTER) { - syncResetFlowCtrl(pNode); - } -} - -static void syncRestartPeer(SSyncPeer *pPeer) { - sDebug("%s, restart peer connection, last sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - - syncClosePeerConn(pPeer); - - pPeer->sstatus = TAOS_SYNC_STATUS_INIT; - sDebug("%s, peer conn is restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { - sDebug("%s, check peer connection in 1000 ms", pPeer->id); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - } -} - -void syncRestartConnection(SSyncPeer *pPeer) { - if (pPeer->fqdn[0] == '\0') return; - - if (syncAcquirePeer(pPeer->rid) == NULL) return; - - syncRestartPeer(pPeer); - syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); - - syncReleasePeer(pPeer); -} - -static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - sInfo("%s, sync-req is received", pPeer->id); - - if (pPeer->fqdn[0] == '\0') return; - - if (nodeRole != TAOS_SYNC_ROLE_MASTER) { - sError("%s, I am not master anymore", pPeer->id); - taosCloseSocket(pPeer->syncFd); - return; - } - - if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started for sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - return; // already started - } - - // start a new thread to retrieve the data - (void)syncAcquirePeer(pPeer->rid); - - pthread_attr_t thattr; - pthread_t thread; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid); - pthread_attr_destroy(&thattr); - - if (ret < 0) { - sError("%s, failed to create sync retrieve thread since %s", pPeer->id, strerror(errno)); - syncReleasePeer(pPeer); - } else { - pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, sync retrieve thread:0x%08" PRIx64 " create successfully, rid:%" PRId64 ", set sstatus:%s", pPeer->id, - taosGetPthreadId(thread), pPeer->rid, syncStatus[pPeer->sstatus]); - } -} - -static void syncNotStarted(void *param, void *tmrId) { - int64_t rid = (int64_t)param; - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) return; - - SSyncNode *pNode = pPeer->pSyncNode; - - pthread_mutex_lock(&pNode->mutex); - pPeer->timer = NULL; - pPeer->sstatus = TAOS_SYNC_STATUS_INIT; - sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - syncRestartConnection(pPeer); - pthread_mutex_unlock(&pNode->mutex); - - syncReleasePeer(pPeer); -} - -static void syncTryRecoverFromMaster(void *param, void *tmrId) { - int64_t rid = (int64_t)param; - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) return; - - SSyncNode *pNode = pPeer->pSyncNode; - - pthread_mutex_lock(&pNode->mutex); - syncRecoverFromMaster(pPeer); - pthread_mutex_unlock(&pNode->mutex); - - syncReleasePeer(pPeer); -} - -static void syncRecoverFromMaster(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started since sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - return; - } - - taosTmrStopA(&pPeer->timer); - - // Ensure the sync of mnode not interrupted - if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) { - sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - return; - } - - sDebug("%s, try to sync", pPeer->id); - - SSyncMsg msg; - syncBuildSyncReqMsg(&msg, pNode->vgId); - - taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - - if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - sError("%s, failed to send sync-req to peer", pPeer->id); - } else { - sInfo("%s, sync-req is sent to peer, tranId:%u, sstatus:%s", pPeer->id, msg.tranId, syncStatus[nodeSStatus]); - } -} - -static void syncProcessFwdResponse(SFwdRsp *pFwdRsp, SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - SSyncFwds *pSyncFwds = pNode->pSyncFwds; - SFwdInfo * pFwdInfo; - - sTrace("%s, forward-rsp is received, code:%x hver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version); - SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first; - - if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { - // find the forwardInfo from first - for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % SYNC_MAX_FWDS; - if (pFwdRsp->version == pFwdInfo->version) { - syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code); - syncRemoveConfirmedFwdInfo(pNode); - return; - } - } - } -} - -static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - SWalHead * pHead = (SWalHead *)(cont + sizeof(SSyncHead)); - - sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); - - int32_t code = 0; - if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { - // nodeVersion = pHead->version; - code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); - syncConfirmForward(pNode->rid, pHead->version, code, false); - } else { - if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { - code = syncSaveIntoBuffer(pPeer, pHead); - } else { - sError("%s, forward discarded since sstatus:%s, hver:%" PRIu64, pPeer->id, syncStatus[nodeSStatus], - pHead->version); - code = -1; - } - } - - if (code != 0) { - sError("%s, failed to process fwd msg, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); - syncRestartConnection(pPeer); - } -} - -static void syncProcessPeersStatusMsg(SPeersStatus *pPeersStatus, SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", - pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], - pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); - - pPeer->version = pPeersStatus->version; - syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role); - - if (pPeersStatus->ack) { - syncSendPeersStatusMsgToPeer(pPeer, 0, pPeersStatus->type + 1, pPeersStatus->tranId); - } -} - -static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) { - if (pPeer->peerFd < 0) return -1; - - int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); - if (hlen != sizeof(SSyncHead)) { - sDebug("%s, failed to read msg since %s, hlen:%d", pPeer->id, tstrerror(errno), hlen); - return -1; - } - - int32_t code = syncCheckHead(pHead); - if (code != 0) { - sError("%s, failed to check msg head since %s, type:%d", pPeer->id, tstrerror(code), pHead->type); - return -1; - } - - int32_t bytes = taosReadMsg(pPeer->peerFd, (char *)pHead + sizeof(SSyncHead), pHead->len); - if (bytes != pHead->len) { - sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); - return -1; - } - - return 0; -} - -static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) return -1; - - SSyncHead *pHead = buffer; - SSyncNode *pNode = pPeer->pSyncNode; - - pthread_mutex_lock(&pNode->mutex); - - int32_t code = syncReadPeerMsg(pPeer, pHead); - - if (code == 0) { - if (pHead->type == TAOS_SMSG_SYNC_FWD) { - syncProcessForwardFromPeer(buffer, pPeer); - } else if (pHead->type == TAOS_SMSG_SYNC_FWD_RSP) { - syncProcessFwdResponse(buffer, pPeer); - } else if (pHead->type == TAOS_SMSG_SYNC_REQ) { - syncProcessSyncRequest(buffer, pPeer); - } else if (pHead->type == TAOS_SMSG_STATUS) { - syncProcessPeersStatusMsg(buffer, pPeer); - } - } - - pthread_mutex_unlock(&pNode->mutex); - syncReleasePeer(pPeer); - - return code; -} - -static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - if (pPeer->peerFd < 0 || pPeer->fqdn[0] == '\0') { - sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); - syncRestartConnection(pPeer); - return -1; - } - - SSyncNode * pNode = pPeer->pSyncNode; - SPeersStatus msg; - - memset(&msg, 0, sizeof(SPeersStatus)); - syncBuildPeersStatus(&msg, pNode->vgId); - - msg.role = nodeRole; - msg.ack = ack; - msg.type = type; - msg.tranId = tranId; - msg.version = nodeVersion; - - for (int32_t i = 0; i < pNode->replica; ++i) { - msg.peersStatus[i].role = pNode->peerInfo[i]->role; - msg.peersStatus[i].version = pNode->peerInfo[i]->version; - } - - if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SPeersStatus)) == sizeof(SPeersStatus)) { - sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", - pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd); - return 0; - } else { - sDebug("%s, failed to send status msg, restart", pPeer->id); - syncRestartConnection(pPeer); - return -1; - } -} - -static void syncSetupPeerConnection(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - taosTmrStopA(&pPeer->timer); - if (pPeer->peerFd >= 0) { - sDebug("%s, send role version to peer", pPeer->id); - syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_SETUP_CONN, syncGenTranId()); - return; - } - - uint32_t ip = syncResolvePeerFqdn(pPeer); - if (!ip) { - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - return; - } - - SOCKET connFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); - if (connFd <= 0) { - sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - return; - } - - SSyncMsg msg; - syncBuildSyncSetupMsg(&msg, pPeer->nodeId ? pNode->vgId : 0); - - if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) == sizeof(SSyncMsg)) { - sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId); - pPeer->peerFd = connFd; - pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); - if (pPeer->isArb) { - tsArbOnline = 1; - if (tsArbOnlineTimestamp == TSDB_ARB_DUMMY_TIME) { - tsArbOnlineTimestamp = taosGetTimestampMs(); - } - } - } else { - sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); - taosCloseSocket(connFd); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); - } -} - -static void syncCheckPeerConnection(void *param, void *tmrId) { - int64_t rid = (int64_t)param; - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) return; - - SSyncNode *pNode = pPeer->pSyncNode; - - pthread_mutex_lock(&pNode->mutex); - - sDebug("%s, check peer connection", pPeer->id); - syncSetupPeerConnection(pPeer); - - pthread_mutex_unlock(&pNode->mutex); - - syncReleasePeer(pPeer); -} - -static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { - taosTmrStopA(&pPeer->timer); - - pthread_attr_t thattr; - pthread_t thread; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - - (void)syncAcquirePeer(pPeer->rid); - - int32_t ret = pthread_create(&thread, &thattr, (void *)syncRestoreData, (void *)pPeer->rid); - pthread_attr_destroy(&thattr); - - if (ret < 0) { - SSyncNode *pNode = pPeer->pSyncNode; - nodeSStatus = TAOS_SYNC_STATUS_INIT; - sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosCloseSocket(pPeer->syncFd); - syncReleasePeer(pPeer); - } else { - sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, - taosGetPthreadId(thread), pPeer->rid); - } -} - -void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { - sInfo("recv sync test msg"); - - SSyncMsg rsp; - syncBuildSyncTestMsg(&rsp, -1); - if (taosWriteMsg(connFd, &rsp, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - sInfo("failed to send sync test rsp since %s", strerror(errno)); - } - - sInfo("send sync test rsp"); - taosMsleep(1000); - taosCloseSocket(connFd); -} - -static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { - char ipstr[24]; - int32_t i; - - tinet_ntoa(ipstr, sourceIp); - sDebug("peer TCP connection from ip:%s", ipstr); - - SSyncMsg msg; - if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); - taosCloseSocket(connFd); - return; - } - - int32_t code = syncCheckHead((SSyncHead *)(&msg)); - if (code != 0) { - sError("failed to check peer sync msg from ip:%s since %s", ipstr, strerror(code)); - taosCloseSocket(connFd); - return; - } - - if (msg.head.type == TAOS_SMSG_TEST) { - syncProcessTestMsg(&msg, connFd); - return; - } - - int32_t vgId = msg.head.vgId; - SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); - if (ppNode == NULL || *ppNode == NULL) { - sError("vgId:%d, vgId could not be found", vgId); - taosCloseSocket(connFd); - return; - } - - sDebug("vgId:%d, sync connection is incoming, tranId:%u", vgId, msg.tranId); - - SSyncNode *pNode = *ppNode; - pthread_mutex_lock(&pNode->mutex); - - SSyncPeer *pPeer; - for (i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - if (pPeer && (strcmp(pPeer->fqdn, msg.fqdn) == 0) && (pPeer->port == msg.port)) break; - } - - pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL; - if (pPeer == NULL) { - sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, msg.fqdn, msg.port); - taosCloseSocket(connFd); - // syncSendVpeerCfgMsg(sync); - } else { - // first packet tells what kind of link - if (msg.head.type == TAOS_SMSG_SYNC_DATA) { - pPeer->syncFd = connFd; - nodeSStatus = TAOS_SYNC_STATUS_START; - sInfo("%s, sync-data msg from master is received, tranId:%u, set sstatus:%s", pPeer->id, msg.tranId, - syncStatus[nodeSStatus]); - syncCreateRestoreDataThread(pPeer); - } else { - sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); - syncClosePeerConn(pPeer); - pPeer->peerFd = connFd; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); - sDebug("%s, ready to exchange data", pPeer->id); - syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); - } - } - - pthread_mutex_unlock(&pNode->mutex); -} - -static void syncProcessBrokenLink(int64_t rid, int32_t closedByApp) { - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) return; - - SSyncNode *pNode = pPeer->pSyncNode; - - pthread_mutex_lock(&pNode->mutex); - - sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d closedByApp:%d", - pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd, closedByApp); - pPeer->peerFd = -1; - if (!closedByApp && pPeer->isArb) { - tsArbOnline = 0; - } - - syncRestartConnection(pPeer); - pthread_mutex_unlock(&pNode->mutex); - - syncReleasePeer(pPeer); -} - -static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t _version, void *mhandle) { - SSyncFwds *pSyncFwds = pNode->pSyncFwds; - int64_t time = taosGetTimestampMs(); - - if (pSyncFwds->fwds >= SYNC_MAX_FWDS) { - // pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; - // pSyncFwds->fwds--; - sError("vgId:%d, failed to save fwd info, hver:%" PRIu64 " fwds:%d", pNode->vgId, _version, pSyncFwds->fwds); - return TSDB_CODE_SYN_TOO_MANY_FWDINFO; - } - - if (pSyncFwds->fwds > 0) { - pSyncFwds->last = (pSyncFwds->last + 1) % SYNC_MAX_FWDS; - } - - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last; - memset(pFwdInfo, 0, sizeof(SFwdInfo)); - pFwdInfo->version = _version; - pFwdInfo->mhandle = mhandle; - pFwdInfo->time = time; - - pSyncFwds->fwds++; - sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, _version, pSyncFwds->fwds); - - return 0; -} - -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { - SSyncFwds *pSyncFwds = pNode->pSyncFwds; - - int32_t fwds = pSyncFwds->fwds; - for (int32_t i = 0; i < fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; - if (pFwdInfo->confirmed == 0) break; - - pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; - pSyncFwds->fwds--; - if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last; - sTrace("vgId:%d, fwd info is removed, hver:%" PRIu64 " fwds:%d", pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); - memset(pFwdInfo, 0, sizeof(SFwdInfo)); - } -} - -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { - int32_t confirm = 0; - if (pFwdInfo->code == 0) pFwdInfo->code = code; - - if (code == 0) { - pFwdInfo->acks++; - if (pFwdInfo->acks >= pNode->quorum - 1) { - confirm = 1; - } - } else { - pFwdInfo->nacks++; - if (pFwdInfo->nacks > pNode->replica - pNode->quorum) { - confirm = 1; - } - } - - if (confirm && pFwdInfo->confirmed == 0) { - sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:0x%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); - (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code); - pFwdInfo->confirmed = 1; - } -} - -static void syncMonitorNodeRole(void *param, void *tmrId) { - int64_t rid = (int64_t)param; - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return; - - for (int32_t index = 0; index < pNode->replica; index++) { - if (index == pNode->selfIndex) continue; - - SSyncPeer *pPeer = pNode->peerInfo[index]; - if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; - if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; - - sDebug("%s, check roles since peer:%s sstatus:%s, self:%s sstatus:%s", pPeer->id, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], syncRole[nodeRole], syncStatus[nodeSStatus]); - syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); - break; - } - - pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - syncReleaseNode(pNode); -} - -static void syncMonitorFwdInfos(void *param, void *tmrId) { - int64_t rid = (int64_t)param; - SSyncNode *pNode = syncAcquireNode(rid); - if (pNode == NULL) return; - - SSyncFwds *pSyncFwds = pNode->pSyncFwds; - - if (pSyncFwds) { - int64_t time = taosGetTimestampMs(); - - if (pSyncFwds->fwds > 0) { - pthread_mutex_lock(&pNode->mutex); - for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS; - if (ABS(time - pFwdInfo->time) < 10000) break; - - sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, - pFwdInfo->version, time, pFwdInfo->time); - syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED); - } - - syncRemoveConfirmedFwdInfo(pNode); - pthread_mutex_unlock(&pNode->mutex); - } - - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - } - - syncReleaseNode(pNode); -} - -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype, bool force) { - SSyncPeer *pPeer; - SSyncHead *pSyncHead; - SWalHead * pWalHead = data; - int32_t fwdLen; - int32_t code = 0; - - if (pWalHead->version > nodeVersion + 1) { - sError("vgId:%d, hver:%" PRIu64 ", inconsistent with sver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion); - if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { - sInfo("vgId:%d, restart connection", pNode->vgId); - for (int32_t i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - syncRestartConnection(pPeer); - } - } - - if (pNode->replica != 1) { - return TSDB_CODE_SYN_INVALID_VERSION; - } - } - - // always update version - sTrace("vgId:%d, update version, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, - syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); - nodeVersion = pWalHead->version; - - if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; - - // only msg from RPC or CQ can be forwarded - if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; - - // a hacker way to improve the performance - pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); - syncBuildSyncFwdMsg(pSyncHead, pNode->vgId, sizeof(SWalHead) + pWalHead->len); - fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head - - pthread_mutex_lock(&pNode->mutex); - - for (int32_t i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - if (pPeer == NULL || pPeer->peerFd < 0) continue; - if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; - - if ((pNode->quorum > 1 || force) && code == 0) { - code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - if (code >= 0) { - code = 1; - } else { - pthread_mutex_unlock(&pNode->mutex); - return code; - } - } - - int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); - if (retLen == fwdLen) { - sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); - } else { - sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], pWalHead->version, retLen); - syncRestartConnection(pPeer); - } - } - - pthread_mutex_unlock(&pNode->mutex); - - return code; -} diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c deleted file mode 100644 index 64d4e72fac3b05911044098e43b6a4689a2f57f3..0000000000000000000000000000000000000000 --- a/src/sync/src/syncMsg.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tchecksum.h" -#include "syncInt.h" - -char *statusType[] = { - "broadcast", - "broadcast-rsp", - "setup-conn", - "setup-conn-rsp", - "exchange-data", - "exchange-data-rsp", - "check-role", - "check-role-rsp" -}; - -uint16_t syncGenTranId() { - return taosRand() & 0XFFFF; -} - -static void syncBuildHead(SSyncHead *pHead) { - pHead->protocol = SYNC_PROTOCOL_VERSION; - pHead->signature = SYNC_SIGNATURE; - pHead->code = 0; - pHead->cId = 0; - taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); -} - -int32_t syncCheckHead(SSyncHead *pHead) { - if (pHead->protocol != SYNC_PROTOCOL_VERSION) return TSDB_CODE_SYN_MISMATCHED_PROTOCOL; - if (pHead->signature != SYNC_SIGNATURE) return TSDB_CODE_SYN_MISMATCHED_SIGNATURE; - if (pHead->cId != 0) return TSDB_CODE_SYN_MISMATCHED_CLUSTERID; - if (pHead->len <= 0 || pHead->len > TSDB_MAX_WAL_SIZE) return TSDB_CODE_SYN_INVALID_MSGLEN; - if (pHead->type <= TAOS_SMSG_START || pHead->type >= TAOS_SMSG_END) return TSDB_CODE_SYN_INVALID_MSGTYPE; - if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SSyncHead))) return TSDB_CODE_SYN_INVALID_CHECKSUM; - - return TSDB_CODE_SUCCESS; -} - -void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len) { - pHead->type = TAOS_SMSG_SYNC_FWD; - pHead->vgId = vgId; - pHead->len = len; - syncBuildHead(pHead); -} - -void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t _version, int32_t code) { - pMsg->head.type = TAOS_SMSG_SYNC_FWD_RSP; - pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SFwdRsp) - sizeof(SSyncHead); - syncBuildHead(&pMsg->head); - - pMsg->version = _version; - pMsg->code = code; -} - -static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) { - pMsg->head.type = type; - pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SSyncMsg) - sizeof(SSyncHead); - syncBuildHead(&pMsg->head); - - pMsg->port = tsSyncPort; - pMsg->tranId = syncGenTranId(); - pMsg->sourceId = vgId; - tstrncpy(pMsg->fqdn, tsNodeFqdn, TSDB_FQDN_LEN); -} - -void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); } -void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); } -void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); } -void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_TEST); } - -void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) { - pMsg->head.type = TAOS_SMSG_STATUS; - pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SPeersStatus) - sizeof(SSyncHead); - syncBuildHead(&pMsg->head); -} - -void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) { - pMsg->head.type = TAOS_SMSG_SYNC_FILE_RSP; - pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SFileAck) - sizeof(SSyncHead); - syncBuildHead(&pMsg->head); -} - -void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) { - pMsg->head.type = TAOS_SMSG_SYNC_FILE; - pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead); - syncBuildHead(&pMsg->head); -} \ No newline at end of file diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c deleted file mode 100644 index bf9d5201a062e048365c2887b7b6a6d70b40547f..0000000000000000000000000000000000000000 --- a/src/sync/src/syncRestore.c +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tlog.h" -#include "tutil.h" -#include "ttimer.h" -#include "tsocket.h" -#include "tqueue.h" -#include "twal.h" -#include "tsync.h" -#include "syncInt.h" - -static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) { - SSyncNode *pNode = pPeer->pSyncNode; - - SFileVersion fileVersion; - memset(&fileVersion, 0, sizeof(SFileVersion)); - int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); - if (ret != sizeof(SFileVersion)) { - sError("%s, failed to read fver since %s", pPeer->id, strerror(errno)); - return -1; - } - - SFileAck fileVersionAck; - memset(&fileVersionAck, 0, sizeof(SFileAck)); - syncBuildFileAck(&fileVersionAck, pNode->vgId); - ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck)); - if (ret != sizeof(SFileAck)) { - sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno)); - return -1; - } - - *fversion = htobe64(fileVersion.fversion); - return 0; -} - -static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { - SSyncNode *pNode = pPeer->pSyncNode; - - if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { - sError("%s, failed to restore file", pPeer->id); - return -1; - } - - if (syncRecvFileVersion(pPeer, fversion) < 0) { - return -1; - } - - sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion); - return 0; -} - -static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { - SSyncNode *pNode = pPeer->pSyncNode; - int32_t ret, code = -1; - uint64_t lastVer = 0; - - SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1); // size for one record - if (pHead == NULL) return -1; - - while (1) { - ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret != sizeof(SWalHead)) { - sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); - break; - } - - if (pHead->len == 0) { - sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer); - code = 0; - break; - } // wal sync over - - ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret != pHead->len) { - sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); - break; - } - - sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); - - if (lastVer == pHead->version) { - sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer); - break; - } - lastVer = pHead->version; - - ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); - if (ret != 0) { - sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version); - break; - } - } - - if (code < 0) { - sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno)); - } - - free(pHead); - *wver = lastVer; - return code; -} - -static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { - SSyncNode *pNode = pPeer->pSyncNode; - SWalHead * pHead = (SWalHead *)offset; - - (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); - offset += pHead->len + sizeof(SWalHead); - - return offset; -} - -static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { - SSyncNode * pNode = pPeer->pSyncNode; - SRecvBuffer *pRecv = pNode->pRecv; - int32_t forwards = 0; - - if (pRecv == NULL) { - sError("%s, recv buffer is null, restart connect", pPeer->id); - return -1; - } - - sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards); - - char *offset = pRecv->buffer; - while (forwards < pRecv->forwards) { - offset = syncProcessOneBufferedFwd(pPeer, offset); - forwards++; - } - - pthread_mutex_lock(&pNode->mutex); - - while (forwards < pRecv->forwards && pRecv->code == 0) { - offset = syncProcessOneBufferedFwd(pPeer, offset); - forwards++; - } - - nodeRole = TAOS_SYNC_ROLE_SLAVE; - sDebug("%s, finish processing buffered fwds:%d", pPeer->id, forwards); - - pthread_mutex_unlock(&pNode->mutex); - - return pRecv->code; -} - -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { - SSyncNode * pNode = pPeer->pSyncNode; - SRecvBuffer *pRecv = pNode->pRecv; - int32_t len = pHead->len + sizeof(SWalHead); - - if (pRecv == NULL) { - sError("%s, recv buffer is not create yet", pPeer->id); - return -1; - } - - if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) { - memcpy(pRecv->offset, pHead, len); - pRecv->offset += len; - pRecv->forwards++; - sTrace("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards); - } else { - sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize); - pRecv->code = -1; // set error code - } - - return pRecv->code; -} - -static void syncCloseRecvBuffer(SSyncNode *pNode) { - if (pNode->pRecv) { - sDebug("vgId:%d, recv buffer:%p is freed", pNode->vgId, pNode->pRecv); - tfree(pNode->pRecv->buffer); - } - - tfree(pNode->pRecv); -} - -static int32_t syncOpenRecvBuffer(SSyncNode *pNode) { - syncCloseRecvBuffer(pNode); - - SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); - if (pRecv == NULL) return -1; - - pRecv->bufferSize = SYNC_RECV_BUFFER_SIZE; - pRecv->buffer = malloc(pRecv->bufferSize); - if (pRecv->buffer == NULL) { - free(pRecv); - return -1; - } - - pRecv->offset = pRecv->buffer; - pRecv->forwards = 0; - - pNode->pRecv = pRecv; - - sDebug("vgId:%d, recv buffer:%p is created", pNode->vgId, pNode->pRecv); - return 0; -} - -static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - nodeSStatus = TAOS_SYNC_STATUS_FILE; - uint64_t fversion = 0; - - sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - SSyncRsp rsp = {.sync = 1, .tranId = syncGenTranId()}; - if (taosWriteMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { - sError("%s, failed to send sync rsp since %s", pPeer->id, strerror(errno)); - return -1; - } - sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId); - - sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - (*pNode->startSyncFileFp)(pNode->vgId); - - int32_t code = syncRestoreFile(pPeer, &fversion); - if (code < 0) { - (*pNode->stopSyncFileFp)(pNode->vgId, fversion); - sError("%s, failed to restore files", pPeer->id); - return -1; - } - - (*pNode->stopSyncFileFp)(pNode->vgId, fversion); - nodeVersion = fversion; - - sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); - uint64_t wver = 0; - code = syncRestoreWal(pPeer, &wver); // lastwar - if (code < 0) { - sError("%s, failed to restore wal, code:%d", pPeer->id, code); - return -1; - } - - if (wver != 0) { - nodeVersion = wver; - sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion); - } - - nodeSStatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - if (syncProcessBufferedFwd(pPeer) < 0) { - sError("%s, failed to insert buffered points", pPeer->id); - return -1; - } - - return 0; -} - -void *syncRestoreData(void *param) { - setThreadName("syncRestoreData"); - int64_t rid = (int64_t)param; - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) { - sError("failed to restore data, invalid peer rid:%" PRId64, rid); - return NULL; - } - - SSyncNode *pNode = pPeer->pSyncNode; - - taosBlockSIGPIPE(); - atomic_add_fetch_32(&tsSyncNum, 1); - sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - - nodeRole = TAOS_SYNC_ROLE_SYNCING; - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - - if (syncOpenRecvBuffer(pNode) < 0) { - sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); - syncRestartConnection(pPeer); - } else { - if (syncRestoreDataStepByStep(pPeer) == 0) { - sInfo("%s, it is synced successfully", pPeer->id); - nodeRole = TAOS_SYNC_ROLE_SLAVE; - syncBroadcastStatus(pNode); - } else { - sError("%s, failed to restore data, restart connection", pPeer->id); - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - syncRestartConnection(pPeer); - } - } - - (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - - nodeSStatus = TAOS_SYNC_STATUS_INIT; - sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - - taosCloseSocket(pPeer->syncFd); - syncCloseRecvBuffer(pNode); - atomic_sub_fetch_32(&tsSyncNum, 1); - - // The ref is obtained in both the create thread and the current thread, so it is released twice - syncReleasePeer(pPeer); - syncReleasePeer(pPeer); - - return NULL; -} diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c deleted file mode 100644 index c86ab8549974712658ad3d381c4141427c000762..0000000000000000000000000000000000000000 --- a/src/sync/src/syncRetrieve.c +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tlog.h" -#include "tutil.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tsocket.h" -#include "twal.h" -#include "tsync.h" -#include "syncInt.h" - -static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { - uint64_t fver, wver; - int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); - if (code != 0) { - sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); - return -1; - } - - pPeer->lastWalVer = wver; - return code; -} - -static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { - uint64_t fver, wver; - int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); - if (code != 0) { - sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); - return true; - } - - if (wver != pPeer->lastWalVer) { - sInfo("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer); - return true; - } - - return false; -} - -static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { - uint64_t fver, wver; - int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); - if (code != 0) { - sInfo("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); - return -1; - } - - pPeer->lastFileVer = fver; - return code; -} - -static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { - uint64_t fver, wver; - int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); - if (code != 0) { - sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); - pPeer->fileChanged = 1; - return true; - } - - if (fver != pPeer->lastFileVer) { - sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer); - pPeer->fileChanged = 1; - return true; - } - - pPeer->fileChanged = 0; - return false; -} - -static int32_t syncSendFileVersion(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - SFileVersion fileVersion; - memset(&fileVersion, 0, sizeof(SFileVersion)); - syncBuildFileVersion(&fileVersion, pNode->vgId); - - uint64_t fver = pPeer->lastFileVer; - fileVersion.fversion = htobe64(fver); - int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); - if (ret != sizeof(SFileVersion)) { - sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno)); - return -1; - } - - SFileAck fileAck; - memset(&fileAck, 0, sizeof(SFileAck)); - ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); - if (ret != sizeof(SFileAck)) { - sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno)); - return -1; - } - - // set the peer sync version - pPeer->sversion = fver; - - return 0; -} - -static int32_t syncRetrieveFile(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - if (syncGetFileVersion(pNode, pPeer) < 0) { - pPeer->fileChanged = 1; - return -1; - } - - if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { - sError("%s, failed to retrieve file", pPeer->id); - return -1; - } - - if (syncSendFileVersion(pPeer) < 0) { - return -1; - } - - sInfo("%s, all files are retrieved", pPeer->id); - return 0; -} - -// if only a partial record is read out, upper layer will reload the file to get a complete record -static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { - int32_t ret = read(sfd, pHead, sizeof(SWalHead)); - if (ret < 0) { - sError("sfd:%d, failed to read wal head since %s, ret:%d", sfd, strerror(errno), ret); - return -1; - } - - if (ret == 0) { - sInfo("sfd:%d, read to the end of file, ret:%d", sfd, ret); - return 0; - } - - if (ret != sizeof(SWalHead)) { - // file is not at end yet, it shall be reloaded - sInfo("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret); - return 0; - } - - assert(pHead->len <= TSDB_MAX_WAL_SIZE); - - ret = read(sfd, pHead->cont, pHead->len); - if (ret < 0) { - sError("sfd:%d, failed to read wal content since %s, ret:%d", sfd, strerror(errno), ret); - return -1; - } - - if (ret != pHead->len) { - // file is not at end yet, it shall be reloaded - sInfo("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret); - return 0; - } - - return sizeof(SWalHead) + pHead->len; -} - -static int64_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { - int32_t sfd = open(name, O_RDONLY | O_BINARY); - if (sfd < 0) { - sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); - return -1; - } - - int64_t code = taosLSeek(sfd, offset, SEEK_SET); - if (code < 0) { - sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); - close(sfd); - return -1; - } - - sInfo("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion); - - SWalHead *pHead = malloc(SYNC_MAX_SIZE); - int64_t bytes = 0; - - while (1) { - code = syncReadOneWalRecord(sfd, pHead); - if (code < 0) { - sError("%s, failed to read one record from wal:%s", pPeer->id, name); - break; - } - - if (code == 0) { - code = bytes; - sInfo("%s, read to the end of wal, bytes:%" PRId64, pPeer->id, bytes); - break; - } - - sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); - - int32_t wsize = (int32_t)code; - int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); - if (ret != wsize) { - code = -1; - sError("%s, failed to forward wal since %s, hver:%" PRIu64, pPeer->id, strerror(errno), pHead->version); - break; - } - - pPeer->sversion = pHead->version; - bytes += wsize; - - if (pHead->version >= fversion && fversion > 0) { - code = 0; - sInfo("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion); - break; - } - } - - free(pHead); - close(sfd); - - return code; -} - -static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { - SSyncNode *pNode = pPeer->pSyncNode; - int32_t once = 0; // last WAL has once ever been processed - int64_t offset = 0; - uint64_t fversion = 0; - char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file - - // get full path to wal file - snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - sInfo("%s, start to retrieve last wal:%s", pPeer->id, fname); - - while (1) { - if (syncAreFilesModified(pNode, pPeer)) return -1; - if (syncGetWalVersion(pNode, pPeer) < 0) return -1; - - int64_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); - if (bytes < 0) { - sInfo("%s, failed to retrieve last wal, bytes:%" PRId64, pPeer->id, bytes); - return bytes; - } - - // check file changes - bool walModified = syncIsWalModified(pNode, pPeer); - - // if file is not updated or updated once, set the fversion and sstatus - if (!walModified || once) { - if (fversion == 0) { - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt - fversion = nodeVersion; // must read data to fversion - sInfo("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion); - } - } - - // if all data up to fversion is read out, it is over - if (pPeer->sversion >= fversion && fversion > 0) { - sInfo("%s, data up to fver:%" PRIu64 " has been read out, bytes:%" PRId64 " sver:%" PRIu64, pPeer->id, fversion, bytes, - pPeer->sversion); - return 0; - } - - // if all data are read out, and no update - if (bytes == 0 && !walModified) { - // wal not closed, it means some data not flushed to disk, wait for a while - taosMsleep(10); - } - - // if bytes > 0, file is updated, or fversion is not reached but file still open, read again - once = 1; - offset += bytes; - sInfo("%s, continue retrieve last wal, bytes:%" PRId64 " offset:%" PRId64 " sver:%" PRIu64 " fver:%" PRIu64, pPeer->id, - bytes, offset, pPeer->sversion, fversion); - } - - return -1; -} - -static int64_t syncRetrieveWal(SSyncPeer *pPeer) { - SSyncNode * pNode = pPeer->pSyncNode; - char fname[TSDB_FILENAME_LEN * 3]; - char wname[TSDB_FILENAME_LEN * 2]; - int32_t size; - int64_t code = -1; - int64_t index = 0; - - while (1) { - // retrieve wal info - wname[0] = 0; - code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index); - if (code < 0) { - sError("%s, failed to get wal info since:%s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); - break; - } - - if (wname[0] == 0) { // no wal file - code = 0; - sInfo("%s, no wal file anymore", pPeer->id); - break; - } - - if (code == 0) { // last wal - code = syncProcessLastWal(pPeer, wname, index); - sInfo("%s, last wal processed, code:%" PRId64, pPeer->id, code); - break; - } - - // get the full path to wal file - snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - - // send wal file, old wal file won't be modified, even remove is ok - struct stat fstat; - if (stat(fname, &fstat) < 0) { - code = -1; - sInfo("%s, failed to stat wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); - break; - } - - size = fstat.st_size; - sInfo("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - - int32_t sfd = open(fname, O_RDONLY | O_BINARY); - if (sfd < 0) { - code = -1; - sError("%s, failed to open wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); - break; - } - - code = taosSendFile(pPeer->syncFd, sfd, NULL, size); - close(sfd); - if (code < 0) { - sError("%s, failed to send wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); - break; - } - - if (syncAreFilesModified(pNode, pPeer)) { - code = -1; - break; - } - } - - if (code == 0) { - SWalHead walHead; - memset(&walHead, 0, sizeof(walHead)); - if (taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)) == sizeof(walHead)) { - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - } else { - sError("%s, failed to send last wal record since %s", pPeer->id, strerror(errno)); - code = -1; - } - } else { - sError("%s, failed to send wal since %s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); - } - - return code; -} - -static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { - SSyncNode *pNode = pPeer->pSyncNode; - - SSyncMsg msg; - syncBuildSyncDataMsg(&msg, pNode->vgId); - - if (taosWriteMsg(pPeer->syncFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { - sError("%s, failed to send sync-data msg since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); - return -1; - } - sInfo("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId); - - SSyncRsp rsp; - if (taosReadMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { - sError("%s, failed to read sync-data rsp since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); - return -1; - } - - sInfo("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId); - return 0; -} - -static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { - sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - if (syncRetrieveFirstPkt(pPeer) < 0) { - sError("%s, failed to start retrieve", pPeer->id); - return -1; - } - - pPeer->sversion = 0; - pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - if (syncRetrieveFile(pPeer) != 0) { - sError("%s, failed to retrieve files", pPeer->id); - return -1; - } - - // if no files are synced, there must be wal to sync, sversion must be larger than one - if (pPeer->sversion == 0) pPeer->sversion = 1; - - sInfo("%s, start to retrieve wals", pPeer->id); - int64_t code = syncRetrieveWal(pPeer); - if (code < 0) { - sError("%s, failed to retrieve wals, code:0x%" PRIx64, pPeer->id, code); - return -1; - } - - return 0; -} - -void *syncRetrieveData(void *param) { - int64_t rid = (int64_t)param; - SSyncPeer *pPeer = syncAcquirePeer(rid); - if (pPeer == NULL) { - sError("failed to retrieve data, invalid peer rid:%" PRId64, rid); - return NULL; - } - - uint32_t ip = syncResolvePeerFqdn(pPeer); - if (!ip) { - syncReleasePeer(pPeer); - return NULL; - } - - SSyncNode *pNode = pPeer->pSyncNode; - - taosBlockSIGPIPE(); - sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus], - pPeer->numOfRetrieves); - - if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves); - - pPeer->syncFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); - if (pPeer->syncFd < 0) { - sError("%s, failed to open socket to sync", pPeer->id); - } else { - sInfo("%s, sync tcp is setup", pPeer->id); - - if (syncRetrieveDataStepByStep(pPeer) == 0) { - sInfo("%s, sync retrieve process is successful", pPeer->id); - } else { - sError("%s, failed to retrieve data, restart connection", pPeer->id); - syncRestartConnection(pPeer); - } - } - - if (pPeer->fileChanged) { - pPeer->numOfRetrieves++; - } else { - pPeer->numOfRetrieves = 0; - // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); - } - - if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); - - pPeer->fileChanged = 0; - taosCloseSocket(pPeer->syncFd); - - // The ref is obtained in both the create thread and the current thread, so it is released twice - sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - - syncReleasePeer(pPeer); - syncReleasePeer(pPeer); - - return NULL; -} diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c deleted file mode 100644 index ccb0a67e5ca99441a1c3026fee498c36795518c5..0000000000000000000000000000000000000000 --- a/src/sync/src/syncTcp.c +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tulog.h" -#include "tutil.h" -#include "tsocket.h" -#include "taoserror.h" -#include "twal.h" -#include "tsync.h" -#include "syncInt.h" -#include "syncTcp.h" - -typedef struct SThreadObj { - pthread_t thread; - bool stop; - SOCKET pollFd; - int32_t numOfFds; - struct SPoolObj *pPool; -} SThreadObj; - -typedef struct SPoolObj { - SPoolInfo info; - SThreadObj **pThread; - pthread_t thread; - int32_t nextId; - SOCKET acceptFd; // FD for accept new connection - int8_t stop; -} SPoolObj; - -typedef struct { - SThreadObj *pThread; - int64_t handleId; - SOCKET fd; - int32_t closedByApp; -} SConnObj; - -static void *syncAcceptPeerTcpConnection(void *argv); -static void *syncProcessTcpData(void *param); -static void syncStopPoolThread(SThreadObj *pThread); -static SThreadObj *syncGetTcpThread(SPoolObj *pPool); - -void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { - pthread_attr_t thattr; - - SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); - if (pPool == NULL) { - sError("failed to alloc pool for TCP server since no enough memory"); - return NULL; - } - - pPool->info = *pInfo; - - pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads); - if (pPool->pThread == NULL) { - sError("failed to alloc pool thread for TCP server since no enough memory"); - tfree(pPool); - return NULL; - } - - pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port); - if (pPool->acceptFd < 0) { - tfree(pPool->pThread); - tfree(pPool); - sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); - return NULL; - } - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { - sError("failed to create accept thread for TCP server since %s", strerror(errno)); - taosCloseSocket(pPool->acceptFd); - tfree(pPool->pThread); - tfree(pPool); - return NULL; - } - - pthread_attr_destroy(&thattr); - - sDebug("%p TCP pool is created", pPool); - return pPool; -} - -void syncCloseTcpThreadPool(void *param) { - SPoolObj * pPool = param; - SThreadObj *pThread; - - pPool->stop = 1; - -#ifdef WINDOWS - closesocket(pPool->acceptFd); -#elif defined(__APPLE__) - if (pPool->acceptFd!=-1) { - close(pPool->acceptFd); - pPool->acceptFd = -1; - } -#else - shutdown(pPool->acceptFd, SHUT_RD); -#endif - - pthread_join(pPool->thread, NULL); - - for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { - pThread = pPool->pThread[i]; - if (pThread) syncStopPoolThread(pThread); - } - - sDebug("%p TCP pool is closed", pPool); - - tfree(pPool->pThread); - tfree(pPool); -} - -void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { - struct epoll_event event; - SPoolObj *pPool = param; - - SConnObj *pConn = calloc(sizeof(SConnObj), 1); - if (pConn == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - SThreadObj *pThread = syncGetTcpThread(pPool); - if (pThread == NULL) { - tfree(pConn); - return NULL; - } - - pConn->fd = connFd; - pConn->pThread = pThread; - pConn->handleId = rid; - pConn->closedByApp = 0; - - event.events = EPOLLIN | EPOLLRDHUP; - event.data.ptr = pConn; - - if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - sError("failed to add fd:%d since %s", connFd, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - tfree(pConn); - pConn = NULL; - } else { - pThread->numOfFds++; - sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); - } - - return pConn; -} - -void syncFreeTcpConn(void *param) { - SConnObj * pConn = param; - SThreadObj *pThread = pConn->pThread; - - sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); - pConn->closedByApp = 1; - shutdown(pConn->fd, SHUT_WR); -} - -static void taosProcessBrokenLink(SConnObj *pConn) { - SThreadObj *pThread = pConn->pThread; - SPoolObj * pPool = pThread->pPool; - SPoolInfo * pInfo = &pPool->info; - - if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->handleId, pConn->closedByApp); - - pThread->numOfFds--; - epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); - sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - taosCloseSocket(pConn->fd); - tfree(pConn); -} - -#define maxEvents 10 - -static void *syncProcessTcpData(void *param) { - SThreadObj *pThread = (SThreadObj *)param; - SPoolObj * pPool = pThread->pPool; - SPoolInfo * pInfo = &pPool->info; - SConnObj * pConn = NULL; - struct epoll_event events[maxEvents]; - - setThreadName("syncTcpData"); - - void *buffer = malloc(pInfo->bufferSize); - taosBlockSIGPIPE(); - - while (1) { - if (pThread->stop) break; - int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); - if (pThread->stop) { - sDebug("%p TCP epoll thread is exiting...", pThread); - break; - } - - if (fdNum < 0) { - sError("epoll_wait failed since %s", strerror(errno)); - continue; - } - - for (int32_t i = 0; i < fdNum; ++i) { - pConn = events[i].data.ptr; - assert(pConn); - - if (events[i].events & EPOLLERR) { - sDebug("conn is broken since EPOLLERR"); - taosProcessBrokenLink(pConn); - continue; - } - - if (events[i].events & EPOLLHUP) { - sDebug("conn is broken since EPOLLHUP"); - taosProcessBrokenLink(pConn); - continue; - } - - if (events[i].events & EPOLLRDHUP) { - sDebug("conn is broken since EPOLLRDHUP"); - taosProcessBrokenLink(pConn); - continue; - } - - if (pConn->closedByApp == 0) { - if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) { - syncFreeTcpConn(pConn); - continue; - } - } - } - - if (pThread->stop) break; - } - - sDebug("%p TCP epoll thread exits", pThread); - - EpollClose(pThread->pollFd); - tfree(pThread); - tfree(buffer); - return NULL; -} - -static void *syncAcceptPeerTcpConnection(void *argv) { - SPoolObj * pPool = (SPoolObj *)argv; - SPoolInfo *pInfo = &pPool->info; - - taosBlockSIGPIPE(); - setThreadName("acceptTcpConn"); - - while (1) { - struct sockaddr_in clientAddr; - socklen_t addrlen = sizeof(clientAddr); - SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); - if (pPool->stop) { - sDebug("%p TCP server accept is stopped", pPool); - break; - } - - if (connFd < 0) { - if (errno == EINVAL) { - sDebug("%p TCP server accept is exiting...", pPool); - break; - } else { - sError("TCP accept failure since %s", strerror(errno)); - continue; - } - } - - // sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); - taosKeepTcpAlive(connFd); - (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); - } - - taosCloseSocket(pPool->acceptFd); - return NULL; -} - -static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { - SThreadObj *pThread = pPool->pThread[pPool->nextId]; - - if (pThread) return pThread; - - pThread = (SThreadObj *)calloc(1, sizeof(SThreadObj)); - if (pThread == NULL) return NULL; - - pThread->pPool = pPool; - pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter - if (pThread->pollFd < 0) { - tfree(pThread); - return NULL; - } - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)syncProcessTcpData, pThread); - pthread_attr_destroy(&thattr); - - if (ret != 0) { - EpollClose(pThread->pollFd); - tfree(pThread); - return NULL; - } - - sDebug("%p TCP epoll thread is created", pThread); - pPool->pThread[pPool->nextId] = pThread; - pPool->nextId++; - pPool->nextId = pPool->nextId % pPool->info.numOfThreads; - - return pThread; -} - -static void syncStopPoolThread(SThreadObj *pThread) { - pthread_t thread = pThread->thread; - if (!taosCheckPthreadValid(thread)) { - return; - } - pThread->stop = true; - if (taosComparePthread(thread, pthread_self())) { - pthread_detach(pthread_self()); - return; - } - pthread_join(thread, NULL); -} diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt deleted file mode 100644 index a5ab8191371ce97ecbaf9ef4dc8dbace6a6c4802..0000000000000000000000000000000000000000 --- a/src/sync/test/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -IF (TD_LINUX) - INCLUDE_DIRECTORIES(../inc) - - LIST(APPEND CLIENT_SRC ./syncClient.c) - ADD_EXECUTABLE(syncClient ${CLIENT_SRC}) - TARGET_LINK_LIBRARIES(syncClient sync trpc common) - - LIST(APPEND SERVER_SRC ./syncServer.c) - ADD_EXECUTABLE(syncServer ${SERVER_SRC}) - TARGET_LINK_LIBRARIES(syncServer sync trpc common) -ENDIF () - diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c deleted file mode 100644 index 23ea54ee0c19b6ad2f93d7577d8d711874b10968..0000000000000000000000000000000000000000 --- a/src/sync/test/syncClient.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tglobal.h" -#include "tulog.h" -#include "trpc.h" -#include "taoserror.h" - -typedef struct { - int index; - SRpcEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; - pthread_t thread; - void * pRpc; -} SInfo; - -void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SInfo *pInfo = (SInfo *)pMsg->ahandle; - uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); - - if (pEpSet) pInfo->epSet = *pEpSet; - rpcFreeCont(pMsg->pCont); - - tsem_post(&pInfo->rspSem); -} - -int tcount = 0; - -void *sendRequest(void *param) { - SInfo * pInfo = (SInfo *)param; - SRpcMsg rpcMsg = {0}; - - uDebug("thread:%d, start to send request", pInfo->index); - - while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { - pInfo->num++; - rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); - rpcMsg.contLen = pInfo->msgSize; - rpcMsg.ahandle = pInfo; - rpcMsg.msgType = 1; - uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) { - uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - } - tsem_wait(&pInfo->rspSem); - } - - uDebug("thread:%d, it is over", pInfo->index); - tcount++; - - return NULL; -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - SRpcEpSet epSet; - char secret[TSDB_KEY_LEN] = "mypassword"; - int msgSize = 128; - int numOfReqs = 0; - int appThreads = 1; - char serverIp[40] = "127.0.0.1"; - struct timeval systemTime; - int64_t startTime, endTime; - pthread_attr_t thattr; - - // server info - epSet.numOfEps = 1; - epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); - - // client info - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "APP"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processResponse; - rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; - rpcInit.user = "michael"; - rpcInit.secret = secret; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.connType = TAOS_CONN_CLIENT; - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - epSet.port[0] = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { - tstrncpy(epSet.fqdn[0], argv[++i], TSDB_FQDN_LEN); - } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { - msgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0 && i < argc - 1) { - numOfReqs = atoi(argv[++i]); - } else if (strcmp(argv[i], "-a") == 0 && i < argc - 1) { - appThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { - tsCompressMsgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0 && i < argc - 1) { - rpcInit.user = argv[++i]; - } else if (strcmp(argv[i], "-k") == 0 && i < argc - 1) { - rpcInit.secret = argv[++i]; - } else if (strcmp(argv[i], "-spi") == 0 && i < argc - 1) { - rpcInit.spi = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - rpcDebugFlag = atoi(argv[++i]); - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); - printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); - printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); - printf(" [-a threads]: number of app threads, default is:%d\n", appThreads); - printf(" [-n requests]: number of requests per thread, default is:%d\n", numOfReqs); - printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-u user]: user name for the connection, default is:%s\n", rpcInit.user); - printf(" [-k secret]: password for the connection, default is:%s\n", rpcInit.secret); - printf(" [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi); - printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - taosInitLog("client.log", 100000, 10); - - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - uError("failed to initialize RPC"); - return -1; - } - - uInfo("client is initialized"); - - gettimeofday(&systemTime, NULL); - startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - for (int i = 0; i < appThreads; ++i) { - pInfo->index = i; - pInfo->epSet = epSet; - pInfo->numOfReqs = numOfReqs; - pInfo->msgSize = msgSize; - tsem_init(&pInfo->rspSem, 0, 0); - pInfo->pRpc = pRpc; - pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); - pInfo++; - } - - do { - usleep(1); - } while (tcount < appThreads); - - gettimeofday(&systemTime, NULL); - endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - float usedTime = (endTime - startTime) / 1000.0; // mseconds - - uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); - - taosCloseLog(); - - return 0; -} diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c deleted file mode 100644 index 4598e16a9d05be29d11612755a079ce0a228a2ff..0000000000000000000000000000000000000000 --- a/src/sync/test/syncServer.c +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -//#define _DEFAULT_SOURCE -#include -#include "os.h" -#include "tulog.h" -#include "tglobal.h" -#include "tsocket.h" -#include "trpc.h" -#include "tqueue.h" -#include "twal.h" -#include "tsync.h" - -int msgSize = 128; -int commit = 0; -int dataFd = -1; -void * qhandle = NULL; -int walNum = 0; -uint64_t tversion = 0; -int64_t syncHandle; -int role; -int nodeId; -char path[256]; -int numOfWrites; -SSyncInfo syncInfo; -SSyncCfg *pCfg; - -int writeIntoWal(SWalHead *pHead) { - if (dataFd < 0) { - char walName[280]; - snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum); - (void)remove(walName); - dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) { - uInfo("failed to open wal file:%s(%s)", walName, strerror(errno)); - return -1; - } else { - walNum++; - uInfo("file:%s is opened to write, walNum:%d", walName, walNum); - } - } - - if (write(dataFd, pHead, sizeof(SWalHead) + pHead->len) < 0) { - uError("ver:%" PRIu64 ", failed to write wal file(%s)", pHead->version, strerror(errno)); - } else { - uDebug("ver:%" PRIu64 ", written to wal", pHead->version); - } - - numOfWrites++; - if (numOfWrites >= 10000) { - uInfo("%d request have been written into disk", numOfWrites); - close(dataFd); - dataFd = -1; - numOfWrites = 0; - } - - return 0; -} - -void confirmForward(int32_t vgId, void *mhandle, int32_t code) { - SRpcMsg * pMsg = (SRpcMsg *)mhandle; - SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); - - uDebug("ver:%" PRIu64 ", confirm is received", pHead->version); - - rpcFreeCont(pMsg->pCont); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pMsg->handle; - rpcMsg.code = code; - rpcSendResponse(&rpcMsg); - - taosFreeQitem(mhandle); -} - -int processRpcMsg(void *item) { - SRpcMsg * pMsg = (SRpcMsg *)item; - SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); - int code = -1; - - if (role != TAOS_SYNC_ROLE_MASTER) { - uError("not master, write failed, role:%s", syncRole[role]); - } else { - pHead->version = ++tversion; - pHead->msgType = pMsg->msgType; - pHead->len = pMsg->contLen; - - uDebug("ver:%" PRIu64 ", rsp from client processed", pHead->version); - writeIntoWal(pHead); - syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC); - - code = 0; - } - - if (pCfg->quorum <= 1) { - rpcFreeCont(pMsg->pCont); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - rpcMsg.handle = pMsg->handle; - rpcMsg.code = code; - rpcSendResponse(&rpcMsg); - taosFreeQitem(item); - } - - return code; -} - -int processFwdMsg(void *item) { - SWalHead *pHead = (SWalHead *)item; - - if (pHead->version <= tversion) { - uError("ver:%" PRIu64 ", forward is even lower than local:%" PRIu64, pHead->version, tversion); - return -1; - } - - uDebug("ver:%" PRIu64 ", forward from peer is received", pHead->version); - writeIntoWal(pHead); - tversion = pHead->version; - - if (pCfg->quorum > 1) syncConfirmForward(syncHandle, pHead->version, 0); - - // write into cache - - /* - if (pHead->handle) { - syncSendFwdAck(syncHandle, pHead->handle, 0); - } - */ - - taosFreeQitem(item); - - return 0; -} - -int processWalMsg(void *item) { - SWalHead *pHead = (SWalHead *)item; - - if (pHead->version <= tversion) { - uError("ver:%" PRIu64 ", wal is even lower than local:%" PRIu64, pHead->version, tversion); - return -1; - }; - - uDebug("ver:%" PRIu64 ", wal from peer is received", pHead->version); - writeIntoWal(pHead); - tversion = pHead->version; - - // write into cache - - /* - if (pHead->handle) { - syncSendFwdAck(syncHandle, pHead->handle, 0); - } - */ - - taosFreeQitem(item); - - return 0; -} - -void *processWriteQueue(void *param) { - int type; - void *item; - - setThreadName("syncWrite"); - - while (1) { - int ret = taosReadQitem(qhandle, &type, &item); - if (ret <= 0) { - usleep(1000); - continue; - } - - if (type == TAOS_QTYPE_RPC) { - processRpcMsg(item); - } else if (type == TAOS_QTYPE_WAL) { - processWalMsg(item); - } else if (type == TAOS_QTYPE_FWD) { - processFwdMsg(item); - } - } - - return NULL; -} - -int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { - // app shall retrieve the auth info based on meterID from DB or a data file - // demo code here only for simple demo - int ret = 0; - - if (strcmp(meterId, "michael") == 0) { - *spi = 1; - *encrypt = 0; - strcpy(secret, "mypassword"); - strcpy(ckey, "key"); - } else if (strcmp(meterId, "jeff") == 0) { - *spi = 0; - *encrypt = 0; - } else { - ret = -1; // user not there - } - - return ret; -} - -void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - SRpcMsg *pTemp; - - pTemp = taosAllocateQitem(sizeof(SRpcMsg)); - memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - - uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen); - taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); -} - -uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { - uint32_t magic; - struct stat fstat; - char aname[280]; - - if (*index == 2) { - uInfo("wait for a while ....."); - sleep(3); - } - - if (name[0] == 0) { - // find the file - snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index); - sprintf(name, "data/data.%d", *index); - } else { - snprintf(aname, sizeof(aname), "%s/%s", path, name); - } - - uInfo("get file info:%s", aname); - if (stat(aname, &fstat) < 0) return 0; - - *size = fstat.st_size; - magic = fstat.st_size; - - return magic; -} - -int getWalInfo(int32_t vgId, char *name, int64_t *index) { - struct stat fstat; - char aname[280]; - - name[0] = 0; - if (*index + 1 > walNum) return 0; - - snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index); - sprintf(name, "wal/wal.%d", *index); - uInfo("get wal info:%s", aname); - - if (stat(aname, &fstat) < 0) return -1; - - if (*index >= walNum - 1) return 0; // no more - - return 1; -} - -int writeToCache(int32_t vgId, void *data, int type) { - SWalHead *pHead = data; - - uDebug("rsp from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); - - int msgSize = pHead->len + sizeof(SWalHead); - void *pMsg = taosAllocateQitem(msgSize); - memcpy(pMsg, pHead, msgSize); - taosWriteQitem(qhandle, type, pMsg); - - return 0; -} - -void confirmFwd(int32_t vgId, int64_t version) { return; } - -void notifyRole(int32_t vgId, int8_t r) { - role = r; - printf("current role:%s\n", syncRole[role]); -} - -void initSync() { - pCfg->replica = 1; - pCfg->quorum = 1; - syncInfo.vgId = 1; - syncInfo.getWalInfoFp = getWalInfo; - syncInfo.writeToCacheFp = writeToCache; - syncInfo.confirmForward = confirmForward; - syncInfo.notifyRoleFp = notifyRole; - - pCfg->nodeInfo[0].nodeId = 1; - pCfg->nodeInfo[0].nodePort = 7010; - taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodeId = 2; - pCfg->nodeInfo[1].nodePort = 7110; - taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodeId = 3; - pCfg->nodeInfo[2].nodePort = 7210; - taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); - - pCfg->nodeInfo[3].nodeId = 4; - pCfg->nodeInfo[3].nodePort = 7310; - taosGetFqdn(pCfg->nodeInfo[3].nodeFqdn); - - pCfg->nodeInfo[4].nodeId = 5; - pCfg->nodeInfo[4].nodePort = 7410; - taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn); -} - -void doSync() { - for (int i = 0; i < 5; ++i) { - if (tsSyncPort == pCfg->nodeInfo[i].nodePort) nodeId = pCfg->nodeInfo[i].nodeId; - } - - snprintf(path, sizeof(path), "/root/test/d%d", nodeId); - tstrncpy(syncInfo.path, path, sizeof(syncInfo.path)); - - if (syncHandle == NULL) { - syncHandle = syncStart(&syncInfo); - } else { - if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL; - } - - uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort); -} - -int main(int argc, char *argv[]) { - SRpcInit rpcInit; - char dataName[20] = "server.data"; - pCfg = &syncInfo.syncCfg; - - initSync(); - - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 7000; - rpcInit.label = "SER"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processRequestMsg; - rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer * 1500; - rpcInit.afp = retrieveAuthInfo; - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - rpcInit.localPort = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { - rpcInit.numOfThreads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { - msgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-s") == 0 && i < argc - 1) { - rpcInit.sessions = atoi(argv[++i]); - } else if (strcmp(argv[i], "-o") == 0 && i < argc - 1) { - tsCompressMsgSize = atoi(argv[++i]); - } else if (strcmp(argv[i], "-w") == 0 && i < argc - 1) { - commit = atoi(argv[++i]); - } else if (strcmp(argv[i], "-v") == 0 && i < argc - 1) { - syncInfo.version = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0 && i < argc - 1) { - pCfg->replica = atoi(argv[++i]); - } else if (strcmp(argv[i], "-q") == 0 && i < argc - 1) { - pCfg->quorum = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { - rpcDebugFlag = atoi(argv[++i]); - } else { - printf("\nusage: %s [options] \n", argv[0]); - printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); - printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); - printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); - printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); - printf(" [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize); - printf(" [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit); - printf(" [-v version]: initial node version, default is:%" PRId64 "\n", syncInfo.version); - printf(" [-r replica]: replicacation number, default is:%d\n", pCfg->replica); - printf(" [-q quorum]: quorum, default is:%d\n", pCfg->quorum); - printf(" [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag); - printf(" [-h help]: print out this help\n\n"); - exit(0); - } - } - - uDebugFlag = rpcDebugFlag; - dDebugFlag = rpcDebugFlag; - // tmrDebugFlag = rpcDebugFlag; - tsAsyncLog = 0; - taosInitLog("server.log", 1000000, 10); - - rpcInit.connType = TAOS_CONN_SERVER; - void *pRpc = rpcOpen(&rpcInit); - if (pRpc == NULL) { - uError("failed to start RPC server"); - return -1; - } - - tsSyncPort = rpcInit.localPort + 10; - qhandle = taosOpenQueue(); - - doSync(); - - pthread_attr_t thattr; - pthread_t thread; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&thread, &thattr, processWriteQueue, NULL) != 0) { - uError("failed to create thread, reason:%s", strerror(errno)); - return -1; - } - - printf("server is running, localPort:%d\n", rpcInit.localPort); - SNodesRole nroles; - - while (1) { - int c = getchar(); - - switch (c) { - case '1': - pCfg->replica = 1; - doSync(); - break; - case '2': - pCfg->replica = 2; - doSync(); - break; - case '3': - pCfg->replica = 3; - doSync(); - break; - case '4': - pCfg->replica = 4; - doSync(); - break; - case '5': - pCfg->replica = 5; - doSync(); - break; - case 's': - syncGetNodesRole(syncHandle, &nroles); - for (int i = 0; i < pCfg->replica; ++i) - printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]); - break; - default: - break; - } - - if (c == 'q') break; - } - - syncStop(syncHandle); - - if (dataFd >= 0) { - close(dataFd); - remove(dataName); - } - - return 0; -}