tstream.c 18.8 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tstream.h"
#include "executor.h"

L
Liu Jicong 已提交
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
  int32_t tlen = 0;
  tlen += taosEncodeFixedI8(buf, pOutput->type);
  tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
  tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
  ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
  tlen += tEncodeDataBlocks(buf, pOutput->blocks);
  return tlen;
}

void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
  buf = taosDecodeFixedI8(buf, &pInput->type);
  buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
  buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
  ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
  buf = tDecodeDataBlocks(buf, &pInput->blocks);
  return (void*)buf;
}

L
Liu Jicong 已提交
38
static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
L
Liu Jicong 已提交
39 40 41 42 43 44 45 46 47 48 49
  SStreamTaskExecReq req = {
      .streamId = pTask->streamId,
      .data = data,
  };

  int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req);
  void*   buf = rpcMallocCont(tlen);

  if (buf == NULL) {
    return -1;
  }
L
Liu Jicong 已提交
50

L
Liu Jicong 已提交
51 52 53
  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
    ((SMsgHead*)buf)->vgId = 0;
    req.taskId = pTask->inplaceDispatcher.taskId;
L
Liu Jicong 已提交
54

L
Liu Jicong 已提交
55 56 57 58
  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
    ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId);
    *ppEpSet = &pTask->fixedEpDispatcher.epSet;
    req.taskId = pTask->fixedEpDispatcher.taskId;
L
Liu Jicong 已提交
59

L
Liu Jicong 已提交
60
  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
L
fix  
Liu Jicong 已提交
61
    // TODO use general name rule of schemaless
L
Liu Jicong 已提交
62
    char ctbName[TSDB_TABLE_FNAME_LEN + 22];
L
Liu Jicong 已提交
63 64 65 66 67 68 69 70
    // all groupId must be the same in an array
    SSDataBlock* pBlock = taosArrayGet(data, 0);
    sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId);

    // TODO: get hash function by hashMethod

    // get groupId, compute hash value
    uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
L
Liu Jicong 已提交
71

L
Liu Jicong 已提交
72 73 74 75
    // get node
    // TODO: optimize search process
    SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
    int32_t sz = taosArrayGetSize(vgInfo);
L
Liu Jicong 已提交
76
    int32_t nodeId = 0;
L
Liu Jicong 已提交
77 78 79 80
    for (int32_t i = 0; i < sz; i++) {
      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
      if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
        nodeId = pVgInfo->vgId;
L
Liu Jicong 已提交
81
        req.taskId = pVgInfo->taskId;
L
Liu Jicong 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
        *ppEpSet = &pVgInfo->epSet;
        break;
      }
    }
    ASSERT(nodeId != 0);
    ((SMsgHead*)buf)->vgId = htonl(nodeId);
  }

  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
  tEncodeSStreamTaskExecReq(&abuf, &req);

  pMsg->pCont = buf;
  pMsg->contLen = tlen;
  pMsg->code = 0;
  pMsg->msgType = pTask->dispatchMsgType;
S
Shengliang Guan 已提交
97
  pMsg->info.noResp = 1;
L
Liu Jicong 已提交
98 99 100 101 102 103 104 105 106

  return 0;
}

static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashObj* data) {
  void* pIter = NULL;
  while (1) {
    pIter = taosHashIterate(data, pIter);
    if (pIter == NULL) return 0;
L
Liu Jicong 已提交
107
    SArray* pData = *(SArray**)pIter;
L
Liu Jicong 已提交
108 109
    SRpcMsg dispatchMsg = {0};
    SEpSet* pEpSet;
L
Liu Jicong 已提交
110
    if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
L
Liu Jicong 已提交
111 112 113
      ASSERT(0);
      return -1;
    }
114
    tmsgSendReq(pEpSet, &dispatchMsg);
L
Liu Jicong 已提交
115 116 117 118
  }
  return 0;
}

L
Liu Jicong 已提交
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) {
  ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK);
  int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
  if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
    streamDataSubmitRefInc(input);
    taosWriteQitem(pTask->inputQ, input);
  }
  return inputStatus;
}

int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
  ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
  taosWriteQitem(pTask->inputQ, input);
  int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
  return inputStatus;
}

L
Liu Jicong 已提交
136
static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
L
Liu Jicong 已提交
137
  void* exec = pTask->exec.executor;
L
Liu Jicong 已提交
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154

  // set input
  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
    SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
    ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);

    qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
  } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
    SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
    ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);

    SArray* blocks = pBlock->blocks;
    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
  }

  // exec
  while (1) {
L
Liu Jicong 已提交
155
    SSDataBlock* output = NULL;
L
Liu Jicong 已提交
156 157 158 159 160
    uint64_t     ts = 0;
    if (qExecTask(exec, &output, &ts) < 0) {
      ASSERT(false);
    }
    if (output == NULL) break;
L
Liu Jicong 已提交
161 162 163
    // TODO: do we need free memory?
    SSDataBlock* outputCopy = createOneDataBlock(output, true);
    taosArrayPush(pRes, outputCopy);
L
Liu Jicong 已提交
164 165 166 167 168
  }

  // destroy
  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
    streamDataSubmitRefDec((SStreamDataSubmit*)data);
L
Liu Jicong 已提交
169
    taosFreeQitem(data);
L
Liu Jicong 已提交
170 171
  } else {
    taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
L
Liu Jicong 已提交
172
    taosFreeQitem(data);
L
Liu Jicong 已提交
173 174 175 176
  }
  return 0;
}

L
Liu Jicong 已提交
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
  while (1) {
    void* data = NULL;
    taosGetQitem(pTask->inputQAll, &data);
    if (data == NULL) break;

    streamTaskExecImpl(pTask, data, pRes);

    if (taosArrayGetSize(pRes) != 0) {
      SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
      qRes->type = STREAM_INPUT__DATA_BLOCK;
      qRes->blocks = pRes;
      taosWriteQitem(pTask->outputQ, qRes);
      return taosArrayInit(0, sizeof(SSDataBlock));
    }
  }
  return pRes;
}

L
Liu Jicong 已提交
196
// TODO: handle version
L
Liu Jicong 已提交
197
int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
L
Liu Jicong 已提交
198 199 200 201
  SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
  if (pRes == NULL) return -1;
  while (1) {
    int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
L
Liu Jicong 已提交
202
    void*  exec = pTask->exec.executor;
L
Liu Jicong 已提交
203 204
    if (execStatus == TASK_STATUS__IDLE) {
      // first run, from qall, handle failure from last exec
L
Liu Jicong 已提交
205 206 207
      pRes = streamExecForQall(pTask, pRes);
      if (pRes == NULL) goto FAIL;

L
Liu Jicong 已提交
208 209
      // second run, from inputQ
      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
L
Liu Jicong 已提交
210 211 212
      pRes = streamExecForQall(pTask, pRes);
      if (pRes == NULL) goto FAIL;

L
Liu Jicong 已提交
213 214
      // set status closing
      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
L
Liu Jicong 已提交
215 216

      // third run, make sure inputQ and qall are cleared
L
Liu Jicong 已提交
217
      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
L
Liu Jicong 已提交
218 219
      pRes = streamExecForQall(pTask, pRes);
      if (pRes == NULL) goto FAIL;
L
Liu Jicong 已提交
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236

      atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
      break;
    } else if (execStatus == TASK_STATUS__CLOSING) {
      continue;
    } else if (execStatus == TASK_STATUS__EXECUTING) {
      break;
    } else {
      ASSERT(0);
    }
  }
  return 0;
FAIL:
  atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
  return -1;
}

L
Liu Jicong 已提交
237
int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
L
Liu Jicong 已提交
238
  bool firstRun = 1;
L
Liu Jicong 已提交
239
  while (1) {
L
Liu Jicong 已提交
240 241 242 243 244 245 246 247 248 249 250
    SStreamDataBlock* pBlock = NULL;
    if (!firstRun) {
      taosReadAllQitems(pTask->outputQ, pTask->outputQAll);
    }
    taosGetQitem(pTask->outputQAll, (void**)&pBlock);
    if (pBlock == NULL) {
      if (firstRun) {
        firstRun = 0;
        continue;
      } else {
        break;
L
Liu Jicong 已提交
251 252 253
      }
    }

L
Liu Jicong 已提交
254 255 256
    SArray* pRes = pBlock->blocks;

    // sink
L
Liu Jicong 已提交
257
    if (pTask->sinkType == TASK_SINK__TABLE) {
L
Liu Jicong 已提交
258 259
      // blockDebugShowData(pRes);
      pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
L
Liu Jicong 已提交
260 261
    } else if (pTask->sinkType == TASK_SINK__SMA) {
      pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
L
Liu Jicong 已提交
262 263 264
      //
    } else if (pTask->sinkType == TASK_SINK__FETCH) {
      //
L
Liu Jicong 已提交
265
    } else {
L
Liu Jicong 已提交
266
      ASSERT(pTask->sinkType == TASK_SINK__NONE);
L
Liu Jicong 已提交
267 268 269
    }

    // dispatch
L
Liu Jicong 已提交
270 271 272 273 274 275 276 277 278
    // TODO dispatch guard
    int8_t outputStatus = atomic_load_8(&pTask->outputStatus);
    if (outputStatus == TASK_OUTPUT_STATUS__NORMAL) {
      if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
        SRpcMsg dispatchMsg = {0};
        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
          ASSERT(0);
          return -1;
        }
L
Liu Jicong 已提交
279

L
Liu Jicong 已提交
280
        int32_t qType;
281
        if (pTask->dispatchMsgType == TDMT_VND_TASK_DISPATCH || pTask->dispatchMsgType == TDMT_SND_TASK_DISPATCH) {
L
Liu Jicong 已提交
282
          qType = FETCH_QUEUE;
283 284 285 286 287
          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||*/
          /*pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {*/
          /*qType = MERGE_QUEUE;*/
          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {*/
          /*qType = WRITE_QUEUE;*/
L
Liu Jicong 已提交
288 289 290 291 292 293 294 295 296 297 298 299
        } else {
          ASSERT(0);
        }
        tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);

      } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
        SRpcMsg dispatchMsg = {0};
        SEpSet* pEpSet = NULL;
        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
          ASSERT(0);
          return -1;
        }
L
Liu Jicong 已提交
300

L
Liu Jicong 已提交
301
        tmsgSendReq(pEpSet, &dispatchMsg);
L
Liu Jicong 已提交
302

L
Liu Jicong 已提交
303 304 305 306 307
      } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
        SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
        if (pShuffleRes == NULL) {
          return -1;
        }
L
Liu Jicong 已提交
308

L
Liu Jicong 已提交
309 310 311 312
        int32_t sz = taosArrayGetSize(pRes);
        for (int32_t i = 0; i < sz; i++) {
          SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
          SArray*      pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
L
Liu Jicong 已提交
313
          if (pArray == NULL) {
L
Liu Jicong 已提交
314 315 316 317 318
            pArray = taosArrayInit(0, sizeof(SSDataBlock));
            if (pArray == NULL) {
              return -1;
            }
            taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
L
Liu Jicong 已提交
319
          }
L
Liu Jicong 已提交
320
          taosArrayPush(pArray, pDataBlock);
L
Liu Jicong 已提交
321 322
        }

L
Liu Jicong 已提交
323 324 325
        if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
          return -1;
        }
L
Liu Jicong 已提交
326

L
Liu Jicong 已提交
327 328 329
      } else {
        ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
      }
L
Liu Jicong 已提交
330 331 332 333 334
    }
  }
  return 0;
}

L
Liu Jicong 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
  SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
  int8_t            status;

  // 1.1 update status
  // TODO cal backpressure
  if (pBlock == NULL) {
    atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
    status = TASK_INPUT_STATUS__FAILED;
  } else {
    status = atomic_load_8(&pTask->inputStatus);
  }

  // 1.2 enqueue
  pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
  pBlock->sourceVg = pReq->sourceVg;
L
Liu Jicong 已提交
351
  /*pBlock->sourceVer = pReq->sourceVer;*/
L
Liu Jicong 已提交
352 353 354 355 356
  taosWriteQitem(pTask->inputQ, pBlock);

  // 1.3 rsp by input status
  SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
  pCont->inputStatus = status;
L
Liu Jicong 已提交
357 358
  pCont->streamId = pReq->streamId;
  pCont->taskId = pReq->sourceTaskId;
L
Liu Jicong 已提交
359 360 361
  pRsp->pCont = pCont;
  pRsp->contLen = sizeof(SStreamDispatchRsp);
  tmsgSendRsp(pRsp);
L
Liu Jicong 已提交
362 363 364 365

  return 0;
}

L
Liu Jicong 已提交
366
int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
L
Liu Jicong 已提交
367 368
  // 1. handle input
  streamTaskEnqueue(pTask, pReq, pRsp);
L
Liu Jicong 已提交
369

L
Liu Jicong 已提交
370 371 372 373
  // 2. try exec
  // 2.1. idle: exec
  // 2.2. executing: return
  // 2.3. closing: keep trying
L
Liu Jicong 已提交
374
  streamExec(pTask, pMsgCb);
L
Liu Jicong 已提交
375

L
Liu Jicong 已提交
376 377 378
  // 3. handle output
  // 3.1 check and set status
  // 3.2 dispatch / sink
L
Liu Jicong 已提交
379
  streamSink(pTask, pMsgCb);
L
Liu Jicong 已提交
380 381 382 383

  return 0;
}

L
Liu Jicong 已提交
384
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) {
L
Liu Jicong 已提交
385 386 387 388 389
  atomic_store_8(&pTask->inputStatus, pRsp->inputStatus);
  if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
    // TODO: init recover timer
  }
  // continue dispatch
L
Liu Jicong 已提交
390
  streamSink(pTask, pMsgCb);
L
Liu Jicong 已提交
391 392 393 394
  return 0;
}

int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) {
L
Liu Jicong 已提交
395 396
  streamExec(pTask, pMsgCb);
  streamSink(pTask, pMsgCb);
L
Liu Jicong 已提交
397 398 399
  return 0;
}

L
Liu Jicong 已提交
400
int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
L
Liu Jicong 已提交
401
  //
L
Liu Jicong 已提交
402 403 404
  return 0;
}

L
Liu Jicong 已提交
405
int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
L
Liu Jicong 已提交
406 407 408 409
  //
  return 0;
}

L
Liu Jicong 已提交
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) {
  int32_t tlen = 0;
  tlen += taosEncodeFixedI64(buf, pReq->streamId);
  tlen += taosEncodeFixedI32(buf, pReq->taskId);
  tlen += tEncodeDataBlocks(buf, pReq->data);
  return tlen;
}

void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) {
  buf = taosDecodeFixedI64(buf, &pReq->streamId);
  buf = taosDecodeFixedI32(buf, &pReq->taskId);
  buf = tDecodeDataBlocks(buf, &pReq->data);
  return (void*)buf;
}

void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); }
L
Liu Jicong 已提交
426 427

SStreamTask* tNewSStreamTask(int64_t streamId) {
wafwerar's avatar
wafwerar 已提交
428
  SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
L
Liu Jicong 已提交
429 430 431 432 433
  if (pTask == NULL) {
    return NULL;
  }
  pTask->taskId = tGenIdPI32();
  pTask->streamId = streamId;
L
Liu Jicong 已提交
434 435
  pTask->status = TASK_STATUS__IDLE;

L
Liu Jicong 已提交
436 437 438
  return pTask;
}

H
Hongze Cheng 已提交
439
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
L
Liu Jicong 已提交
440 441 442
  /*if (tStartEncode(pEncoder) < 0) return -1;*/
  if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
  if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
L
Liu Jicong 已提交
443
  if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
L
Liu Jicong 已提交
444 445 446 447 448 449 450
  if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
  if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
  if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
  if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
  if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
  if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;

L
Liu Jicong 已提交
451 452 453 454
  if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
  if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;

  if (pTask->execType != TASK_EXEC__NONE) {
L
Liu Jicong 已提交
455 456 457 458
    if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1;
    if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1;
  }

L
Liu Jicong 已提交
459
  if (pTask->sinkType == TASK_SINK__TABLE) {
L
Liu Jicong 已提交
460
    if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1;
L
Liu Jicong 已提交
461
    if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1;
L
Liu Jicong 已提交
462
    if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
L
Liu Jicong 已提交
463
  } else if (pTask->sinkType == TASK_SINK__SMA) {
L
Liu Jicong 已提交
464
    if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1;
L
Liu Jicong 已提交
465 466 467 468
  } else if (pTask->sinkType == TASK_SINK__FETCH) {
    if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1;
  } else {
    ASSERT(pTask->sinkType == TASK_SINK__NONE);
L
Liu Jicong 已提交
469 470 471
  }

  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
L
Liu Jicong 已提交
472
    if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1;
L
Liu Jicong 已提交
473
  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
L
Liu Jicong 已提交
474
    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1;
L
Liu Jicong 已提交
475 476 477
    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1;
    if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
L
Liu Jicong 已提交
478 479
    if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
    /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
L
Liu Jicong 已提交
480 481 482 483 484 485
  }

  /*tEndEncode(pEncoder);*/
  return pEncoder->pos;
}

H
Hongze Cheng 已提交
486
int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
L
Liu Jicong 已提交
487 488 489
  /*if (tStartDecode(pDecoder) < 0) return -1;*/
  if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
  if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
L
Liu Jicong 已提交
490
  if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
L
Liu Jicong 已提交
491 492 493 494 495 496 497
  if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
  if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
  if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
  if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
  if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
  if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;

L
Liu Jicong 已提交
498 499 500 501
  if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
  if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;

  if (pTask->execType != TASK_EXEC__NONE) {
L
Liu Jicong 已提交
502 503 504 505
    if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1;
    if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1;
  }

L
Liu Jicong 已提交
506
  if (pTask->sinkType == TASK_SINK__TABLE) {
L
Liu Jicong 已提交
507
    if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1;
L
Liu Jicong 已提交
508
    if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1;
L
Liu Jicong 已提交
509 510 511
    pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
    if (pTask->tbSink.pSchemaWrapper == NULL) return -1;
    if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
L
Liu Jicong 已提交
512
  } else if (pTask->sinkType == TASK_SINK__SMA) {
L
Liu Jicong 已提交
513
    if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1;
L
Liu Jicong 已提交
514 515 516 517
  } else if (pTask->sinkType == TASK_SINK__FETCH) {
    if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1;
  } else {
    ASSERT(pTask->sinkType == TASK_SINK__NONE);
L
Liu Jicong 已提交
518 519 520
  }

  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
L
Liu Jicong 已提交
521
    if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1;
L
Liu Jicong 已提交
522
  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
L
Liu Jicong 已提交
523
    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1;
L
Liu Jicong 已提交
524 525 526
    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1;
    if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
L
Liu Jicong 已提交
527 528
    /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
    if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
L
Liu Jicong 已提交
529 530 531 532 533 534 535
  }

  /*tEndDecode(pDecoder);*/
  return 0;
}

void tFreeSStreamTask(SStreamTask* pTask) {
L
Liu Jicong 已提交
536 537
  taosCloseQueue(pTask->inputQ);
  taosCloseQueue(pTask->outputQ);
L
Liu Jicong 已提交
538
  // TODO
L
Liu Jicong 已提交
539
  if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
L
Liu Jicong 已提交
540
  qDestroyTask(pTask->exec.executor);
L
Liu Jicong 已提交
541
  taosMemoryFree(pTask);
L
Liu Jicong 已提交
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
}

#if 0
int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) {
  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
  /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/
  return pEncoder->size;
}
int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) {
  return pEncoder->size;
}
void    tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) {
  taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock);
}
#endif