diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index 55a56d685f94ea83d0ae640e7e0b6bd40fc4e8ed..e92a68c8c174d13f244f0b210f63d1eefd78d853 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -169,6 +169,10 @@ export default { title: 'DVC', link: '/en-us/docs/dev/user_doc/guide/task/dvc.html', }, + { + title: 'Dinky', + link: '/en-us/docs/dev/user_doc/guide/task/dinky.html', + }, ], }, { @@ -553,6 +557,10 @@ export default { title: 'DVC', link: '/zh-cn/docs/dev/user_doc/guide/task/dvc.html', }, + { + title: 'Dinky', + link: '/zh-cn/docs/dev/user_doc/guide/task/dinky.html', + }, ], }, { diff --git a/docs/docs/en/guide/task/dinky.md b/docs/docs/en/guide/task/dinky.md new file mode 100644 index 0000000000000000000000000000000000000000..be44e29a692236bbe57bcd7eb5043a7017c31bf0 --- /dev/null +++ b/docs/docs/en/guide/task/dinky.md @@ -0,0 +1,40 @@ +# Dinky + +## Overview + +Use `Dinky Task` to create a dinky-type task and support one-stop development, debugging, operation and maintenance of FlinkSql, Flink jar and SQL. When the worker executes `Dinky Task`, +it will call `Dinky API` to trigger dinky task. Click [here](http://www.dlink.top/) for details about `Dinky`. + +## Create Task + +- Click Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page. +- Drag from the toolbar to the canvas. + +## Task Parameter + +| **Parameter** | **Description** | +| ------- | ---------- | +| Node Name | Set the name of the task. Node names within a workflow definition are unique. | +| Run flag | Indicates whether the node can be scheduled normally. If it is not necessary to execute, you can turn on the prohibiting execution switch. | +| Description | Describes the function of this node. | +| Task priority | When the number of worker threads is insufficient, they are executed in order from high to low according to the priority, and they are executed according to the first-in, first-out principle when the priority is the same. | +| Worker group | The task is assigned to the machines in the worker group for execution. If Default is selected, a worker machine will be randomly selected for execution. | +| Task group name | The group in Resources, if not configured, it will not be used. | +| Environment Name | Configure the environment in which to run the script. | +| Number of failed retries | The number of times the task is resubmitted after failure. It supports drop-down and manual filling. | +| Failure Retry Interval | The time interval for resubmitting the task if the task fails. It supports drop-down and manual filling. | +| Timeout alarm | Check Timeout Alarm and Timeout Failure. When the task exceeds the "timeout duration", an alarm email will be sent and the task execution will fail. | +| Dinky Address | The url for a dinky server. | +| Dinky Task ID | The unique task id for a dinky task. | +| Online Task | Specify whether the current dinky job is online. If yes, the submitted job can only be submitted successfully when it is published and there is no corresponding Flink job instance running. | + +## Task Example + +### Dinky Task Example + +This example illustrates how to create a dinky task node. + +![demo-dinky](../../../../img/tasks/demo/dinky.png) + +![demo-get-dinky-task-id](../../../../img/tasks/demo/dinky_task_id.png) + diff --git a/docs/docs/zh/guide/task/dinky.md b/docs/docs/zh/guide/task/dinky.md new file mode 100644 index 0000000000000000000000000000000000000000..e4741296b8578dfa4063ecb9c073c1a095db8f9a --- /dev/null +++ b/docs/docs/zh/guide/task/dinky.md @@ -0,0 +1,40 @@ +# Dinky + +## Overview + +`Dinky`任务类型，用于创建并执行`Dinky`类型任务以支撑一站式的开发、调试、运维 FlinkSQL、Flink Jar、SQL。worker 执行该任务的时候，会通过`Dinky API`触发`Dinky 的作业`。 +点击[这里](http://www.dlink.top/) 获取更多关于`Dinky`的信息。 + +## Create Task + +- 点击项目管理-项目名称-工作流定义，点击"创建工作流"按钮，进入DAG编辑页面。 +- 工具栏中拖动 到画板中，即可完成创建。 + +## Task Parameter + +| **参数** | **描述** | +|-------------|--------------------------------------------------------------------| +| 任务名称 | 设置任务的名称。一个工作流定义中的节点名称是唯一的。 | +| 运行标志 | 标识这个节点是否可以正常调度。如果不需要执行，可以打开禁止执行开关。 | +| 描述 | 描述该节点的功能。 | +| 任务优先级 | worker 线程数不足时，根据优先级从高到低依次执行，优先级一样时根据先进先出原则执行。 | +| Worker 分组 | 任务分配给 worker 组的机器机执行，选择 Default，会随机选择一台 worker 机执行。 | +| 任务组名 | 资源中的组，如果未配置，将不会使用。 | +| 环境名称 | 配置运行脚本的环境。 | +| 失败重试次数 | 任务失败重新提交的次数，支持下拉和手填。 | +| 失败重试间隔 | 任务失败重新提交任务的时间间隔，支持下拉和手填。 | +| 超时告警 | 勾选超时告警、超时失败，当任务超过"超时时长"后，会发送告警邮件并且任务执行失败. | +| Dinky 地址 | Dinky 服务的 url。 | +| Dinky 任务 ID | Dinky 作业对应的唯一ID。 | +| 上线作业 | 指定当前 Dinky 作业是否上线，如果是，则该被提交的作业只能处于已发布且当前无对应的 Flink Job 实例在运行才可提交成功。 | + +## Task Example + +### Dinky Task Example + +这个示例展示了如何创建 Dinky 任务节点： + +![demo-dinky](../../../../img/tasks/demo/dinky.png) + +![demo-get-dinky-task-id](../../../../img/tasks/demo/dinky_task_id.png) + diff --git a/docs/img/tasks/demo/dinky.png b/docs/img/tasks/demo/dinky.png new file mode 100644 index 0000000000000000000000000000000000000000..a2027faeab090a3a1589a32d35b8e8975e03a83e Binary files /dev/null and b/docs/img/tasks/demo/dinky.png differ diff --git a/docs/img/tasks/demo/dinky_task_id.png b/docs/img/tasks/demo/dinky_task_id.png new file mode 100644 index 0000000000000000000000000000000000000000..280da5f30f16d3b8714a80f1dfd96dd6f2b962c2 Binary files /dev/null and b/docs/img/tasks/demo/dinky_task_id.png differ diff --git a/docs/img/tasks/icons/dinky.png b/docs/img/tasks/icons/dinky.png new file mode 100644 index 0000000000000000000000000000000000000000..7f4ad3997b65d74c7711f361ef09755035410001 Binary files /dev/null and b/docs/img/tasks/icons/dinky.png differ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index 7a9d8d91ae0a41b0e2b46b2a80cd7921aba19822..759b94505e9d651aa4679c5c443885d780cfc1ef 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -177,6 +177,12 @@ dolphinscheduler-task-dvc ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-task-dinky + ${project.version} + \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..ec3259c0bcdf05a3a964a60413419dc604fd083e --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml @@ -0,0 +1,50 @@ + + + + + dolphinscheduler-task-plugin + org.apache.dolphinscheduler + dev-SNAPSHOT + + 4.0.0 + jar + + dolphinscheduler-task-dinky + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java new file mode 100644 index 0000000000000000000000000000000000000000..2b0d6c1d327135c0394c7acd1ad1a0780796c236 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.Collections; +import java.util.List; + +public class DinkyParameters extends AbstractParameters { + + /** + * parameters for dinky Open API + * + * @see Dinky_Open_API + */ + private String address; + private String taskId; + private boolean online = false; + + @Override + public boolean checkParameters() { + return StringUtils.isNotEmpty(this.address) && StringUtils.isNotEmpty(this.taskId); + } + + @Override + public List getResourceFilesList() { + return Collections.emptyList(); + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public boolean isOnline() { + return online; + } + + public void setOnline(boolean online) { + this.online = online; + } + + @Override + public String toString() { + return "DinkyParameters{" + + "address='" + address + '\'' + + ", taskId='" + taskId + '\'' + + ", online='" + online + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTask.java new file mode 100644 index 0000000000000000000000000000000000000000..622a67478023c45377e07a42cdb3125f16886e50 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTask.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.MissingNode; + +public class DinkyTask extends AbstractTaskExecutor { + + /** + * taskExecutionContext + */ + private final TaskExecutionContext taskExecutionContext; + + /** + * dinky parameters + */ + private DinkyParameters dinkyParameters; + + /** + * constructor + * + * @param taskExecutionContext taskExecutionContext + */ + protected DinkyTask(TaskExecutionContext taskExecutionContext) { + super(taskExecutionContext); + this.taskExecutionContext = taskExecutionContext; + } + + @Override + public void init() { + final String taskParams = taskExecutionContext.getTaskParams(); + logger.info("dinky task params:{}", taskParams); + this.dinkyParameters = JSONUtils.parseObject(taskParams, DinkyParameters.class); + if (this.dinkyParameters == null || !this.dinkyParameters.checkParameters()) { + throw new DinkyTaskException("dinky task params is not valid"); + } + } + + @Override + public void handle() throws Exception { + String address = this.dinkyParameters.getAddress(); + String taskId = this.dinkyParameters.getTaskId(); + boolean isOnline = this.dinkyParameters.isOnline(); + JsonNode result; + if (isOnline) { + // Online dinky task, and only one job is allowed to execute + result = onlineTask(address, taskId); + } else { + // Submit dinky task + result = submitTask(address, taskId); + } + if (checkResult(result)) { + boolean status = result.get(DinkyTaskConstants.API_RESULT_DATAS).get("success").asBoolean(); + String jobInstanceId = result.get(DinkyTaskConstants.API_RESULT_DATAS).get("jobInstanceId").asText(); + boolean finishFlag = false; + while (!finishFlag) { + JsonNode jobInstanceInfoResult = getJobInstanceInfo(address, jobInstanceId); + if (!checkResult(jobInstanceInfoResult)) { + break; + } + String jobInstanceStatus = jobInstanceInfoResult.get(DinkyTaskConstants.API_RESULT_DATAS).get("status").asText(); + switch (jobInstanceStatus) { + case DinkyTaskConstants.STATUS_FINISHED: + final int exitStatusCode = mapStatusToExitCode(status); + // Use address-taskId as app id + setAppIds(String.format("%s-%s", address, taskId)); + setExitStatusCode(exitStatusCode); + logger.info("dinky task finished with results: {}", result.get(DinkyTaskConstants.API_RESULT_DATAS)); + finishFlag = true; + break; + case DinkyTaskConstants.STATUS_FAILED: + case DinkyTaskConstants.STATUS_CANCELED: + case DinkyTaskConstants.STATUS_UNKNOWN: + errorHandle(jobInstanceInfoResult.get(DinkyTaskConstants.API_RESULT_DATAS).get("error").asText()); + finishFlag = true; + break; + default: + Thread.sleep(DinkyTaskConstants.SLEEP_MILLIS); + } + } + } + } + + /** + * map dinky task status to exitStatusCode + * + * @param status dinky job status + * @return exitStatusCode + */ + private int mapStatusToExitCode(boolean status) { + if (status) { + return TaskConstants.EXIT_CODE_SUCCESS; + } else { + return TaskConstants.EXIT_CODE_FAILURE; + } + } + + private boolean checkResult(JsonNode result) { + if (result instanceof MissingNode || result == null) { + errorHandle(DinkyTaskConstants.API_VERSION_ERROR_TIPS); + return false; + } else if (result.get("code").asInt() == DinkyTaskConstants.API_ERROR) { + errorHandle(result.get("msg")); + return false; + } + return true; + } + + private void errorHandle(Object msg) { + setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); + logger.error("dinky task submit failed with error: {}", msg); + } + + @Override + public AbstractParameters getParameters() { + return dinkyParameters; + } + + @Override + public void cancelApplication(boolean status) throws Exception { + super.cancelApplication(status); + String address = this.dinkyParameters.getAddress(); + String taskId = this.dinkyParameters.getTaskId(); + logger.info("trying terminate dinky task, taskId: {}, address: {}, taskId: {}", + this.taskExecutionContext.getTaskInstanceId(), + address, + taskId); + cancelTask(address, taskId); + logger.warn("dinky task terminated, taskId: {}, address: {}, taskId: {}", + this.taskExecutionContext.getTaskInstanceId(), + address, + taskId); + } + + private JsonNode submitTask(String address, String taskId) { + Map params = new HashMap<>(); + params.put(DinkyTaskConstants.PARAM_TASK_ID, taskId); + return parse(doGet(address + DinkyTaskConstants.SUBMIT_TASK, params)); + } + + private JsonNode onlineTask(String address, String taskId) { + Map params = new HashMap<>(); + params.put(DinkyTaskConstants.PARAM_TASK_ID, taskId); + return parse(doGet(address + DinkyTaskConstants.ONLINE_TASK, params)); + } + + private JsonNode cancelTask(String address, String taskId) { + Map params = new HashMap<>(); + params.put(DinkyTaskConstants.PARAM_JSON_TASK_ID, taskId); + params.put(DinkyTaskConstants.PARAM_SAVEPOINT_TYPE, DinkyTaskConstants.SAVEPOINT_CANCEL); + return parse(sendJsonStr(address + DinkyTaskConstants.SAVEPOINT_TASK, JSONUtils.toJsonString(params))); + } + + private JsonNode getJobInstanceInfo(String address, String taskId) { + Map params = new HashMap<>(); + params.put(DinkyTaskConstants.PARAM_JOB_INSTANCE_ID, taskId); + return parse(doGet(address + DinkyTaskConstants.GET_JOB_INFO, params)); + } + + private JsonNode parse(String res) { + ObjectMapper mapper = new ObjectMapper(); + JsonNode result = null; + try { + result = mapper.readTree(res); + } catch (JsonProcessingException e) { + logger.error("dinky task submit failed with error", e); + } + return result; + } + + private String doGet(String url, Map params) { + String result = ""; + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpGet httpGet = null; + try { + URIBuilder uriBuilder = new URIBuilder(url); + if (null != params && !params.isEmpty()) { + for (Map.Entry entry : params.entrySet()) { + uriBuilder.addParameter(entry.getKey(), entry.getValue()); + } + } + URI uri = uriBuilder.build(); + httpGet = new HttpGet(uri); + logger.info("access url: {}", uri); + HttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + result = EntityUtils.toString(response.getEntity()); + logger.info("dinky task succeed with results: {}", result); + } else { + logger.error("dinky task terminated,response: {}", response); + } + } catch (IllegalArgumentException ie) { + logger.error("dinky task terminated: {}", ie.getMessage()); + } catch (Exception e) { + logger.error("dinky task terminated: ", e); + } finally { + if (null != httpGet) { + httpGet.releaseConnection(); + } + } + return result; + } + + private String sendJsonStr(String url, String params) { + String result = ""; + HttpClient httpClient = HttpClientBuilder.create().build(); + HttpPost httpPost = new HttpPost(url); + try { + httpPost.addHeader("Content-type", "application/json; charset=utf-8"); + httpPost.setHeader("Accept", "application/json"); + if (StringUtils.isNotBlank(params)) { + httpPost.setEntity(new StringEntity(params, StandardCharsets.UTF_8)); + } + HttpResponse response = httpClient.execute(httpPost); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + result = EntityUtils.toString(response.getEntity()); + logger.info("dinky task succeed with results: {}", result); + } else { + logger.error("dinky task terminated,response: {}", response); + } + } catch (IllegalArgumentException ie) { + logger.error("dinky task terminated: {}", ie.getMessage()); + } catch (Exception he) { + logger.error("dinky task terminated: ", he); + } + return result; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java new file mode 100644 index 0000000000000000000000000000000000000000..6d419b3dbb2b8473313e3c04172747317d844633 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.spi.utils.JSONUtils; + +public class DinkyTaskChannel implements TaskChannel { + + @Override + public void cancelApplication(boolean status) { + // nothing to do + } + + @Override + public AbstractTask createTask(TaskExecutionContext taskRequest) { + return new DinkyTask(taskRequest); + } + + @Override + public AbstractParameters parseParameters(ParametersNode parametersNode) { + return JSONUtils.parseObject(parametersNode.getTaskParams(), DinkyParameters.class); + } + + @Override + public ResourceParametersHelper getResources(String parameters) { + return null; + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..6c80a9769d14e5f6e259f3359f6013c784e340a1 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +import com.google.auto.service.AutoService; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; +import org.apache.dolphinscheduler.spi.params.base.PluginParams; + +import java.util.ArrayList; +import java.util.List; + +@AutoService(TaskChannelFactory.class) +public class DinkyTaskChannelFactory implements TaskChannelFactory { + @Override + public String getName() { + return "DINKY"; + } + + @Override + public List getParams() { + return new ArrayList<>(); + } + + @Override + public TaskChannel create() { + return new DinkyTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..847d4e3a31ac1144acbda763a5aa1c6d03ddf5f2 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +/** + * Custom DinkyTaskConstants + */ +public class DinkyTaskConstants { + + private DinkyTaskConstants() { + throw new IllegalStateException("Utility class"); + } + + private static final String API_ROUTE = "/openapi/"; + public static final String SUBMIT_TASK = API_ROUTE + "submitTask"; + public static final String ONLINE_TASK = API_ROUTE + "onLineTask"; + public static final String SAVEPOINT_TASK = API_ROUTE + "savepointTask"; + public static final String GET_JOB_INFO = API_ROUTE + "getJobInstance"; + public static final int API_ERROR = 1; + public static final String API_VERSION_ERROR_TIPS = "Please check that the dinky version is greater than or equal to 0.6.5"; + public static final String API_RESULT_DATAS = "datas"; + + public static final String SAVEPOINT_CANCEL = "cancel"; + + public static final String PARAM_TASK_ID = "id"; + public static final String PARAM_JSON_TASK_ID = "taskId"; + public static final String PARAM_SAVEPOINT_TYPE = "type"; + public static final String PARAM_JOB_INSTANCE_ID = "id"; + + public static final String STATUS_FINISHED = "FINISHED"; + public static final String STATUS_CANCELED = "CANCELED"; + public static final String STATUS_FAILED = "FAILED"; + public static final String STATUS_UNKNOWN = "UNKNOWN"; + + public static final long SLEEP_MILLIS = 3000; + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskException.java new file mode 100644 index 0000000000000000000000000000000000000000..86393bf3a6be051748cb4b1bc5c0fa93163c9185 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.dinky; + +/** + * Custom DinkyTaskException + */ +public class DinkyTaskException extends RuntimeException { + + public DinkyTaskException() { + super(); + } + + public DinkyTaskException(String message) { + super(message); + } + + public DinkyTaskException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index f40e882d47463a8160d6e76cafba6188058a7dcb..49f874d904021a208ede0aec5a3ef164511f568d 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -56,5 +56,6 @@ dolphinscheduler-task-mlflow dolphinscheduler-task-openmldb dolphinscheduler-task-dvc + dolphinscheduler-task-dinky \ No newline at end of file diff --git a/dolphinscheduler-ui/public/images/task-icons/dinky.png b/dolphinscheduler-ui/public/images/task-icons/dinky.png new file mode 100644 index 0000000000000000000000000000000000000000..7f4ad3997b65d74c7711f361ef09755035410001 Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/dinky.png differ diff --git a/dolphinscheduler-ui/public/images/task-icons/dinky_hover.png b/dolphinscheduler-ui/public/images/task-icons/dinky_hover.png new file mode 100644 index 0000000000000000000000000000000000000000..1db1980ce9146d0bbdd34f9dec9cbd0f0218bd4f Binary files /dev/null and b/dolphinscheduler-ui/public/images/task-icons/dinky_hover.png differ diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 93e643690f343b2f1efce3b2fa9093458e2178e9..b5514c8a75fdd53ad3a7597f815dd0127e5b58a3 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -706,6 +706,11 @@ export default { 'Please select column, only single column is supported', please_enter_threshold_number_is_needed: 'Please enter threshold number is needed', - please_enter_comparison_title: 'please select comparison title' + please_enter_comparison_title: 'please select comparison title', + dinky_address: 'Dinky address', + dinky_address_tips: 'Please enter the url of your dinky', + dinky_task_id: 'Dinky task id', + dinky_task_id_tips: 'Please enter the task id of your dinky', + dinky_online: 'Online task' } } diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index baed96432ef988e1fe4ec3f86ca6101be99657c2..1def17bf06103cd93a935cc0583ffa8e94b4521d 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -692,6 +692,11 @@ export default { please_enter_filter_expression: '请输入源表过滤条件', please_enter_column_only_single_column_is_supported: '请选择源表检测列', please_enter_threshold_number_is_needed: '请输入阈值', - please_enter_comparison_title: '请选择期望值类型' + please_enter_comparison_title: '请选择期望值类型', + dinky_address: 'dinky 地址', + dinky_address_tips: '请输入 Dinky 地址', + dinky_task_id: 'dinky 作业ID', + dinky_task_id_tips: '请输入作业 ID', + dinky_online: '是否上线作业' } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts index 971fd0447bc21ef6a51a2cd4bfbee6a517c2383f..30541292145f6007ee49ff80372627f1e4fad4f5 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/index.ts @@ -71,3 +71,4 @@ export { useMlflowProjects } from './use-mlflow-projects' export { useMlflowModels } from './use-mlflow-models' export { useOpenmldb } from './use-openmldb' export { useDvc } from './use-dvc' +export { useDinky } from './use-dinky' diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-dinky.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-dinky.ts new file mode 100644 index 0000000000000000000000000000000000000000..d043b2209d3b5ff9741551d5bc67f4bd21cfaa5d --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-dinky.ts @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useI18n } from 'vue-i18n' +import { useCustomParams } from '.' +import type { IJsonItem } from '../types' + +export function useDinky(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + + return [ + { + type: 'input', + field: 'address', + name: t('project.node.dinky_address'), + props: { + placeholder: t('project.node.dinky_address_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(_validate: any, value: string) { + if (!value) { + return new Error(t('project.node.dinky_address_tips')) + } + } + } + }, + { + type: 'input', + field: 'taskId', + name: t('project.node.dinky_task_id'), + props: { + placeholder: t('project.node.dinky_task_id_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(_validate: any, value: string) { + if (!value) { + return new Error(t('project.node.dinky_task_id_tips')) + } + } + } + }, + { + type: 'switch', + field: 'online', + name: t('project.node.dinky_online') + }, + ...useCustomParams({ model, field: 'localParams', isSimple: false }) + ] +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index ae5a2efbae51b75c7a0b54d31a6d0ff43e910d97..aea00972953773b2d4aff682ccb733f6f38caced 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -360,7 +360,6 @@ export function formatParams(data: INodeData): { } if (data.taskType === 'DVC') { - taskParams.dvcTaskType = data.dvcTaskType taskParams.dvcRepository = data.dvcRepository taskParams.dvcVersion = data.dvcVersion @@ -370,6 +369,12 @@ export function formatParams(data: INodeData): { taskParams.dvcStoreUrl = data.dvcStoreUrl } + if (data.taskType === 'DINKY') { + taskParams.address = data.address + taskParams.taskId = data.taskId + taskParams.online = data.online + } + if (data.taskType === 'OPENMLDB') { taskParams.zk = data.zk taskParams.zkPath = data.zkPath diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts index d2a5a6dd0174c500fafb2f5d3b30f182356e36f4..4240892efff275b91bb237c6b4c528b48681b4ca 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/index.ts @@ -39,6 +39,7 @@ import { useJupyter } from './use-jupyter' import { useMlflow } from './use-mlflow' import { useOpenmldb } from './use-openmldb' import { useDvc } from './use-dvc' +import { useDinky } from './use-dinky' export default { SHELL: useShell, @@ -64,5 +65,6 @@ export default { JUPYTER: useJupyter, MLFLOW: useMlflow, OPENMLDB: useOpenmldb, - DVC: useDvc + DVC: useDvc, + DINKY: useDinky } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-dinky.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-dinky.ts new file mode 100644 index 0000000000000000000000000000000000000000..5e7a0aa0cb4154649957de2f4fc432e2ea41a020 --- /dev/null +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-dinky.ts @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive } from 'vue' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData, ITaskData } from '../types' + +export function useDinky({ + projectCode, + from = 0, + readonly, + data +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData +}) { + const model = reactive({ + name: '', + taskType: 'DINKY', + flag: 'YES', + description: '', + timeoutFlag: false, + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + delayTime: 0, + timeout: 30, + timeoutNotifyStrategy: ['WARN'] + } as INodeData) + + let extra: IJsonItem[] = [] + if (from === 1) { + extra = [ + Fields.useTaskType(model, readonly), + Fields.useProcessName({ + model, + projectCode, + isCreate: !data?.id, + from, + processName: data?.processName + }) + ] + } + + return { + json: [ + Fields.useName(from), + ...extra, + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(), + Fields.useEnvironmentName(model, !model.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useDinky(model), + Fields.usePreTasks() + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 7ceca70723d0f5268c74d7454eceb427bff3041c..06a5c4adfe4ceb9bc439855c5eb29534344a7ec6 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -348,6 +348,9 @@ interface ITaskParams { dvcMessage?: string dvcLoadSaveDataPath?: string dvcStoreUrl?: string + address?: string + taskId?: string + online?: boolean } interface INodeData diff --git a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts index ebd3f741e126f3e9c835b977da46d4b29b277658..1cc0e00c5bd4496fd3c988769840c2c46799b955 100644 --- a/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/constants/task-type.ts @@ -39,6 +39,7 @@ export type TaskType = | 'MLFLOW' | 'OPENMLDB' | 'DVC' + | 'DINKY' export const TASK_TYPES_MAP = { SHELL: { @@ -123,5 +124,9 @@ export const TASK_TYPES_MAP = { DVC: { alias: 'DVC', helperLinkDisable: true + }, + DINKY: { + alias: 'DINKY', + helperLinkDisable: true } } as { [key in TaskType]: { alias: string; helperLinkDisable?: boolean } } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss index 656e5ae890e2d386dba18be674c7680076973718..90615483fddec4f419ef09d709a65b29792152e8 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag.module.scss @@ -173,6 +173,9 @@ $bgLight: #ffffff; &.icon-dvc { background-image: url('/images/task-icons/dvc.png'); } + &.icon-dinky { + background-image: url('/images/task-icons/dinky.png'); + } } &:hover { @@ -249,6 +252,9 @@ $bgLight: #ffffff; &.icon-dvc { background-image: url('/images/task-icons/dvc_hover.png'); } + &.icon-dinky { + background-image: url('/images/task-icons/dinky_hover.png'); + } } } } diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx index 0856c60953122c768ce3e60e55766b8717cd5ee6..2edf0c862697c3b6bee3ec0daf38aa9faa28625e 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/tree/index.tsx @@ -133,6 +133,11 @@ export default defineComponent({ color: '#8c8c8f', image: `${import.meta.env.BASE_URL}images/task-icons/seatunnel.png` }, + { + taskType: 'DINKY', + color: '#d69f5b', + image: `${import.meta.env.BASE_URL}images/task-icons/dinky.png` + }, { taskType: 'DAG', color: '#bbdde9' } ])