Skip to content

DolphinScheduler
DolphinScheduler

pentaLiker / DolphinScheduler
与 Fork 源项目一致

Fork自 apache / DolphinScheduler

通知 1
Star 1
Fork 0
DolphinScheduler
DolphinScheduler

体验新版 GitCode，发现更多精彩内容 >>

提交 dad94369 编写于 作者: C CalvinKirs
浏览文件
操作

Merge branch 'dev' into rpc

上级 fba133c9 66b8288d
展开全部 隐藏空白更改
内联 并排
Showing with 3805 addition and 2252 deletion
.github/ISSUE_TEMPLATE/bug_report.md
浏览文件 @ dad94369
......@@ -2,7 +2,6 @@
name: Bug report
about: Create a report to help us improve
title: "[Bug][Module Name] Bug title "
labels: bug
assignees: ''
---
......
.github/ISSUE_TEMPLATE/feature_request.md
浏览文件 @ dad94369
......@@ -2,7 +2,6 @@
name: Feature request
about: Suggest an idea for this project
title: "[Feature][Module Name] Feature title"
labels: new feature
assignees: ''
---
......
.github/ISSUE_TEMPLATE/improvement_suggestion.md
浏览文件 @ dad94369
......@@ -2,7 +2,6 @@
name: Improvement suggestion
about: Improvement suggestion for this project
title: "[Improvement][Module Name] Improvement title"
labels: improvement
assignees: ''
---
......
.github/ISSUE_TEMPLATE/question.md
浏览文件 @ dad94369
......@@ -2,7 +2,6 @@
name: Question
about: Have a question wanted to be help
title: "[Question] Question title"
labels: question
assignees: ''
---
......
.gitignore
浏览文件 @ dad94369
......@@ -4,15 +4,9 @@
.zip
.gz
.DS_Store
.idea
.idea/
.idea/*
.target
.target/
**/**/target/**
target/*
*/target
*/target/*
.idea/
target/
.settings
.nbproject
.classpath
......@@ -28,123 +22,23 @@ node_modules
npm-debug.log
.vscode
logs/*
.mvn/
.www
t.*
.factorypath
Chart.lock
yarn.lock
package-lock.json
config.gypi
test/coverage
/docs/zh_CN/介绍
/docs/zh_CN/贡献代码.md
/dolphinscheduler-common/src/main/resources/zookeeper.properties
dolphinscheduler-common/src/main/resources/zookeeper.properties
dolphinscheduler-dao/src/main/resources/dao/data_source.properties
dolphinscheduler-alert/logs/
dolphinscheduler-alert/src/main/resources/alert.properties_bak
dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist
dolphinscheduler-ui/node
dolphinscheduler-ui/dist/css/common.16ac5d9.css
dolphinscheduler-ui/dist/css/home/index.b444b91.css
dolphinscheduler-ui/dist/css/login/index.5866c64.css
dolphinscheduler-ui/dist/js/0.ac94e5d.js
dolphinscheduler-ui/dist/js/0.ac94e5d.js.map
dolphinscheduler-ui/dist/js/1.0b043a3.js
dolphinscheduler-ui/dist/js/1.0b043a3.js.map
dolphinscheduler-ui/dist/js/10.1bce3dc.js
dolphinscheduler-ui/dist/js/10.1bce3dc.js.map
dolphinscheduler-ui/dist/js/11.79f04d8.js
dolphinscheduler-ui/dist/js/11.79f04d8.js.map
dolphinscheduler-ui/dist/js/12.420daa5.js
dolphinscheduler-ui/dist/js/12.420daa5.js.map
dolphinscheduler-ui/dist/js/13.e5bae1c.js
dolphinscheduler-ui/dist/js/13.e5bae1c.js.map
dolphinscheduler-ui/dist/js/14.f2a0dca.js
dolphinscheduler-ui/dist/js/14.f2a0dca.js.map
dolphinscheduler-ui/dist/js/15.45373e8.js
dolphinscheduler-ui/dist/js/15.45373e8.js.map
dolphinscheduler-ui/dist/js/16.fecb0fc.js
dolphinscheduler-ui/dist/js/16.fecb0fc.js.map
dolphinscheduler-ui/dist/js/17.84be279.js
dolphinscheduler-ui/dist/js/17.84be279.js.map
dolphinscheduler-ui/dist/js/18.307ea70.js
dolphinscheduler-ui/dist/js/18.307ea70.js.map
dolphinscheduler-ui/dist/js/19.144db9c.js
dolphinscheduler-ui/dist/js/19.144db9c.js.map
dolphinscheduler-ui/dist/js/2.8b4ef29.js
dolphinscheduler-ui/dist/js/2.8b4ef29.js.map
dolphinscheduler-ui/dist/js/20.4c527e9.js
dolphinscheduler-ui/dist/js/20.4c527e9.js.map
dolphinscheduler-ui/dist/js/21.831b2a2.js
dolphinscheduler-ui/dist/js/21.831b2a2.js.map
dolphinscheduler-ui/dist/js/22.2b4bb2a.js
dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map
dolphinscheduler-ui/dist/js/23.81467ef.js
dolphinscheduler-ui/dist/js/23.81467ef.js.map
dolphinscheduler-ui/dist/js/24.54a00e4.js
dolphinscheduler-ui/dist/js/24.54a00e4.js.map
dolphinscheduler-ui/dist/js/25.8d7bd36.js
dolphinscheduler-ui/dist/js/25.8d7bd36.js.map
dolphinscheduler-ui/dist/js/26.2ec5e78.js
dolphinscheduler-ui/dist/js/26.2ec5e78.js.map
dolphinscheduler-ui/dist/js/27.3ab48c2.js
dolphinscheduler-ui/dist/js/27.3ab48c2.js.map
dolphinscheduler-ui/dist/js/28.363088a.js
dolphinscheduler-ui/dist/js/28.363088a.js.map
dolphinscheduler-ui/dist/js/29.6c5853a.js
dolphinscheduler-ui/dist/js/29.6c5853a.js.map
dolphinscheduler-ui/dist/js/3.a0edb5b.js
dolphinscheduler-ui/dist/js/3.a0edb5b.js.map
dolphinscheduler-ui/dist/js/30.940fdd3.js
dolphinscheduler-ui/dist/js/30.940fdd3.js.map
dolphinscheduler-ui/dist/js/31.168a460.js
dolphinscheduler-ui/dist/js/31.168a460.js.map
dolphinscheduler-ui/dist/js/32.8df6594.js
dolphinscheduler-ui/dist/js/32.8df6594.js.map
dolphinscheduler-ui/dist/js/33.4480bbe.js
dolphinscheduler-ui/dist/js/33.4480bbe.js.map
dolphinscheduler-ui/dist/js/34.b407fe1.js
dolphinscheduler-ui/dist/js/34.b407fe1.js.map
dolphinscheduler-ui/dist/js/35.f340b0a.js
dolphinscheduler-ui/dist/js/35.f340b0a.js.map
dolphinscheduler-ui/dist/js/36.8880c2d.js
dolphinscheduler-ui/dist/js/36.8880c2d.js.map
dolphinscheduler-ui/dist/js/37.ea2a25d.js
dolphinscheduler-ui/dist/js/37.ea2a25d.js.map
dolphinscheduler-ui/dist/js/38.98a59ee.js
dolphinscheduler-ui/dist/js/38.98a59ee.js.map
dolphinscheduler-ui/dist/js/39.a5e958a.js
dolphinscheduler-ui/dist/js/39.a5e958a.js.map
dolphinscheduler-ui/dist/js/4.4ca44db.js
dolphinscheduler-ui/dist/js/4.4ca44db.js.map
dolphinscheduler-ui/dist/js/40.e187b1e.js
dolphinscheduler-ui/dist/js/40.e187b1e.js.map
dolphinscheduler-ui/dist/js/41.0e89182.js
dolphinscheduler-ui/dist/js/41.0e89182.js.map
dolphinscheduler-ui/dist/js/42.341047c.js
dolphinscheduler-ui/dist/js/42.341047c.js.map
dolphinscheduler-ui/dist/js/43.27b8228.js
dolphinscheduler-ui/dist/js/43.27b8228.js.map
dolphinscheduler-ui/dist/js/44.e8869bc.js
dolphinscheduler-ui/dist/js/44.e8869bc.js.map
dolphinscheduler-ui/dist/js/45.8d54901.js
dolphinscheduler-ui/dist/js/45.8d54901.js.map
dolphinscheduler-ui/dist/js/5.e1ed7f3.js
dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map
dolphinscheduler-ui/dist/js/6.241ba07.js
dolphinscheduler-ui/dist/js/6.241ba07.js.map
dolphinscheduler-ui/dist/js/7.ab2e297.js
dolphinscheduler-ui/dist/js/7.ab2e297.js.map
dolphinscheduler-ui/dist/js/8.83ff814.js
dolphinscheduler-ui/dist/js/8.83ff814.js.map
dolphinscheduler-ui/dist/js/9.39cb29f.js
dolphinscheduler-ui/dist/js/9.39cb29f.js.map
dolphinscheduler-ui/dist/js/common.733e342.js
dolphinscheduler-ui/dist/js/common.733e342.js.map
dolphinscheduler-ui/dist/js/home/index.78a5d12.js
dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map
dolphinscheduler-ui/dist/js/login/index.291b8e3.js
dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map
dolphinscheduler-ui/dist/lib/external/
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
/dolphinscheduler-dao/src/main/resources/dao/data_source.properties
docker/build/apache-dolphinscheduler*
README.md
浏览文件 @ dad94369
......@@ -7,46 +7,44 @@ Dolphin Scheduler Official Website
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data
[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
### Design features:
### Design Features:
Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system. It dedicates to solving the complex dependencies in data processing to make the scheduling system `out of the box` for the data processing process.
DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`.
Its main objectives are as follows:
- Associate the tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of the task in real-time.
- Support many task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Sub_Process, Procedure, etc.
- Support process scheduling, dependency scheduling, manual scheduling, manual pause/stop/recovery, support for failed retry/alarm, recovery from specified nodes, Kill task, etc.
- Support the priority of process & task, task failover, and task timeout alarm or failure.
- Support process global parameters and node custom parameter settings.
- Support online upload/download of resource files, management, etc. Support online file creation and editing.
- Support task log online viewing and scrolling, online download log, etc.
- Implement cluster HA, decentralize Master cluster and Worker cluster through Zookeeper.
- Support various task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Sub_Process, Procedure, etc.
- Support scheduling of workflows and dependencies, manual scheduling to pause/stop/recover task, support failure task retry/alarm, recover specified nodes from failure, kill task, etc.
- Support the priority of workflows & tasks, task failover, and task timeout alarm or failure.
- Support workflow global parameters and node customized parameter settings.
- Support online upload/download/management of resource files, etc. Support online file creation and editing.
- Support task log online viewing and scrolling and downloading, etc.
- Have implemented cluster HA, decentralize Master cluster and Worker cluster through Zookeeper.
- Support the viewing of Master/Worker CPU load, memory, and CPU usage metrics.
- Support presenting tree or Gantt chart of workflow history as well as the statistics results of task & process status in each workflow.
- Support backfilling data.
- Support displaying workflow history in tree/Gantt chart, as well as statistical analysis on the task status & process status in each workflow.
- Support back-filling data.
- Support multi-tenant.
- Support internationalization.
- There are more waiting for partners to explore...
- More features waiting for partners to explore...
### What's in Dolphin Scheduler
### What's in DolphinScheduler
Stability | Easy to use | Features | Scalability |
-- | -- | -- | --
Decentralized multi-master and multi-worker | Visualization process defines key information such as task status, task type, retry times, task running machine, visual variables, and so on at a glance.  |  Support pause, recover operation | Support custom task types
HA is supported by itself | All process definition operations are visualized, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, the API mode operation is provided. | Users on Dolphin Scheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler uses distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic online and offline.
Overload processing: Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | |
Decentralized multi-master and multi-worker | Visualization of workflow key information, such as task status, task type, retry times, task operation machine information, visual variables, and so on at a glance.  |  Support pause, recover operation | Support customized task types
support HA | Visualization of all workflow operations, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, provide API mode operations. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler supports distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic adjustment.
Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | |
### System partial screenshot
### User Interface Screenshots
![home page](https://user-images.githubusercontent.com/15833811/75218288-bf286400-57d4-11ea-8263-d639c6511d5f.jpg)
![dag](https://user-images.githubusercontent.com/15833811/75236750-3374fe80-57f9-11ea-857d-62a66a5a559d.png)
......@@ -57,13 +55,9 @@ Overload processing: Overload processing: By using the task queue mechanism, the
![security](https://user-images.githubusercontent.com/15833811/75236441-bfd2f180-57f8-11ea-88bd-f24311e01b7e.png)
![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png)
### QuickStart in Docker
Please referer the official website document:[[QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/1.3.4/user_doc/docker-deployment.html)]
### Recent R&D plan
The work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), which `In Develop` card shows the features that are currently being developed and TODO card lists what needs to be done(including feature ideas).
### How to contribute
Welcome to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/docs/development/contribute.html)]
### How to Build
......@@ -80,14 +74,16 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release
### Thanks
Dolphin Scheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on.
We would like to express our deep gratitude to all the open-source projects which contribute to making the dream of Dolphin Scheduler comes true. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we expect the partners who have the same passion and conviction to open-source will join in and contribute to the open-source community!
DolphinScheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on.
We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community!
### Get Help
1. Submit an issue
1. Submit an [[issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose)]
1. Subscribe to the mail list: https://dolphinscheduler.apache.org/en-us/docs/development/subscribe.html, then email dev@dolphinscheduler.apache.org
### How to Contribute
The community welcomes everyone to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html)]
### License
Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file.
......
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml
浏览文件 @ dad94369
......@@ -15,53 +15,64 @@
~ limitations under the License.
-->
<configuration>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.listen.port</name>
<value>1234</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker listen port</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.groups</name>
<value>default</value>
<description>default worker group</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10,
unit is G.
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.listen.port</name>
<value>1234</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker listen port</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.groups</name>
<value>default</value>
<description>default worker group</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.weigth</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker weight</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>
\ No newline at end of file
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map;
......@@ -34,8 +33,10 @@ public class DingTalkAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
String alertParams = alertInfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams);
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "ding talk params is null");
}
return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent());
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java
浏览文件 @ dad94369
......@@ -22,36 +22,24 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk;
*/
public class DingTalkParamsConstants {
public DingTalkParamsConstants() {
static final String DING_TALK_PROXY_ENABLE = "isEnableProxy";
static final String DING_TALK_WEB_HOOK = "webhook";
static final String NAME_DING_TALK_WEB_HOOK = "WebHook";
static final String DING_TALK_KEYWORD = "keyword";
static final String NAME_DING_TALK_KEYWORD = "Keyword";
static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy";
static final String DING_TALK_PROXY = "proxy";
static final String NAME_DING_TALK_PROXY = "Proxy";
static final String DING_TALK_PORT = "port";
static final String NAME_DING_TALK_PORT = "Port";
static final String DING_TALK_USER = "user";
static final String NAME_DING_TALK_USER = "User";
static final String DING_TALK_PASSWORD = "password";
static final String NAME_DING_TALK_PASSWORD = "Password";
private DingTalkParamsConstants() {
throw new IllegalStateException("Utility class");
}
static final String DING_TALK_WEB_HOOK = "dingtalk.webhook";
static final String NAME_DING_TALK_WEB_HOOK = "dingTalkWebHook";
static final String DING_TALK_KEYWORD = "dingtalk.keyword";
static final String NAME_DING_TALK_KEYWORD = "dingTalkKeyword";
public static final String DING_TALK_PROXY_ENABLE = "dingtalk.isEnableProxy";
static final String NAME_DING_TALK_PROXY_ENABLE = "dingTalkIsEnableProxy";
static final String DING_TALK_PROXY = "dingtalk.proxy";
static final String NAME_DING_TALK_PROXY = "dingTalkProxy";
static final String DING_TALK_PORT = "dingtalk.port";
static final String NAME_DING_TALK_PORT = "dingTalkPort";
static final String DING_TALK_USER = "dingtalk.user";
static final String NAME_DING_TALK_USER = "dingTalkUser";
static final String DING_TALK_PASSWORD = "dingtalk.password";
static final String NAME_DING_TALK_PASSWORD = "dingTalkPassword";
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java
浏览文件 @ dad94369
......@@ -75,51 +75,6 @@ public class DingTalkSender {
}
public AlertResult sendDingTalkMsg(String msg, String charset) {
AlertResult alertResult;
try {
String resp = sendMsg(msg, charset);
return checkSendDingTalkSendMsgResult(resp);
} catch (Exception e) {
logger.info("send ding talk alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send ding talk alert fail.");
}
return alertResult;
}
private String sendMsg(String msg, String charset) throws IOException {
String msgToJson = textToJsonString(msg + "#" + keyword);
HttpPost httpPost = constructHttpPost(url, msgToJson, charset);
CloseableHttpClient httpClient;
if (Boolean.TRUE.equals(enableProxy)) {
httpClient = getProxyClient(proxy, port, user, password);
RequestConfig rcf = getProxyConfig(proxy, port);
httpPost.setConfig(rcf);
} else {
httpClient = getDefaultClient();
}
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Ding Talk send [ %s ], resp:{%s}", msg, resp);
return resp;
} finally {
httpClient.close();
}
}
private static HttpPost constructHttpPost(String url, String msg, String charset) {
HttpPost post = new HttpPost(url);
StringEntity entity = new StringEntity(msg, charset);
......@@ -155,27 +110,6 @@ public class DingTalkSender {
return JSONUtils.toJsonString(items);
}
public static class DingTalkSendMsgResponse {
private Integer errcode;
private String errmsg;
public Integer getErrcode() {
return errcode;
}
public void setErrcode(Integer errcode) {
this.errcode = errcode;
}
public String getErrmsg() {
return errmsg;
}
public void setErrmsg(String errmsg) {
this.errmsg = errmsg;
}
}
private static AlertResult checkSendDingTalkSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
......@@ -201,4 +135,70 @@ public class DingTalkSender {
return alertResult;
}
public AlertResult sendDingTalkMsg(String title, String content) {
AlertResult alertResult;
try {
String resp = sendMsg(title, content);
return checkSendDingTalkSendMsgResult(resp);
} catch (Exception e) {
logger.info("send ding talk alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send ding talk alert fail.");
}
return alertResult;
}
private String sendMsg(String title, String content) throws IOException {
String msgToJson = textToJsonString(title + content + "#" + keyword);
HttpPost httpPost = constructHttpPost(url, msgToJson, "UTF-8");
CloseableHttpClient httpClient;
if (Boolean.TRUE.equals(enableProxy)) {
httpClient = getProxyClient(proxy, port, user, password);
RequestConfig rcf = getProxyConfig(proxy, port);
httpPost.setConfig(rcf);
} else {
httpClient = getDefaultClient();
}
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, "UTF-8");
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Ding Talk send title :{},content : {}, resp: {}", title, content, resp);
return resp;
} finally {
httpClient.close();
}
}
public static class DingTalkSendMsgResponse {
private Integer errcode;
private String errmsg;
public Integer getErrcode() {
return errcode;
}
public void setErrcode(Integer errcode) {
this.errcode = errcode;
}
public String getErrmsg() {
return errmsg;
}
public void setErrmsg(String errmsg) {
this.errmsg = errmsg;
}
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java
浏览文件 @ dad94369
......@@ -50,7 +50,7 @@ public class DingTalkSenderTest {
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true");
dingTalkSender = new DingTalkSender(dingTalkConfig);
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8");
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test");
Assert.assertEquals("false",alertResult.getStatus());
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map;
......@@ -38,8 +37,10 @@ public class EmailAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo info) {
AlertData alert = info.getAlertData();
String alertParams = info.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams);
Map<String, String> paramsMap = info.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "mail params is null");
}
MailSender mailSender = new MailSender(paramsMap);
AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent());
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java
浏览文件 @ dad94369
......@@ -19,6 +19,10 @@ package org.apache.dolphinscheduler.plugin.alert.email;
public class EmailConstants {
private EmailConstants() {
throw new IllegalStateException(EmailConstants.class.getName());
}
public static final String XLS_FILE_PATH = "xls.file.path";
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java
浏览文件 @ dad94369
......@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.alert.email;
import org.apache.dolphinscheduler.plugin.alert.email.exception.AlertEmailException;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.CollectionUtils;
......@@ -31,7 +32,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
......@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
*/
public class ExcelUtils {
public ExcelUtils() {
private ExcelUtils() {
throw new IllegalStateException("Utility class");
}
......@@ -65,16 +65,14 @@ public class ExcelUtils {
if (CollectionUtils.isEmpty(itemsList)) {
logger.error("itemsList is null");
throw new RuntimeException("itemsList is null");
throw new AlertEmailException("itemsList is null");
}
LinkedHashMap<String, Object> headerMap = itemsList.get(0);
List<String> headerList = new ArrayList<>();
Iterator<Map.Entry<String, Object>> iter = headerMap.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<String, Object> en = iter.next();
for (Map.Entry<String, Object> en : headerMap.entrySet()) {
headerList.add(en.getKey());
}
......@@ -130,8 +128,7 @@ public class ExcelUtils {
wb.write(fos);
} catch (Exception e) {
logger.error("generate excel error", e);
throw new RuntimeException("generate excel error", e);
throw new AlertEmailException("generate excel error", e);
} finally {
if (wb != null) {
try {
......@@ -150,4 +147,4 @@ public class ExcelUtils {
}
}
}
\ No newline at end of file
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java
浏览文件 @ dad94369
......@@ -22,7 +22,7 @@ package org.apache.dolphinscheduler.plugin.alert.email;
*/
public class MailParamsConstants {
public MailParamsConstants() {
private MailParamsConstants() {
throw new IllegalStateException("Utility class");
}
......@@ -33,33 +33,33 @@ public class MailParamsConstants {
public static final String NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs";
public static final String MAIL_PROTOCOL = "transport.protocol";
public static final String NAME_MAIL_PROTOCOL = "protocol";
public static final String NAME_MAIL_PROTOCOL = "mail.protocol";
public static final String MAIL_SMTP_HOST = "smtp.host";
public static final String MAIL_SMTP_HOST = "mail.smtp.host";
public static final String NAME_MAIL_SMTP_HOST = "serverHost";
public static final String MAIL_SMTP_PORT = "smtp.port";
public static final String MAIL_SMTP_PORT = "mail.smtp.port";
public static final String NAME_MAIL_SMTP_PORT = "serverPort";
public static final String MAIL_SENDER = "sender";
public static final String MAIL_SENDER = "mail.sender";
public static final String NAME_MAIL_SENDER = "sender";
public static final String MAIL_SMTP_AUTH = "smtp.auth";
public static final String MAIL_SMTP_AUTH = "mail.smtp.auth";
public static final String NAME_MAIL_SMTP_AUTH = "enableSmtpAuth";
public static final String MAIL_USER = "user";
public static final String MAIL_USER = "mail.user";
public static final String NAME_MAIL_USER = "user";
public static final String MAIL_PASSWD = "passwd";
public static final String MAIL_PASSWD = "mail.passwd";
public static final String NAME_MAIL_PASSWD = "passwd";
public static final String MAIL_SMTP_STARTTLS_ENABLE = "smtp.starttls.enable";
public static final String MAIL_SMTP_STARTTLS_ENABLE = "mail.smtp.starttls.enable";
public static final String NAME_MAIL_SMTP_STARTTLS_ENABLE = "starttlsEnable";
public static final String MAIL_SMTP_SSL_ENABLE = "smtp.ssl.enable";
public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable";
public static final String NAME_MAIL_SMTP_SSL_ENABLE = "sslEnable";
public static final String MAIL_SMTP_SSL_TRUST = "smtp.ssl.trust";
public static final String MAIL_SMTP_SSL_TRUST = "mail.smtp.ssl.trust";
public static final String NAME_MAIL_SMTP_SSL_TRUST = "smtpSslTrust";
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java
浏览文件 @ dad94369
......@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.email;
import static java.util.Objects.requireNonNull;
import org.apache.dolphinscheduler.plugin.alert.email.exception.AlertEmailException;
import org.apache.dolphinscheduler.plugin.alert.email.template.AlertTemplate;
import org.apache.dolphinscheduler.plugin.alert.email.template.DefaultHTMLTemplate;
import org.apache.dolphinscheduler.spi.alert.AlertConstants;
......@@ -67,7 +68,7 @@ public class MailSender {
private String mailProtocol = "SMTP";
private String mailSmtpHost;
private String mailSmtpPort;
private String mailSender;
private String mailSenderEmail;
private String enableSmtpAuth;
private String mailUser;
private String mailPasswd;
......@@ -77,12 +78,13 @@ public class MailSender {
private String sslTrust;
private String showType;
private AlertTemplate alertTemplate;
private String mustNotNull = "must not be null";
public MailSender(Map<String, String> config) {
String receiversConfig = config.get(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS);
if (receiversConfig == null || "".equals(receiversConfig)) {
throw new RuntimeException(MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS + "must not be null");
throw new AlertEmailException(MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS + mustNotNull);
}
receivers = Arrays.asList(receiversConfig.split(","));
......@@ -95,33 +97,33 @@ public class MailSender {
}
mailSmtpHost = config.get(MailParamsConstants.NAME_MAIL_SMTP_HOST);
requireNonNull(mailSmtpHost, MailParamsConstants.MAIL_SMTP_HOST + " must not null");
requireNonNull(mailSmtpHost, MailParamsConstants.MAIL_SMTP_HOST + mustNotNull);
mailSmtpPort = config.get(MailParamsConstants.NAME_MAIL_SMTP_PORT);
requireNonNull(mailSmtpPort, MailParamsConstants.MAIL_SMTP_PORT + " must not null");
requireNonNull(mailSmtpPort, MailParamsConstants.MAIL_SMTP_PORT + mustNotNull);
mailSender = config.get(MailParamsConstants.NAME_MAIL_SENDER);
requireNonNull(mailSender, MailParamsConstants.MAIL_SENDER + " must not null");
mailSenderEmail = config.get(MailParamsConstants.NAME_MAIL_SENDER);
requireNonNull(mailSenderEmail, MailParamsConstants.MAIL_SENDER + mustNotNull);
enableSmtpAuth = config.get(MailParamsConstants.NAME_MAIL_SMTP_AUTH);
mailUser = config.get(MailParamsConstants.NAME_MAIL_USER);
requireNonNull(mailUser, MailParamsConstants.MAIL_USER + " must not null");
requireNonNull(mailUser, MailParamsConstants.MAIL_USER + mustNotNull);
mailPasswd = config.get(MailParamsConstants.NAME_MAIL_PASSWD);
requireNonNull(mailPasswd, MailParamsConstants.MAIL_PASSWD + " must not null");
requireNonNull(mailPasswd, MailParamsConstants.MAIL_PASSWD + mustNotNull);
mailUseStartTLS = config.get(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE);
requireNonNull(mailUseStartTLS, MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE + " must not null");
requireNonNull(mailUseStartTLS, MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE + mustNotNull);
mailUseSSL = config.get(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE);
requireNonNull(mailUseSSL, MailParamsConstants.MAIL_SMTP_SSL_ENABLE + " must not null");
requireNonNull(mailUseSSL, MailParamsConstants.MAIL_SMTP_SSL_ENABLE + mustNotNull);
sslTrust = config.get(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST);
requireNonNull(sslTrust, MailParamsConstants.MAIL_SMTP_SSL_TRUST + " must not null");
requireNonNull(sslTrust, MailParamsConstants.MAIL_SMTP_SSL_TRUST + mustNotNull);
showType = config.get(AlertConstants.SHOW_TYPE);
requireNonNull(showType, AlertConstants.SHOW_TYPE + " must not null");
requireNonNull(showType, AlertConstants.SHOW_TYPE + mustNotNull);
xlsFilePath = config.get(EmailConstants.XLS_FILE_PATH);
if (StringUtils.isBlank(xlsFilePath)) {
......@@ -134,9 +136,8 @@ public class MailSender {
/**
* send mail to receivers
*
* @param title title
* @param title title
* @param content content
* @return
*/
public AlertResult sendMails(String title, String content) {
return sendMails(this.receivers, this.receiverCcs, title, content);
......@@ -145,9 +146,8 @@ public class MailSender {
/**
* send mail to receivers
*
* @param title email title
* @param title email title
* @param content email content
* @return
*/
public AlertResult sendMailsToReceiverOnly(String title, String content) {
return sendMails(this.receivers, null, title, content);
......@@ -156,11 +156,10 @@ public class MailSender {
/**
* send mail
*
* @param receivers receivers
* @param receivers receivers
* @param receiverCcs receiverCcs
* @param title title
* @param content content
* @return
* @param title title
* @param content content
*/
public AlertResult sendMails(List<String> receivers, List<String> receiverCcs, String title, String content) {
AlertResult alertResult = new AlertResult();
......@@ -180,7 +179,7 @@ public class MailSender {
try {
Session session = getSession();
email.setMailSession(session);
email.setFrom(mailSender);
email.setFrom(mailSenderEmail);
email.setCharset(EmailConstants.UTF_8);
if (CollectionUtils.isNotEmpty(receivers)) {
// receivers mail
......@@ -251,11 +250,6 @@ public class MailSender {
/**
* send mail as Excel attachment
*
* @param title
* @param content
* @param partContent
* @throws Exception
*/
private void attachment(String title, String content, String partContent) throws Exception {
MimeMessage msg = getMimeMessage();
......@@ -265,9 +259,6 @@ public class MailSender {
/**
* get MimeMessage
*
* @return
* @throws MessagingException
*/
private MimeMessage getMimeMessage() throws MessagingException {
......@@ -279,7 +270,7 @@ public class MailSender {
// 2. creating mail: Creating a MimeMessage
MimeMessage msg = new MimeMessage(session);
// 3. set sender
msg.setFrom(new InternetAddress(mailSender));
msg.setFrom(new InternetAddress(mailSenderEmail));
// 4. set receivers
for (String receiver : receivers) {
msg.addRecipients(Message.RecipientType.TO, InternetAddress.parse(receiver));
......@@ -317,16 +308,9 @@ public class MailSender {
/**
* attach content
*
* @param title
* @param content
* @param partContent
* @param msg
* @throws MessagingException
* @throws IOException
*/
private void attachContent(String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException {
/**
/*
* set receiverCc
*/
if (CollectionUtils.isNotEmpty(receiverCcs)) {
......@@ -365,21 +349,14 @@ public class MailSender {
/**
* the string object map
*
* @param title
* @param content
* @param alertResult
* @param email
* @return
* @throws EmailException
*/
private AlertResult getStringObjectMap(String title, String content, AlertResult alertResult, HtmlEmail email) throws EmailException {
/**
/*
* the subject of the message to be sent
*/
email.setSubject(title);
/**
/*
* to send information, you can use HTML tags in mail content because of the use of HtmlEmail
*/
if (showType.equals(ShowType.TABLE.getDescp())) {
......@@ -417,9 +394,6 @@ public class MailSender {
/**
* handle exception
*
* @param alertResult
* @param e
*/
private void handleException(AlertResult alertResult, Exception e) {
logger.error("Send email to {} failed", receivers, e);
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.email.exception;
public class AlertEmailException extends RuntimeException {
/**
* Create Runtime exception
*
* @param errMsg - Error message
*/
public AlertEmailException(String errMsg) {
super(errMsg);
}
/**
* Create Runtime exception
*
* @param errMsg - Error message
* @param cause - cause
*/
public AlertEmailException(String errMsg, Throwable cause) {
super(errMsg, cause);
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java
浏览文件 @ dad94369
......@@ -50,7 +50,7 @@ public class DefaultHTMLTemplate implements AlertTemplate {
case TABLE:
return getTableTypeMessage(content, showAll);
case TEXT:
return getTextTypeMessage(content, showAll);
return getTextTypeMessage(content);
default:
throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType));
}
......@@ -77,7 +77,7 @@ public class DefaultHTMLTemplate implements AlertTemplate {
boolean flag = true;
String title = "";
for (LinkedHashMap mapItems : mapItemsList) {
for (LinkedHashMap<String, Object> mapItems : mapItemsList) {
Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
......@@ -111,10 +111,9 @@ public class DefaultHTMLTemplate implements AlertTemplate {
* get alert message which type is TEXT
*
* @param content message content
* @param showAll weather to show all
* @return alert message
*/
private String getTextTypeMessage(String content, boolean showAll) {
private String getTextTypeMessage(String content) {
if (StringUtils.isNotEmpty(content)) {
ArrayNode list = JSONUtils.parseArray(content);
......@@ -135,14 +134,14 @@ public class DefaultHTMLTemplate implements AlertTemplate {
/**
* get alert message from a html template
*
* @param title message title
* @param title message title
* @param content message content
* @return alert message which use html template
*/
private String getMessageFromHtmlTemplate(String title, String content) {
requireNonNull(content, "content must not null");
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>\n", title);
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>%n", title);
return EmailConstants.HTML_HEADER_PREFIX + htmlTableThead + content + EmailConstants.TABLE_BODY_HTML_TAIL;
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactoryTest.java
浏览文件 @ dad94369
......@@ -19,13 +19,10 @@ package org.apache.dolphinscheduler.plugin.alert.email;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
......@@ -36,29 +33,13 @@ import org.junit.Test;
*/
public class EmailAlertChannelFactoryTest {
@Before
public void before() throws Exception {
}
@After
public void after() throws Exception {
}
/**
* Method: getName()
*/
@Test
public void testGetName() throws Exception {
}
/**
* Method: getParams()
*/
@Test
public void testGetParams() throws Exception {
public void testGetParams() {
EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory();
List<PluginParams> params = emailAlertChannelFactory.getParams();
System.out.println(JSONUtils.toJsonString(params));
Assert.assertEquals(12, params.size());
}
......@@ -66,7 +47,7 @@ public class EmailAlertChannelFactoryTest {
* Method: create()
*/
@Test
public void testCreate() throws Exception {
public void testCreate() {
EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory();
AlertChannel alertChannel = emailAlertChannelFactory.create();
Assert.assertNotNull(alertChannel);
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java
浏览文件 @ dad94369
......@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.alert.ShowType;
import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PasswordParam;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.RadioParam;
import org.apache.dolphinscheduler.spi.params.base.DataType;
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
......@@ -34,6 +35,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
......@@ -66,7 +68,9 @@ public class EmailAlertChannelTest {
.setTitle("test");
AlertInfo alertInfo = new AlertInfo();
alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(getEmailAlertParams());
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(getEmailAlertParams());
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult = emailAlertChannel.process(alertInfo);
Assert.assertNotNull(alertResult);
Assert.assertEquals("false", alertResult.getStatus());
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java
浏览文件 @ dad94369
......@@ -53,6 +53,7 @@ public class MailUtilsTest {
emailConfig.put(MailParamsConstants.NAME_MAIL_SENDER, "xxx1.xxx.com");
emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "xxx2.xxx.com");
emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "111111");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_AUTH, "true");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, "true");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, "false");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, "false");
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml 0 → 100644
浏览文件 @ dad94369
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert-feishu</artifactId>
<packaging>dolphinscheduler-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>dolphinscheduler-alert-feishu-${project.version}</finalName>
</build>
</project>
\ No newline at end of file
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.Map;
public class FeiShuAlertChannel implements AlertChannel {
@Override
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "fei shu params is null");
}
return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData);
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertChannelFactory;
import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PasswordParam;
import org.apache.dolphinscheduler.spi.params.RadioParam;
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.params.base.Validate;
import java.util.Arrays;
import java.util.List;
public class FeiShuAlertChannelFactory implements AlertChannelFactory {
@Override
public String getName() {
return "Feishu";
}
@Override
public List<PluginParams> getParams() {
InputParam webHookParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK)
.addValidate(Validate.newBuilder()
.setRequired(true)
.build())
.build();
RadioParam isEnableProxy =
RadioParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE)
.addParamsOptions(new ParamsOptions("YES", true, false))
.addParamsOptions(new ParamsOptions("NO", false, false))
.setValue(true)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
InputParam proxyParam =
InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
InputParam portParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
InputParam userParam =
InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
PasswordParam passwordParam = PasswordParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD)
.setPlaceholder("if enable use authentication, you need input password")
.build();
return Arrays.asList(webHookParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam);
}
@Override
public AlertChannel create() {
return new FeiShuAlertChannel();
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertPlugin.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
import org.apache.dolphinscheduler.spi.alert.AlertChannelFactory;
import com.google.common.collect.ImmutableList;
public class FeiShuAlertPlugin implements DolphinSchedulerPlugin {
@Override
public Iterable<AlertChannelFactory> getAlertChannelFactorys() {
return ImmutableList.of(new FeiShuAlertChannelFactory());
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
public class FeiShuParamsConstants {
private FeiShuParamsConstants() {
throw new IllegalStateException("Utility class");
}
static final String WEB_HOOK = "webhook";
static final String NAME_WEB_HOOK = "webHook";
public static final String FEI_SHU_PROXY_ENABLE = "isEnableProxy";
static final String NAME_FEI_SHU_PROXY_ENABLE = "isEnableProxy";
static final String FEI_SHU_PROXY = "proxy";
static final String NAME_FEI_SHU_PROXY = "proxy";
static final String FEI_SHU_PORT = "port";
static final String NAME_FEI_SHU_PORT = "port";
static final String FEI_SHU_USER = "user";
static final String NAME_FEI_SHU_USER = "user";
static final String FEI_SHU_PASSWORD = "password";
static final String NAME_FEI_SHU_PASSWORD = "password";
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
public class FeiShuSender {
private static final Logger logger = LoggerFactory.getLogger(FeiShuSender.class);
private String url;
private Boolean enableProxy;
private String proxy;
private Integer port;
private String user;
private String password;
FeiShuSender(Map<String, String> config) {
url = config.get(FeiShuParamsConstants.NAME_WEB_HOOK);
enableProxy = Boolean.valueOf(config.get(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE));
if (Boolean.TRUE.equals(enableProxy)) {
port = Integer.parseInt(config.get(FeiShuParamsConstants.NAME_FEI_SHU_PORT));
proxy = config.get(FeiShuParamsConstants.NAME_FEI_SHU_PROXY);
user = config.get(FeiShuParamsConstants.NAME_FEI_SHU_USER);
password = config.get(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD);
}
}
private static RequestConfig getProxyConfig(String proxy, int port) {
HttpHost httpProxy = new HttpHost(proxy, port);
return RequestConfig.custom().setProxy(httpProxy).build();
}
private static String textToJsonString(AlertData alertData) {
Map<String, Object> items = new HashMap<>(2);
items.put("msg_type", "text");
Map<String, String> textContent = new HashMap<>();
byte[] byt = StringUtils.getBytesUtf8(formatContent(alertData));
String txt = StringUtils.newStringUtf8(byt);
textContent.put("text", txt);
items.put("content", textContent);
return JSONUtils.toJsonString(items);
}
private static AlertResult checkSendFeiShuSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(result)) {
alertResult.setMessage("send fei shu msg error");
logger.info("send fei shu msg error,fei shu server resp is null");
return alertResult;
}
FeiShuSendMsgResponse sendMsgResponse = JSONUtils.parseObject(result, FeiShuSendMsgResponse.class);
if (null == sendMsgResponse) {
alertResult.setMessage("send fei shu msg fail");
logger.info("send fei shu msg error,resp error");
return alertResult;
}
if (sendMsgResponse.statusCode == 0) {
alertResult.setStatus("true");
alertResult.setMessage("send fei shu msg success");
return alertResult;
}
alertResult.setMessage(String.format("alert send fei shu msg error : %s", sendMsgResponse.getStatusMessage()));
logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), sendMsgResponse.getExtra());
return alertResult;
}
public static String formatContent(AlertData alertData) {
if (alertData.getContent() != null) {
List<Map> list;
try {
list = JSONUtils.toList(alertData.getContent(), Map.class);
} catch (Exception e) {
logger.error("json format exception", e);
return null;
}
StringBuilder contents = new StringBuilder(100);
contents.append(String.format("`%s`%n", alertData.getTitle()));
for (Map map : list) {
Iterator<Entry<String, Object>> entries = map.entrySet().iterator();
while (entries.hasNext()) {
Entry<String, Object> entry = entries.next();
String key = entry.getKey();
String value = entry.getValue().toString();
contents.append(key + ":" + value);
contents.append("\n");
}
}
return contents.toString();
}
return null;
}
public AlertResult sendFeiShuMsg(AlertData alertData) {
AlertResult alertResult;
try {
String resp = sendMsg(alertData);
return checkSendFeiShuSendMsgResult(resp);
} catch (Exception e) {
logger.info("send fei shu alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send fei shu alert fail.");
}
return alertResult;
}
private String sendMsg(AlertData alertData) throws IOException {
String msgToJson = textToJsonString(alertData);
HttpPost httpPost = HttpRequestUtil.constructHttpPost(url, msgToJson);
CloseableHttpClient httpClient;
httpClient = HttpRequestUtil.getHttpClient(enableProxy, proxy, port, user, password);
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
logger.error("send feishu message error, return http status code: " + statusCode);
}
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, "utf-8");
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Ding Talk send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), resp);
return resp;
} finally {
httpClient.close();
}
}
public static class FeiShuSendMsgResponse {
@JsonProperty("Extra")
private String extra;
@JsonProperty("StatusCode")
private Integer statusCode;
@JsonProperty("StatusMessage")
private String statusMessage;
public String getExtra() {
return extra;
}
public void setExtra(String extra) {
this.extra = extra;
}
public Integer getStatusCode() {
return statusCode;
}
public void setStatusCode(Integer statusCode) {
this.statusCode = statusCode;
}
public String getStatusMessage() {
return statusMessage;
}
public void setStatusMessage(String statusMessage) {
this.statusMessage = statusMessage;
}
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
public class HttpRequestUtil {
public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, String password) {
if (enableProxy) {
HttpHost httpProxy = new HttpHost(proxy, port);
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(user, password));
return HttpClients.custom().setDefaultCredentialsProvider(provider).build();
} else {
return HttpClients.createDefault();
}
}
public static HttpPost constructHttpPost(String url, String msg) {
HttpPost post = new HttpPost(url);
StringEntity entity = new StringEntity(msg, ContentType.APPLICATION_JSON);
post.setEntity(entity);
return post;
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactoryTest.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class FeiShuAlertChannelFactoryTest {
@Test
public void testGetParams() {
FeiShuAlertChannelFactory feiShuAlertChannelFactory = new FeiShuAlertChannelFactory();
List<PluginParams> params = feiShuAlertChannelFactory.getParams();
JSONUtils.toJsonString(params);
Assert.assertEquals(6, params.size());
}
@Test
public void testCreate() {
FeiShuAlertChannelFactory feiShuAlertChannelFactory = new FeiShuAlertChannelFactory();
AlertChannel alertChannel = feiShuAlertChannelFactory.create();
Assert.assertNotNull(alertChannel);
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class FeiShuSenderTest {
private static Map<String, String> feiShuConfig = new HashMap<>();
@Before
public void initFeiShuConfig() {
feiShuConfig.put(FeiShuParamsConstants.WEB_HOOK, "https://open.feishu.cn/open-apis/bot/v2/hook/xxxxx");
}
@Test
public void testSend() {
AlertData alertData = new AlertData();
alertData.setTitle("feishu test title");
alertData.setContent("feishu test content");
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData);
Assert.assertEquals("false", alertResult.getStatus());
}
@Test
public void testFormatContent() {
String alertMsg = "[\n"
+ " {\n"
+ " \"owner\": \"dolphinscheduler\",\n"
+ " \"processEndTime\": \"2021-01-29 19:01:11\",\n"
+ " \"processHost\": \"10.81.129.4:5678\",\n"
+ " \"processId\": 2926,\n"
+ " \"processName\": \"3-20210129190038108\",\n"
+ " \"processStartTime\": \"2021-01-29 19:00:38\",\n"
+ " \"processState\": \"SUCCESS\",\n"
+ " \"processType\": \"START_PROCESS\",\n"
+ " \"projectId\": 2,\n"
+ " \"projectName\": \"testdelproject\",\n"
+ " \"recovery\": \"NO\",\n"
+ " \"retryTimes\": 0,\n"
+ " \"runTimes\": 1,\n"
+ " \"taskId\": 0\n"
+ " }\n"
+ "]";
AlertData alertData = new AlertData();
alertData.setTitle("");
alertData.setContent(alertMsg);
Assert.assertNotNull(FeiShuSender.formatContent(alertData));
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map;
......@@ -33,8 +32,10 @@ public class HttpAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
String alertParams = alertInfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams);
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "http params is null");
}
return new HttpSender(paramsMap).send(alertData.getContent());
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java
浏览文件 @ dad94369
......@@ -21,12 +21,14 @@ import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.params.base.Validate;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
......@@ -45,7 +47,7 @@ public class HttpAlertChannelTest {
alertData.setContent("Fault tolerance warning");
alertInfo.setAlertData(alertData);
AlertResult alertResult = alertChannel.process(alertInfo);
Assert.assertEquals("Request types are not supported", alertResult.getMessage());
Assert.assertEquals("http params is null", alertResult.getMessage());
}
@Test
......@@ -56,7 +58,8 @@ public class HttpAlertChannelTest {
AlertData alertData = new AlertData();
alertData.setContent("Fault tolerance warning");
alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(getParams());
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(getParams());
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult = alertChannel.process(alertInfo);
Assert.assertEquals("true", alertResult.getStatus());
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java
浏览文件 @ dad94369
......@@ -22,7 +22,7 @@ package org.apache.dolphinscheduler.plugin.alert.script;
*/
public class OSUtils {
public OSUtils() {
private OSUtils() {
throw new UnsupportedOperationException("Construct OSUtils");
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map;
......@@ -33,8 +32,10 @@ public class ScriptAlertChannel implements AlertChannel {
@Override
public AlertResult process(AlertInfo alertinfo) {
AlertData alertData = alertinfo.getAlertData();
String alertParams = alertinfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams);
Map<String, String> paramsMap = alertinfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "ding talk params is null");
}
return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle());
}
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java
浏览文件 @ dad94369
......@@ -22,7 +22,7 @@ package org.apache.dolphinscheduler.plugin.alert.script;
*/
public class ScriptParamsConstants {
public ScriptParamsConstants() {
private ScriptParamsConstants() {
throw new IllegalStateException("Utility class");
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java
浏览文件 @ dad94369
......@@ -51,11 +51,11 @@ public class StreamGobbler extends Thread {
output.append(System.getProperty("line.separator"));
}
if (output.length() > 0) {
logger.info("out put msg is{}",output.toString());
logger.info("out put msg is{}", output);
}
} catch (IOException e) {
logger.error("I/O error occurs {}", e.getMessage());
}
}
}
\ No newline at end of file
}
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactoryTest.java
浏览文件 @ dad94369
......@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.plugin.alert.script;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List;
......@@ -35,7 +34,6 @@ public class ScriptAlertChannelFactoryTest {
public void testGetParams() {
ScriptAlertChannelFactory scriptAlertChannelFactory = new ScriptAlertChannelFactory();
List<PluginParams> params = scriptAlertChannelFactory.getParams();
JSONUtils.toJsonString(params);
Assert.assertEquals(3, params.size());
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map;
......@@ -33,8 +32,10 @@ public class WeChatAlertChannel implements AlertChannel {
@Override
public AlertResult process(AlertInfo info) {
AlertData alertData = info.getAlertData();
String alertParams = info.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams);
Map<String, String> paramsMap = info.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "we chat params is null");
}
return new WeChatSender(paramsMap).sendEnterpriseWeChat(alertData.getTitle(), alertData.getContent());
}
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java
浏览文件 @ dad94369
......@@ -22,6 +22,10 @@ package org.apache.dolphinscheduler.plugin.alert.wechat;
*/
public class WeChatAlertConstants {
private WeChatAlertConstants() {
throw new IllegalStateException(WeChatAlertConstants.class.getName());
}
static final String MARKDOWN_QUOTE = ">";
static final String MARKDOWN_ENTER = "\n";
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java
浏览文件 @ dad94369
......@@ -22,6 +22,9 @@ package org.apache.dolphinscheduler.plugin.alert.wechat;
*/
public class WeChatAlertParamsConstants {
private WeChatAlertParamsConstants() {
throw new IllegalStateException(WeChatAlertParamsConstants.class.getName());
}
static final String ENTERPRISE_WE_CHAT_CORP_ID = "corp.id";
......
dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java
浏览文件 @ dad94369
......@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.alert.wechat;
import static java.util.Objects.requireNonNull;
import org.apache.dolphinscheduler.plugin.alert.wechat.exception.WeChatAlertException;
import org.apache.dolphinscheduler.spi.alert.AlertConstants;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.alert.ShowType;
......@@ -59,8 +60,6 @@ public class WeChatSender {
private String weChatUsers;
private String weChatTeamSendMsg;
private String weChatUserSendMsg;
private String weChatTokenUrlReplace;
......@@ -70,14 +69,14 @@ public class WeChatSender {
private String showType;
private static final String agentIdRegExp = "{agentId}";
private static final String msgRegExp = "{msg}";
private static final String userRegExp = "{toUser}";
private static final String corpIdRegex = "{corpId}";
private static final String secretRegex = "{secret}";
private static final String toPartyRegex = "{toParty}";
private static final String toUserRegex = "{toUser}";
private static final String tokenRegex = "{token}";
private static final String MUST_NOT_NULL = " must not null";
private static final String ALERT_STATUS = "false";
private static final String AGENT_ID_REG_EXP = "{agentId}";
private static final String MSG_REG_EXP = "{msg}";
private static final String USER_REG_EXP = "{toUser}";
private static final String CORP_ID_REGEX = "{corpId}";
private static final String SECRET_REGEX = "{secret}";
private static final String TOKEN_REGEX = "{token}";
WeChatSender(Map<String, String> config) {
weChatAgentId = config.get(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_AGENT_ID);
......@@ -85,13 +84,12 @@ public class WeChatSender {
String weChatCorpId = config.get(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_CORP_ID);
String weChatSecret = config.get(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SECRET);
String weChatTokenUrl = WeChatAlertConstants.WE_CHAT_TOKEN_URL;
weChatTeamSendMsg = config.get(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_TEAM_SEND_MSG);
weChatUserSendMsg = config.get(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_USER_SEND_MSG);
showType = config.get(AlertConstants.SHOW_TYPE);
requireNonNull(showType, AlertConstants.SHOW_TYPE + " must not null");
requireNonNull(showType, AlertConstants.SHOW_TYPE + MUST_NOT_NULL);
weChatTokenUrlReplace = weChatTokenUrl
.replace(corpIdRegex, weChatCorpId)
.replace(secretRegex, weChatSecret);
.replace(CORP_ID_REGEX, weChatCorpId)
.replace(SECRET_REGEX, weChatSecret);
weChatToken = getToken();
}
......@@ -105,16 +103,15 @@ public class WeChatSender {
*/
private String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = mkString(toUser);
return weChatUserSendMsg.replace(userRegExp, listUser)
.replace(agentIdRegExp, agentId)
.replace(msgRegExp, msg);
return weChatUserSendMsg.replace(USER_REG_EXP, listUser)
.replace(AGENT_ID_REG_EXP, agentId)
.replace(MSG_REG_EXP, msg);
}
/**
* send Enterprise WeChat
*
* @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""}
* @throws Exception the Exception
*/
public AlertResult sendEnterpriseWeChat(String title, String content) {
AlertResult alertResult;
......@@ -124,10 +121,10 @@ public class WeChatSender {
if (null == weChatToken) {
alertResult = new AlertResult();
alertResult.setMessage("send we chat alert fail,get weChat token error");
alertResult.setStatus("false");
alertResult.setStatus(ALERT_STATUS);
return alertResult;
}
String enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_PUSH_URL.replace(tokenRegex, weChatToken);
String enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken);
try {
return checkWeChatSendMsgResult(post(enterpriseWeChatPushUrlReplace, msg));
......@@ -135,7 +132,7 @@ public class WeChatSender {
logger.info("send we chat alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setMessage("send we chat alert fail");
alertResult.setStatus("false");
alertResult.setStatus(ALERT_STATUS);
}
return alertResult;
}
......@@ -170,7 +167,7 @@ public class WeChatSender {
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if (null == mapItemsList || mapItemsList.isEmpty()) {
logger.error("itemsList is null");
throw new RuntimeException("itemsList is null");
throw new WeChatAlertException("itemsList is null");
}
StringBuilder contents = new StringBuilder(200);
for (LinkedHashMap mapItems : mapItemsList) {
......@@ -259,7 +256,7 @@ public class WeChatSender {
EntityUtils.consume(entity);
}
HashMap map = JSONUtils.parseObject(resp, HashMap.class);
HashMap<String, Object> map = JSONUtils.parseObject(resp, HashMap.class);
if (map != null && null != map.get("access_token")) {
return map.get("access_token").toString();
} else {
......@@ -310,7 +307,7 @@ public class WeChatSender {
private static AlertResult checkWeChatSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setStatus(ALERT_STATUS);
if (null == result) {
alertResult.setMessage("we chat send fail");
......@@ -328,7 +325,7 @@ public class WeChatSender {
alertResult.setMessage("we chat alert send success");
return alertResult;
}
alertResult.setStatus("false");
alertResult.setStatus(ALERT_STATUS);
alertResult.setMessage(sendMsgResponse.getErrmsg());
return alertResult;
}
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/MsgManager.java → dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/exception/WeChatAlertException.java
浏览文件 @ dad94369
......@@ -15,26 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.manager;
package org.apache.dolphinscheduler.plugin.alert.wechat.exception;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SMS send manager
*/
public class MsgManager {
private static final Logger logger = LoggerFactory.getLogger(MsgManager.class);
public class WeChatAlertException extends RuntimeException {
/**
* SMS send
* Create Runtime Exception
*
* @param alert the alert
* @param errMsg - Error message
*/
public void send(Alert alert) {
logger.info("send message {}", alert);
public WeChatAlertException(String errMsg) {
super(errMsg);
}
}
dolphinscheduler-alert-plugin/pom.xml
浏览文件 @ dad94369
......@@ -35,6 +35,7 @@
<module>dolphinscheduler-alert-dingtalk</module>
<module>dolphinscheduler-alert-script</module>
<module>dolphinscheduler-alert-http</module>
<module>dolphinscheduler-alert-feishu</module>
</modules>
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
浏览文件 @ dad94369
......@@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.alert.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.command.CommandType;
......@@ -53,8 +52,6 @@ public class AlertServer {
*/
private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class);
private PluginDao pluginDao = DaoFactory.getDaoInstance(PluginDao.class);
private AlertSender alertSender;
private static AlertServer instance;
......@@ -114,7 +111,7 @@ public class AlertServer {
NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(ALERT_RPC_PORT);
this.server = new NettyRemotingServer(serverConfig);
this.server.registerProcessor(CommandType.ALERT_SEND_REQUEST, new AlertRequestProcessor(alertDao, alertPluginManager, pluginDao));
this.server.registerProcessor(CommandType.ALERT_SEND_REQUEST, new AlertRequestProcessor(alertDao, alertPluginManager));
this.server.start();
}
......@@ -133,7 +130,7 @@ public class AlertServer {
logger.warn("No Alert Plugin . Can not send alert info. ");
} else {
List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao, alertPluginManager, pluginDao);
alertSender = new AlertSender(alerts, alertDao, alertPluginManager);
alertSender.run();
}
}
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java 已删除 100644 → 0
浏览文件 @ fba133c9
///*
// * Licensed to the Apache Software Foundation (ASF) under one or more
// * contributor license agreements. See the NOTICE file distributed with
// * this work for additional information regarding copyright ownership.
// * The ASF licenses this file to You under the Apache License, Version 2.0
// * (the "License"); you may not use this file except in compliance with
// * the License. You may obtain a copy of the License at
// *
// * http://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
//package org.apache.dolphinscheduler.alert.manager;
//
//import org.apache.dolphinscheduler.alert.utils.MailUtils;
//
//import java.util.List;
//import java.util.Map;
//
///**
// * email send manager
// */
//public class EmailManager {
// /**
// * email send
// * @param receiversList the receiver list
// * @param receiversCcList the cc List
// * @param title the title
// * @param content the content
// * @param showType the showType
// * @return the send result
// */
// public Map<String,Object> send(List<String> receiversList,List<String> receiversCcList,String title,String content,String showType){
//
// return MailUtils.sendMails(receiversList, receiversCcList, title, content, showType);
// }
//
// /**
// * msg send
// * @param receiversList the receiver list
// * @param title the title
// * @param content the content
// * @param showType the showType
// * @return the send result
// */
// public Map<String,Object> send(List<String> receiversList,String title,String content,String showType){
//
// return MailUtils.sendMails(receiversList,title, content, showType);
// }
//
//}
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java
浏览文件 @ dad94369
......@@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.spi.classloader.ThreadContextClassLoader;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
......@@ -47,6 +48,11 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
private final Map<String, AlertChannelFactory> alertChannelFactoryMap = new ConcurrentHashMap<>();
private final Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>();
/**
* k->pluginDefineId v->pluginDefineName
*/
private final Map<Integer, String> pluginDefineMap = new HashMap<>();
public void addAlertChannelFactory(AlertChannelFactory alertChannelFactory) {
requireNonNull(alertChannelFactory, "alertChannelFactory is null");
......@@ -83,6 +89,10 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
return alertChannelMap;
}
public String getPluginNameById(int id) {
return pluginDefineMap.get(id);
}
@Override
public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) {
for (AlertChannelFactory alertChannelFactory : dolphinSchedulerPlugin.getAlertChannelFactorys()) {
......@@ -93,7 +103,8 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
String paramsJson = PluginParamsTransfer.transferParamsToJson(params);
PluginDefine pluginDefine = new PluginDefine(nameEn, PluginType.ALERT.getDesc(), paramsJson);
pluginDao.addOrUpdatePluginDefine(pluginDefine);
int id = pluginDao.addOrUpdatePluginDefine(pluginDefine);
pluginDefineMap.put(id, pluginDefine.getPluginName());
}
}
}
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessor.java
浏览文件 @ dad94369
......@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.alert.runner.AlertSender;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
......@@ -35,18 +34,16 @@ import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
/**
* alert request processor
* alert request processor
*/
public class AlertRequestProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(AlertRequestProcessor.class);
private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager;
public AlertRequestProcessor(AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) {
public AlertRequestProcessor(AlertDao alertDao, AlertPluginManager alertPluginManager) {
this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager;
}
......@@ -59,7 +56,7 @@ public class AlertRequestProcessor implements NettyRequestProcessor {
command.getBody(), AlertSendRequestCommand.class);
logger.info("received command : {}", alertSendRequestCommand);
AlertSender alertSender = new AlertSender(alertDao, alertPluginManager, pluginDao);
AlertSender alertSender = new AlertSender(alertDao, alertPluginManager);
AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertSendRequestCommand.getGroupId(), alertSendRequestCommand.getTitle(), alertSendRequestCommand.getContent());
channel.writeAndFlush(alertSendResponseCommand.convert2Command(command.getOpaque()));
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
浏览文件 @ dad94369
......@@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.alert.runner;
import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand;
......@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -46,25 +47,22 @@ public class AlertSender {
private List<Alert> alertList;
private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager;
public AlertSender(AlertPluginManager alertPluginManager) {
this.alertPluginManager = alertPluginManager;
}
public AlertSender(AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) {
public AlertSender(AlertDao alertDao, AlertPluginManager alertPluginManager) {
super();
this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager;
}
public AlertSender(List<Alert> alertList, AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) {
public AlertSender(List<Alert> alertList, AlertDao alertDao, AlertPluginManager alertPluginManager) {
super();
this.alertList = alertList;
this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager;
}
......@@ -75,13 +73,14 @@ public class AlertSender {
List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId);
if (CollectionUtils.isEmpty(alertInstanceList)) {
logger.error("send alert msg fail,no bind plugin instance.");
return;
alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "no bind plugin instance", alert.getId());
continue;
}
AlertData alertData = new AlertData();
alertData.setId(alert.getId())
.setContent(alert.getContent())
.setLog(alert.getLog())
.setTitle(alert.getTitle());
.setContent(alert.getContent())
.setLog(alert.getLog())
.setTitle(alert.getTitle());
for (AlertPluginInstance instance : alertInstanceList) {
......@@ -106,8 +105,8 @@ public class AlertSender {
List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId);
AlertData alertData = new AlertData();
alertData.setContent(title)
.setTitle(content);
alertData.setContent(content)
.setTitle(title);
boolean sendResponseStatus = true;
List<AlertSendResponseResult> sendResponseResults = new ArrayList<>();
......@@ -126,7 +125,7 @@ public class AlertSender {
for (AlertPluginInstance instance : alertInstanceList) {
AlertResult alertResult = this.alertResultHandler(instance, alertData);
AlertSendResponseResult alertSendResponseResult = new AlertSendResponseResult(
Boolean.parseBoolean(String.valueOf(alertResult.getStatus())), alertResult.getMessage());
Boolean.parseBoolean(String.valueOf(alertResult.getStatus())), alertResult.getMessage());
sendResponseStatus = sendResponseStatus && alertSendResponseResult.getStatus();
sendResponseResults.add(alertSendResponseResult);
}
......@@ -142,7 +141,7 @@ public class AlertSender {
* @return AlertResult
*/
private AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) {
String pluginName = pluginDao.getPluginDefineById(instance.getPluginDefineId()).getPluginName();
String pluginName = alertPluginManager.getPluginNameById(instance.getPluginDefineId());
AlertChannel alertChannel = alertPluginManager.getAlertChannelMap().get(pluginName);
AlertResult alertResultExtend = new AlertResult();
String pluginInstanceName = instance.getInstanceName();
......@@ -156,8 +155,16 @@ public class AlertSender {
AlertInfo alertInfo = new AlertInfo();
alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(instance.getPluginInstanceParams());
AlertResult alertResult = alertChannel.process(alertInfo);
Map<String, String> paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams());
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult;
try {
alertResult = alertChannel.process(alertInfo);
} catch (Exception e) {
alertResult = new AlertResult("false", e.getMessage());
logger.error("send alert error alert data id :{},", alertData.getId(), e);
}
if (alertResult == null) {
String message = String.format("Alert Plugin %s send error : return alertResult value is null", pluginInstanceName);
......
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/AlertServerTest.java
浏览文件 @ dad94369
......@@ -34,12 +34,13 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({AlertServer.class,DaoFactory.class})
@PrepareForTest({AlertServer.class, DaoFactory.class})
public class AlertServerTest {
@Before
......@@ -61,7 +62,8 @@ public class AlertServerTest {
AlertPluginManager alertPluginManager = PowerMockito.mock(AlertPluginManager.class);
PowerMockito.whenNew(AlertPluginManager.class).withNoArguments().thenReturn(alertPluginManager);
ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put("pluginName",alertChannelMock);
alertChannelMap.put("pluginName", alertChannelMock);
PowerMockito.when(alertPluginManager.getPluginNameById(Mockito.anyInt())).thenReturn("pluginName");
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
DolphinPluginManagerConfig alertPluginManagerConfig = PowerMockito.mock(DolphinPluginManagerConfig.class);
......@@ -79,7 +81,8 @@ public class AlertServerTest {
Assert.assertNotNull(alertServer);
new Thread(() -> {
alertServer.start(); })
alertServer.start();
})
.start();
Thread.sleep(5 * Constants.ALERT_SCAN_INTERVAL);
......
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java
浏览文件 @ dad94369
......@@ -136,7 +136,7 @@ public class EmailAlertPluginTest {
alertPluginInstance.setPluginInstanceParams(getEmailAlertParams());
alertDao.getAlertPluginInstanceMapper().insert(alertPluginInstance);
AlertSender alertSender = new AlertSender(alertList, alertDao, alertPluginManager, pluginDao);
AlertSender alertSender = new AlertSender(alertList, alertDao, alertPluginManager);
alertSender.run();
Alert alertResult = alertDao.getAlertMapper().selectById(alert1.getId());
......
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessorTest.java
浏览文件 @ dad94369
......@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.processor;
import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
......@@ -37,7 +36,6 @@ import io.netty.channel.Channel;
public class AlertRequestProcessorTest {
private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager;
private AlertRequestProcessor alertRequestProcessor;
......@@ -45,17 +43,16 @@ public class AlertRequestProcessorTest {
@Before
public void before() {
alertDao = PowerMockito.mock(AlertDao.class);
pluginDao = PowerMockito.mock(PluginDao.class);
alertPluginManager = PowerMockito.mock(AlertPluginManager.class);
alertRequestProcessor = new AlertRequestProcessor(alertDao,alertPluginManager,pluginDao);
alertRequestProcessor = new AlertRequestProcessor(alertDao, alertPluginManager);
}
@Test
public void testProcess() {
Channel channel = PowerMockito.mock(Channel.class);
AlertSendRequestCommand alertSendRequestCommand = new AlertSendRequestCommand(1,"title","content");
AlertSendRequestCommand alertSendRequestCommand = new AlertSendRequestCommand(1, "title", "content");
Command reqCommand = alertSendRequestCommand.convert2Command();
Assert.assertEquals(CommandType.ALERT_SEND_REQUEST,reqCommand.getType());
alertRequestProcessor.process(channel,reqCommand);
Assert.assertEquals(CommandType.ALERT_SEND_REQUEST, reqCommand.getType());
alertRequestProcessor.process(channel, reqCommand);
}
}
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java
浏览文件 @ dad94369
......@@ -67,7 +67,7 @@ public class AlertSenderTest {
int alertGroupId = 1;
String title = "alert mail test title";
String content = "alert mail test content";
alertSender = new AlertSender(alertDao,alertPluginManager,pluginDao);
alertSender = new AlertSender(alertDao, alertPluginManager);
//1.alert instance does not exist
PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(null);
......@@ -75,7 +75,7 @@ public class AlertSenderTest {
AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage()));
logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//2.alert plugin does not exist
int pluginDefineId = 1;
......@@ -83,30 +83,31 @@ public class AlertSenderTest {
String pluginInstanceName = "alert-instance-mail";
List<AlertPluginInstance> alertInstanceList = new ArrayList<>();
AlertPluginInstance alertPluginInstance = new AlertPluginInstance(
pluginDefineId,pluginInstanceParams,pluginInstanceName);
pluginDefineId, pluginInstanceParams, pluginInstanceName);
alertInstanceList.add(alertPluginInstance);
PowerMockito.when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList);
String pluginName = "alert-plugin-mail";
PluginDefine pluginDefine = new PluginDefine(pluginName,"1",null);
PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null);
PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage()));
logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//3.alert result value is null
AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class);
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(null);
Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock);
alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
PowerMockito.when(alertPluginManager.getPluginNameById(Mockito.anyInt())).thenReturn("alert-plugin-mail");
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage()));
logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//4.abnormal information inside the alert plug-in code
AlertResult alertResult = new AlertResult();
......@@ -114,27 +115,27 @@ public class AlertSenderTest {
alertResult.setMessage("Abnormal information inside the alert plug-in code");
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock);
alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage()));
logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//5.alert plugin send success
alertResult = new AlertResult();
alertResult.setStatus(String.valueOf(true));
alertResult.setMessage(String.format("Alert Plugin %s send success",pluginInstanceName));
alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName));
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock);
alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertTrue(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage()));
logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
}
......@@ -150,28 +151,29 @@ public class AlertSenderTest {
alert.setContent(content);
alertList.add(alert);
alertSender = new AlertSender(alertList,alertDao,alertPluginManager,pluginDao);
alertSender = new AlertSender(alertList, alertDao, alertPluginManager);
int pluginDefineId = 1;
String pluginInstanceParams = "alert-instance-mail-params";
String pluginInstanceName = "alert-instance-mail";
List<AlertPluginInstance> alertInstanceList = new ArrayList<>();
AlertPluginInstance alertPluginInstance = new AlertPluginInstance(
pluginDefineId,pluginInstanceParams,pluginInstanceName);
pluginDefineId, pluginInstanceParams, pluginInstanceName);
alertInstanceList.add(alertPluginInstance);
PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(alertInstanceList);
String pluginName = "alert-plugin-mail";
PluginDefine pluginDefine = new PluginDefine(pluginName,"1",null);
PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null);
PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine);
PowerMockito.when(alertPluginManager.getPluginNameById(1)).thenReturn("alert-instance-mail");
AlertResult alertResult = new AlertResult();
alertResult.setStatus(String.valueOf(true));
alertResult.setMessage(String.format("Alert Plugin %s send success",pluginInstanceName));
alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName));
AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class);
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock);
alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
Assert.assertTrue(Boolean.parseBoolean(alertResult.getStatus()));
alertSender.run();
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
浏览文件 @ dad94369
......@@ -17,17 +17,20 @@
package org.apache.dolphinscheduler.api.configuration;
import org.apache.dolphinscheduler.api.interceptor.LocaleChangeInterceptor;
import org.apache.dolphinscheduler.api.interceptor.LoginHandlerInterceptor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
import org.springframework.web.servlet.LocaleResolver;
import org.springframework.web.servlet.config.annotation.*;
import org.springframework.web.servlet.i18n.CookieLocaleResolver;
import org.springframework.web.servlet.i18n.LocaleChangeInterceptor;
import java.util.Locale;
/**
* application configuration
*/
......@@ -39,7 +42,17 @@ public class AppConfiguration implements WebMvcConfigurer {
public static final String REGISTER_PATH_PATTERN = "/users/register";
public static final String PATH_PATTERN = "/**";
public static final String LOCALE_LANGUAGE_COOKIE = "language";
public static final int COOKIE_MAX_AGE = 3600;
@Bean
public CorsFilter corsFilter() {
CorsConfiguration config = new CorsConfiguration();
config.addAllowedOrigin("*");
config.addAllowedMethod("*");
config.addAllowedHeader("*");
UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource();
configSource.registerCorsConfiguration(PATH_PATTERN, config);
return new CorsFilter(configSource);
}
@Bean
public LoginHandlerInterceptor loginInterceptor() {
......@@ -56,16 +69,14 @@ public class AppConfiguration implements WebMvcConfigurer {
localeResolver.setCookieName(LOCALE_LANGUAGE_COOKIE);
// set default locale
localeResolver.setDefaultLocale(Locale.US);
// set cookie max age
localeResolver.setCookieMaxAge(COOKIE_MAX_AGE);
// set language tag compliant
localeResolver.setLanguageTagCompliant(false);
return localeResolver;
}
@Bean
public LocaleChangeInterceptor localeChangeInterceptor() {
LocaleChangeInterceptor lci = new LocaleChangeInterceptor();
lci.setParamName("language");
return lci;
return new LocaleChangeInterceptor();
}
@Override
......@@ -94,11 +105,6 @@ public class AppConfiguration implements WebMvcConfigurer {
registry.addViewController("/").setViewName("forward:/ui/index.html");
}
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping(PATH_PATTERN).allowedOrigins("*").allowedMethods("*");
}
/**
* Turn off suffix-based content negotiation
*
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
浏览文件 @ dad94369
......@@ -133,7 +133,7 @@ public enum Status {
QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"),
DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"),
CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"),
VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称已存在"),
VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称验证错误"),
UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"),
RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"),
QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"),
......@@ -199,6 +199,7 @@ public enum Status {
FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}]，无法执行强制成功操作"),
DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"),
PROCESS_DEFINITION_NAME_EXIST(10168, "process definition name {0} already exists", "工作流定义名称[{0}]已存在"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),
......@@ -290,6 +291,7 @@ public enum Status {
QUERY_ALL_ALERT_PLUGIN_INSTANCE_ERROR(110009, "query all alert plugin instance error", "查询所有告警实例失败"),
PLUGIN_INSTANCE_ALREADY_EXIT(110010,"plugin instance already exit","该告警插件实例已存在"),
LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011,"query plugin instance page error","分页查询告警实例失败"),
DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED(110012,"failed to delete the alert instance, there is an alarm group associated with this alert instance","删除告警实例失败，存在与此告警实例关联的警报组")
;
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LocaleChangeInterceptor.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.common.Constants;
import java.util.Locale;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.lang.Nullable;
import org.springframework.util.StringUtils;
import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
import org.springframework.web.util.WebUtils;
public class LocaleChangeInterceptor extends HandlerInterceptorAdapter {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Cookie cookie = WebUtils.getCookie(request, Constants.LOCALE_LANGUAGE);
if (cookie != null) {
// Proceed in cookie
return true;
}
// Proceed in header
String newLocale = request.getHeader(Constants.LOCALE_LANGUAGE);
if (newLocale != null) {
LocaleContextHolder.setLocale(parseLocaleValue(newLocale));
}
return true;
}
@Nullable
protected Locale parseLocaleValue(String localeValue) {
return StringUtils.parseLocale(localeValue);
}
}
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
浏览文件 @ dad94369
......@@ -113,27 +113,6 @@ public class BaseService {
return false;
}
/**
* get cookie info by name
*
* @param request request
* @param name 'sessionId'
* @return get cookie info
*/
public static Cookie getCookie(HttpServletRequest request, String name) {
Cookie[] cookies = request.getCookies();
if (cookies != null && cookies.length > 0) {
for (Cookie cookie : cookies) {
if (StringUtils.isNotEmpty(name) && name.equalsIgnoreCase(cookie.getName())) {
return cookie;
}
}
}
return null;
}
/**
* create tenant dir if not exists
*
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
浏览文件 @ dad94369
......@@ -428,10 +428,9 @@ public class ProcessInstanceService extends BaseService {
return result;
}
Date schedule = null;
schedule = processInstance.getScheduleTime();
if (scheduleTime != null) {
schedule = DateUtils.getScheduleDate(scheduleTime);
} else {
schedule = processInstance.getScheduleTime();
}
processInstance.setScheduleTime(schedule);
processInstance.setLocations(locations);
......@@ -460,13 +459,18 @@ public class ProcessInstanceService extends BaseService {
if (tenant != null) {
processInstance.setTenantCode(tenant.getTenantCode());
}
// get the processinstancejson before saving,and then save the name and taskid
String oldJson = processInstance.getProcessInstanceJson();
if (StringUtils.isNotEmpty(oldJson)) {
processInstanceJson = processService.changeJson(processData,oldJson);
}
processInstance.setProcessInstanceJson(processInstanceJson);
processInstance.setGlobalParams(globalParams);
}
int update = processService.updateProcessInstance(processInstance);
int updateDefine = 1;
if (Boolean.TRUE.equals(syncDefine) && StringUtils.isNotEmpty(processInstanceJson)) {
if (Boolean.TRUE.equals(syncDefine)) {
processDefinition.setProcessDefinitionJson(processInstanceJson);
processDefinition.setGlobalParams(originDefParams);
processDefinition.setLocations(locations);
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
浏览文件 @ dad94369
......@@ -153,19 +153,21 @@ public class WorkerGroupService extends BaseService {
}
}
// available workerGroup list
List<String> availableWorkerGroupList = new ArrayList<>();
for (String workerGroup : workerGroupList) {
String workerGroupPath = workerPath + "/" + workerGroup;
List<String> childrenNodes = zookeeperCachedOperator.getChildrenKeys(workerGroupPath);
String timeStamp = "";
for (int i = 0; i < childrenNodes.size(); i++) {
String ip = childrenNodes.get(i);
childrenNodes.set(i, ip.substring(0, ip.lastIndexOf(":")));
timeStamp = ip.substring(ip.lastIndexOf(":"));
}
if (CollectionUtils.isNotEmpty(childrenNodes)) {
availableWorkerGroupList.add(workerGroup);
WorkerGroup wg = new WorkerGroup();
wg.setName(workerGroup);
if (isPaging) {
wg.setIpList(childrenNodes);
String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0));
String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0) + timeStamp);
wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6]));
wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[7]));
}
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
浏览文件 @ dad94369
......@@ -24,16 +24,25 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.commons.collections4.MapUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -56,6 +65,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Autowired
private PluginDefineMapper pluginDefineMapper;
@Autowired
private AlertGroupMapper alertGroupMapper;
/**
* creat alert plugin instance
*
......@@ -67,7 +79,8 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Override
public Map<String, Object> create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams) {
AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setPluginInstanceParams(pluginInstanceParams);
String paramsMapJson = parsePluginParamsMap(pluginInstanceParams);
alertPluginInstance.setPluginInstanceParams(paramsMapJson);
alertPluginInstance.setInstanceName(instanceName);
alertPluginInstance.setPluginDefineId(pluginDefineId);
......@@ -82,7 +95,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (i > 0) {
putMsg(result, Status.SUCCESS);
return result;
}
putMsg(result, Status.SAVE_ERROR);
return result;
}
......@@ -98,7 +113,8 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
public Map<String, Object> update(User loginUser, int pluginInstanceId, String instanceName, String pluginInstanceParams) {
AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setPluginInstanceParams(pluginInstanceParams);
String paramsMapJson = parsePluginParamsMap(pluginInstanceParams);
alertPluginInstance.setPluginInstanceParams(paramsMapJson);
alertPluginInstance.setInstanceName(instanceName);
alertPluginInstance.setId(pluginInstanceId);
Map<String, Object> result = new HashMap<>();
......@@ -106,8 +122,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (i > 0) {
putMsg(result, Status.SUCCESS);
return result;
}
putMsg(result, Status.SAVE_ERROR);
return result;
}
......@@ -121,6 +138,13 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Override
public Map<String, Object> delete(User loginUser, int id) {
Map<String, Object> result = new HashMap<>();
//check if there is an associated alert group
boolean hasAssociatedAlertGroup = checkHasAssociatedAlertGroup(String.valueOf(id));
if (hasAssociatedAlertGroup) {
putMsg(result, Status.DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED);
return result;
}
int i = alertPluginInstanceMapper.deleteById(id);
if (i > 0) {
putMsg(result, Status.SUCCESS);
......@@ -188,21 +212,73 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (CollectionUtils.isEmpty(pluginDefineList)) {
return null;
}
Map<Integer, String> pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, PluginDefine::getPluginName));
Map<Integer, PluginDefine> pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity()));
List<AlertPluginInstanceVO> alertPluginInstanceVOS = new ArrayList<>();
alertPluginInstances.forEach(alertPluginInstance -> {
AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO();
alertPluginInstanceVO.setAlertPluginName(pluginDefineMap.get(alertPluginInstance.getPluginDefineId()));
alertPluginInstanceVO.setCreateTime(alertPluginInstance.getCreateTime());
alertPluginInstanceVO.setUpdateTime(alertPluginInstance.getUpdateTime());
alertPluginInstanceVO.setPluginDefineId(alertPluginInstance.getPluginDefineId());
alertPluginInstanceVO.setInstanceName(alertPluginInstance.getInstanceName());
alertPluginInstanceVO.setId(alertPluginInstance.getId());
PluginDefine pluginDefine = pluginDefineMap.get(alertPluginInstance.getPluginDefineId());
//FIXME When the user removes the plug-in, this will happen. At this time, maybe we should add a new field to indicate that the plug-in has expired?
if (null == pluginDefine) {
return;
}
alertPluginInstanceVO.setAlertPluginName(pluginDefine.getPluginName());
//todo List pages do not recommend returning this parameter
alertPluginInstanceVO.setPluginInstanceParams(alertPluginInstance.getPluginInstanceParams());
String pluginParamsMapString = alertPluginInstance.getPluginInstanceParams();
String uiPluginParams = parseToPluginUiParams(pluginParamsMapString, pluginDefine.getPluginParams());
alertPluginInstanceVO.setPluginInstanceParams(uiPluginParams);
alertPluginInstanceVOS.add(alertPluginInstanceVO);
});
return alertPluginInstanceVOS;
}
/**
* Get the parameters actually needed by the plugin
*
* @param pluginParams Complete parameters(include ui)
* @return k, v(json string)
*/
private String parsePluginParamsMap(String pluginParams) {
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(pluginParams);
return JSONUtils.toJsonString(paramsMap);
}
/**
* parseToPluginUiParams
*
* @param pluginParamsMapString k-v data
* @param pluginUiParams Complete parameters(include ui)
* @return Complete parameters list(include ui)
*/
private String parseToPluginUiParams(String pluginParamsMapString, String pluginUiParams) {
Map<String, String> paramsMap = JSONUtils.toMap(pluginParamsMapString);
if (MapUtils.isEmpty(paramsMap)) {
return null;
}
List<PluginParams> pluginParamsList = JSONUtils.toList(pluginUiParams, PluginParams.class);
List<PluginParams> newPluginParamsList = new ArrayList<>(pluginParamsList.size());
pluginParamsList.forEach(pluginParams -> {
pluginParams.setValue(paramsMap.get(pluginParams.getName()));
newPluginParamsList.add(pluginParams);
});
return JSONUtils.toJsonString(newPluginParamsList);
}
private boolean checkHasAssociatedAlertGroup(String id) {
List<String> idsList = alertGroupMapper.queryInstanceIdsList();
if (CollectionUtils.isEmpty(idsList)) {
return false;
}
Optional<String> first = idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst();
return first.isPresent();
}
}
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
浏览文件 @ dad94369
......@@ -429,11 +429,13 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
// check whether the new process define name exist
ProcessDefinition definition = processDefineMapper.verifyByDefineName(project.getId(), name);
if (definition != null) {
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
putMsg(result, Status.PROCESS_DEFINITION_NAME_EXIST, name);
return result;
}
}
// get the processdefinitionjson before saving,and then save the name and taskid
String oldJson = processDefine.getProcessDefinitionJson();
processDefinitionJson = processService.changeJson(processData,oldJson);
Date now = new Date();
processDefine.setId(id);
......@@ -495,7 +497,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (processDefinition == null) {
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
putMsg(result, Status.PROCESS_DEFINITION_NAME_EXIST, name);
}
return result;
}
......
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
浏览文件 @ dad94369
......@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.util.WebUtils;
/**
* session service implement
......@@ -60,7 +61,7 @@ public class SessionServiceImpl extends BaseService implements SessionService {
String sessionId = request.getHeader(Constants.SESSION_ID);
if (StringUtils.isBlank(sessionId)) {
Cookie cookie = getCookie(request, Constants.SESSION_ID);
Cookie cookie = WebUtils.getCookie(request, Constants.SESSION_ID);
if (cookie != null) {
sessionId = cookie.getValue();
......
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
浏览文件 @ dad94369
......@@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest {
public void testVerifyProcessDefinitionName() throws Exception {
Map<String, Object> result = new HashMap<>();
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR);
putMsg(result, Status.PROCESS_DEFINITION_NAME_EXIST);
String projectName = "test";
String name = "dag_test";
Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result);
Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name);
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue());
Assert.assertEquals(Status.PROCESS_DEFINITION_NAME_EXIST.getCode(), response.getCode().intValue());
}
......
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/LocaleChangeInterceptorTest.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class LocaleChangeInterceptorTest {
@Autowired
LocaleChangeInterceptor interceptor;
@Test
public void testPreHandle() {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
// test no language
Assert.assertTrue(interceptor.preHandle(request, response, null));
}
}
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.AlertPluginInstanceServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class AlertPluginInstanceServiceTest {
@InjectMocks
AlertPluginInstanceServiceImpl alertPluginInstanceService;
@Mock
private AlertPluginInstanceMapper alertPluginInstanceMapper;
@Mock
private PluginDefineMapper pluginDefineMapper;
@Mock
private AlertGroupMapper alertGroupMapper;
private List<AlertPluginInstance> alertPluginInstances;
private User user;
private String uiParams = "[\n"
+ " {\n"
+ " \"field\":\"userParams\",\n"
+ " \"name\":\"user.params\",\n"
+ " \"props\":{\n"
+ " \"placeholder\":\"please enter your custom parameters, which will be passed to you when calling your script\",\n"
+ " \"size\":\"small\"\n"
+ " },\n"
+ " \"type\":\"input\",\n"
+ " \"title\":\"user.params\",\n"
+ " \"value\":\"userParams\",\n"
+ " \"validate\":[\n"
+ " {\n"
+ " \"required\":false,\n"
+ " \"message\":null,\n"
+ " \"type\":\"string\",\n"
+ " \"trigger\":\"blur\",\n"
+ " \"min\":null,\n"
+ " \"max\":null\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " {\n"
+ " \"field\":\"path\",\n"
+ " \"name\":\"path\",\n"
+ " \"props\":{\n"
+ " \"placeholder\":\"please upload the file to the disk directory of the alert server, and ensure that the path is absolute and has the corresponding access rights\",\n"
+ " \"size\":\"small\"\n"
+ " },\n"
+ " \"type\":\"input\",\n"
+ " \"title\":\"path\",\n"
+ " \"value\":\"/kris/script/path\",\n"
+ " \"validate\":[\n"
+ " {\n"
+ " \"required\":true,\n"
+ " \"message\":null,\n"
+ " \"type\":\"string\",\n"
+ " \"trigger\":\"blur\",\n"
+ " \"min\":null,\n"
+ " \"max\":null\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " {\n"
+ " \"field\":\"type\",\n"
+ " \"name\":\"type\",\n"
+ " \"props\":{\n"
+ " \"placeholder\":null,\n"
+ " \"size\":\"small\"\n"
+ " },\n"
+ " \"type\":\"radio\",\n"
+ " \"title\":\"type\",\n"
+ " \"value\":0,\n"
+ " \"validate\":[\n"
+ " {\n"
+ " \"required\":true,\n"
+ " \"message\":null,\n"
+ " \"type\":\"string\",\n"
+ " \"trigger\":\"blur\",\n"
+ " \"min\":null,\n"
+ " \"max\":null\n"
+ " }\n"
+ " ],\n"
+ " \"options\":[\n"
+ " {\n"
+ " \"label\":\"SHELL\",\n"
+ " \"value\":0,\n"
+ " \"disabled\":false\n"
+ " }\n"
+ " ]\n"
+ " }\n"
+ "]\n"
+ "\n";
private String paramsMap = "{\"path\":\"/kris/script/path\",\"userParams\":\"userParams\",\"type\":\"0\"}";
@Before
public void before() {
user = new User();
user.setUserType(UserType.ADMIN_USER);
user.setId(1);
AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setPluginInstanceParams("test1");
alertPluginInstance.setPluginDefineId(1);
alertPluginInstance.setId(1);
alertPluginInstance.setPluginInstanceParams("test");
alertPluginInstances = new ArrayList<>();
alertPluginInstances.add(alertPluginInstance);
}
@Test
public void testCreate() {
Mockito.when(alertPluginInstanceMapper.queryByInstanceName("test")).thenReturn(alertPluginInstances);
Map<String, Object> result = alertPluginInstanceService.create(user, 1, "test", uiParams);
Assert.assertEquals(Status.PLUGIN_INSTANCE_ALREADY_EXIT, result.get(Constants.STATUS));
Mockito.when(alertPluginInstanceMapper.insert(Mockito.any())).thenReturn(1);
result = alertPluginInstanceService.create(user, 1, "test1", uiParams);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testDelete() {
List<String> ids = Arrays.asList("11,2,3", null, "98,1");
Mockito.when(alertGroupMapper.queryInstanceIdsList()).thenReturn(ids);
Map<String, Object> result = alertPluginInstanceService.delete(user, 1);
Assert.assertEquals(Status.DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED, result.get(Constants.STATUS));
Mockito.when(alertPluginInstanceMapper.deleteById(9)).thenReturn(1);
result = alertPluginInstanceService.delete(user, 9);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testUpdate() {
Mockito.when(alertPluginInstanceMapper.updateById(Mockito.any())).thenReturn(0);
Map<String, Object> result = alertPluginInstanceService.update(user, 1, "testUpdate", uiParams);
Assert.assertEquals(Status.SAVE_ERROR, result.get(Constants.STATUS));
Mockito.when(alertPluginInstanceMapper.updateById(Mockito.any())).thenReturn(1);
result = alertPluginInstanceService.update(user, 1, "testUpdate", uiParams);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@Test
public void testQueryAll() {
AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setId(1);
alertPluginInstance.setPluginDefineId(1);
alertPluginInstance.setPluginInstanceParams(paramsMap);
alertPluginInstance.setInstanceName("test");
PluginDefine pluginDefine = new PluginDefine("script", "script", uiParams);
pluginDefine.setId(1);
List<PluginDefine> pluginDefines = Collections.singletonList(pluginDefine);
List<AlertPluginInstance> pluginInstanceList = Collections.singletonList(alertPluginInstance);
Mockito.when(alertPluginInstanceMapper.queryAllAlertPluginInstanceList()).thenReturn(pluginInstanceList);
Mockito.when(pluginDefineMapper.queryAllPluginDefineList()).thenReturn(pluginDefines);
Map<String, Object> result = alertPluginInstanceService.queryAll();
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
}
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java
浏览文件 @ dad94369
......@@ -94,20 +94,6 @@ public class BaseServiceTest {
baseService.putMsg(result,Status.PROJECT_NOT_FOUNT,"test");
}
@Test
public void testGetCookie(){
MockHttpServletRequest request = new MockHttpServletRequest();
MockCookie mockCookie = new MockCookie("userId","1");
request.setCookies(mockCookie);
//cookie is not null
Cookie cookie = BaseService.getCookie(request,"userId");
Assert.assertNotNull(cookie);
//cookie is null
cookie = BaseService.getCookie(request,"userName");
Assert.assertNull(cookie);
}
@Test
public void testCreateTenantDirIfNotExists(){
PowerMockito.mockStatic(HadoopUtils.class);
......
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
浏览文件 @ dad94369
......@@ -657,7 +657,7 @@ public class ProcessDefinitionServiceTest {
Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition());
Map<String, Object> processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser,
"project_test1", "test_pdf");
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS));
Assert.assertEquals(Status.PROCESS_DEFINITION_NAME_EXIST, processExistRes.get(Constants.STATUS));
}
@Test
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
浏览文件 @ dad94369
......@@ -906,6 +906,11 @@ public final class Constants {
public static final String PASSWORD_DEFAULT = "******";
/**
* locale
*/
public static final String LOCALE_LANGUAGE = "language";
/**
* driver
*/
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java
浏览文件 @ dad94369
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.task.conditions;
import org.apache.dolphinscheduler.common.enums.DependentRelation;
......@@ -36,7 +37,6 @@ public class ConditionsParameters extends AbstractParameters {
// node list to run when failed
private List<String> failedNode;
@Override
public boolean checkParameters() {
return true;
......@@ -78,4 +78,12 @@ public class ConditionsParameters extends AbstractParameters {
public void setFailedNode(List<String> failedNode) {
this.failedNode = failedNode;
}
public String getConditionResult() {
return "{"
+ "\"successNode\": [\"" + successNode.get(0)
+ "\"],\"failedNode\": [\"" + failedNode.get(0)
+ "\"]}";
}
}
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
浏览文件 @ dad94369
......@@ -263,13 +263,13 @@ public class CollectionUtils {
}
Map<String, Object> instanceMap;
for (T instance : originList) {
Map<String, Object> dataMap = new BeanMap(instance);
BeanMap beanMap = new BeanMap(instance);
instanceMap = new LinkedHashMap<>(16, 0.75f, true);
for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
for (Map.Entry<Object, Object> entry : beanMap.entrySet()) {
if (exclusionSet.contains(entry.getKey())) {
continue;
}
instanceMap.put(entry.getKey(), entry.getValue());
instanceMap.put((String) entry.getKey(), entry.getValue());
}
instanceList.add(instanceMap);
}
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
浏览文件 @ dad94369
......@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.lang.management.OperatingSystemMXBean;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
......@@ -130,11 +131,17 @@ public class OSUtils {
* @return load average
*/
public static double loadAverage() {
double loadAverage = hal.getProcessor().getSystemLoadAverage();
if (Double.isNaN(loadAverage)) {
return NEGATIVE_ONE;
double loadAverage;
try {
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
loadAverage = osBean.getSystemLoadAverage();
} catch (Exception e) {
logger.error("get operation system load average exception, try another method ", e);
loadAverage = hal.getProcessor().getSystemLoadAverage();
if (Double.isNaN(loadAverage)) {
return NEGATIVE_ONE;
}
}
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(loadAverage));
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
浏览文件 @ dad94369
......@@ -61,4 +61,8 @@ public class StringUtils {
public static String trim(String str) {
return str == null ? null : str.trim();
}
public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
}
}
dolphinscheduler-common/src/main/resources/common.properties
浏览文件 @ dad94369
......@@ -62,7 +62,7 @@ yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000)
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
# system env path
# system env path, If you want to set your own path, you need to set this env file to an absolute path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
development.state=false
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
浏览文件 @ dad94369
......@@ -58,22 +58,22 @@ public class PluginDao extends AbstractBaseDao {
*
* @param pluginDefine new pluginDefine
*/
public void addOrUpdatePluginDefine(PluginDefine pluginDefine) {
public int addOrUpdatePluginDefine(PluginDefine pluginDefine) {
requireNonNull(pluginDefine, "pluginDefine is null");
requireNonNull(pluginDefine.getPluginName(), "pluginName is null");
requireNonNull(pluginDefine.getPluginType(), "pluginType is null");
List<PluginDefine> pluginDefineList = pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType());
if (pluginDefineList == null || pluginDefineList.size() == 0) {
pluginDefineMapper.insert(pluginDefine);
} else {
PluginDefine currPluginDefine = pluginDefineList.get(0);
if (!currPluginDefine.getPluginParams().equals(pluginDefine.getPluginParams())) {
currPluginDefine.setUpdateTime(pluginDefine.getUpdateTime());
currPluginDefine.setPluginParams(pluginDefine.getPluginParams());
pluginDefineMapper.updateById(currPluginDefine);
}
return pluginDefineMapper.insert(pluginDefine);
}
PluginDefine currPluginDefine = pluginDefineList.get(0);
if (!currPluginDefine.getPluginParams().equals(pluginDefine.getPluginParams())) {
currPluginDefine.setUpdateTime(pluginDefine.getUpdateTime());
currPluginDefine.setPluginParams(pluginDefine.getPluginParams());
pluginDefineMapper.updateById(currPluginDefine);
}
return currPluginDefine.getId();
}
/**
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertPluginInstance.java
浏览文件 @ dad94369
......@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.entity;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
......@@ -39,7 +40,7 @@ public class AlertPluginInstance {
/**
* plugin_define_id
*/
@TableField("plugin_define_id")
@TableField(value = "plugin_define_id", updateStrategy = FieldStrategy.NEVER)
private int pluginDefineId;
/**
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java
浏览文件 @ dad94369
......@@ -61,6 +61,12 @@ public interface AlertGroupMapper extends BaseMapper<AlertGroup> {
*/
List<AlertGroup> queryAllGroupList();
/**
* query instance ids All
* @return list
*/
List<String> queryInstanceIdsList();
/**
* queryAlertGroupInstanceIdsById
* @param alertGroupId
......
dolphinscheduler-dao/src/main/resources/datasource.properties
浏览文件 @ dad94369
......@@ -14,12 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
#spring.datasource.username=root
#spring.datasource.password=123456
# postgresql
spring.datasource.driver-class-name=org.postgresql.Driver
......@@ -33,7 +27,6 @@ spring.datasource.password=test
#spring.datasource.username=xxxx
#spring.datasource.password=xxxx
# connection configuration
#spring.datasource.initialSize=5
# min connection number
......
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.xml
浏览文件 @ dad94369
......@@ -52,6 +52,13 @@
order by update_time desc
</select>
<select id="queryInstanceIdsList" resultType="String">
select
alert_instance_ids
from t_ds_alertgroup
order by update_time desc
</select>
<select id="queryAlertGroupInstanceIdsById" resultType="String">
select alert_instance_ids from t_ds_alertgroup
where id = #{alertGroupId}
......
dolphinscheduler-dist/release-docs/LICENSE
浏览文件 @ dad94369
......@@ -227,7 +227,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
byte-buddy 1.9.16: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.16, Apache 2.0
classmate 1.4.0: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.4.0, Apache 2.0
clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0
commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0
commons-beanutils 1.9.4 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.9.4, Apache 2.0
commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0
commons-codec 1.11: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.11, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0
......
dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-@form-create-element-ui 已删除 100644 → 0
浏览文件 @ fba133c9
MIT License
Copyright (c) 2018 xaboy
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml
浏览文件 @ dad94369
......@@ -64,4 +64,9 @@
<unpack/>
</artifact>
</artifactSet>
<artifactSet to="lib/plugin/alert/feishu">
<artifact id="${project.groupId}:dolphinscheduler-alert-feishu:zip:${project.version}">
<unpack/>
</artifact>
</artifactSet>
</runtime>
\ No newline at end of file
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java
浏览文件 @ dad94369
......@@ -55,7 +55,6 @@ import org.slf4j.LoggerFactory;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
......@@ -84,7 +83,7 @@ public class NettyRemotingClient {
/**
* channels
*/
private final ConcurrentHashMap<Host, Channel> channels = new ConcurrentHashMap(128);
private final ConcurrentHashMap<Host, Channel> channels = new ConcurrentHashMap<>(128);
/**
* started flag
......@@ -130,7 +129,7 @@ public class NettyRemotingClient {
this.clientConfig = clientConfig;
if (NettyUtils.useEpoll()) {
this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -139,7 +138,7 @@ public class NettyRemotingClient {
});
} else {
this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -148,8 +147,8 @@ public class NettyRemotingClient {
});
}
this.callbackExecutor = new ThreadPoolExecutor(5, 10, 1, TimeUnit.MINUTES,
new LinkedBlockingQueue<>(1000), new NamedThreadFactory("CallbackExecutor", 10),
new CallerThreadExecutePolicy());
new LinkedBlockingQueue<>(1000), new NamedThreadFactory("CallbackExecutor", 10),
new CallerThreadExecutePolicy());
this.clientHandler = new NettyClientHandler(this, callbackExecutor);
this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor"));
......@@ -163,40 +162,32 @@ public class NettyRemotingClient {
private void start() {
this.bootstrap
.group(this.workerGroup)
.channel(NettyUtils.getSocketChannelClass())
.option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive())
.option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay())
.option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize())
.option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize())
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis())
.handler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ch.pipeline()
.addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS))
.addLast(new NettyDecoder(), clientHandler, encoder);
}
});
this.responseFutureExecutor.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
ResponseFuture.scanFutureTable();
}
}, 5000, 1000, TimeUnit.MILLISECONDS);
//
.group(this.workerGroup)
.channel(NettyUtils.getSocketChannelClass())
.option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive())
.option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay())
.option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize())
.option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize())
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis())
.handler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ch.pipeline()
.addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS))
.addLast(new NettyDecoder(), clientHandler, encoder);
}
});
this.responseFutureExecutor.scheduleAtFixedRate(ResponseFuture::scanFutureTable, 5000, 1000, TimeUnit.MILLISECONDS);
isStarted.compareAndSet(false, true);
}
/**
* async send
*
* @param host host
* @param command command
* @param timeoutMillis timeoutMillis
* @param host host
* @param command command
* @param timeoutMillis timeoutMillis
* @param invokeCallback callback function
* @throws InterruptedException
* @throws RemotingException
*/
public void sendAsync(final Host host, final Command command,
final long timeoutMillis,
......@@ -205,53 +196,49 @@ public class NettyRemotingClient {
if (channel == null) {
throw new RemotingException("network error");
}
/**
/*
* request unique identification
*/
final long opaque = command.getOpaque();
/**
/*
* control concurrency number
*/
boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
if (acquired) {
final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore);
/**
/*
* response future
*/
final ResponseFuture responseFuture = new ResponseFuture(opaque,
timeoutMillis,
invokeCallback,
releaseSemaphore);
timeoutMillis,
invokeCallback,
releaseSemaphore);
try {
channel.writeAndFlush(command).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
} else {
responseFuture.setSendOk(false);
}
responseFuture.setCause(future.cause());
responseFuture.putResponse(null);
try {
responseFuture.executeInvokeCallback();
} catch (Throwable ex) {
logger.error("execute callback error", ex);
} finally {
responseFuture.release();
}
channel.writeAndFlush(command).addListener(future -> {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
} else {
responseFuture.setSendOk(false);
}
responseFuture.setCause(future.cause());
responseFuture.putResponse(null);
try {
responseFuture.executeInvokeCallback();
} catch (Exception ex) {
logger.error("execute callback error", ex);
} finally {
responseFuture.release();
}
});
} catch (Throwable ex) {
} catch (Exception ex) {
responseFuture.release();
throw new RemotingException(String.format("send command to host: %s failed", host), ex);
}
} else {
String message = String.format("try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d",
timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits());
timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits());
throw new RemotingTooMuchRequestException(message);
}
}
......@@ -259,12 +246,10 @@ public class NettyRemotingClient {
/**
* sync send
*
* @param host host
* @param command command
* @param host host
* @param command command
* @param timeoutMillis timeoutMillis
* @return command
* @throws InterruptedException
* @throws RemotingException
*/
public Command sendSync(final Host host, final Command command, final long timeoutMillis) throws InterruptedException, RemotingException {
final Channel channel = getChannel(host);
......@@ -273,21 +258,18 @@ public class NettyRemotingClient {
}
final long opaque = command.getOpaque();
final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null);
channel.writeAndFlush(command).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
} else {
responseFuture.setSendOk(false);
}
responseFuture.setCause(future.cause());
responseFuture.putResponse(null);
logger.error("send command {} to host {} failed", command, host);
channel.writeAndFlush(command).addListener(future -> {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
} else {
responseFuture.setSendOk(false);
}
responseFuture.setCause(future.cause());
responseFuture.putResponse(null);
logger.error("send command {} to host {} failed", command, host);
});
/**
/*
* sync wait for result
*/
Command result = responseFuture.waitResponse();
......@@ -304,9 +286,8 @@ public class NettyRemotingClient {
/**
* send task
*
* @param host host
* @param host host
* @param command command
* @throws RemotingException
*/
public void send(final Host host, final Command command) throws RemotingException {
Channel channel = getChannel(host);
......@@ -332,7 +313,7 @@ public class NettyRemotingClient {
* register processor
*
* @param commandType command type
* @param processor processor
* @param processor processor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) {
this.registerProcessor(commandType, processor, null);
......@@ -342,8 +323,8 @@ public class NettyRemotingClient {
* register processor
*
* @param commandType command type
* @param processor processor
* @param executor thread executor
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
this.clientHandler.registerProcessor(commandType, processor, executor);
......@@ -351,9 +332,6 @@ public class NettyRemotingClient {
/**
* get channel
*
* @param host
* @return
*/
public Channel getChannel(Host host) {
Channel channel = channels.get(host);
......@@ -366,7 +344,7 @@ public class NettyRemotingClient {
/**
* create channel
*
* @param host host
* @param host host
* @param isSync sync flag
* @return channel
*/
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java
浏览文件 @ dad94369
......@@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
import org.apache.dolphinscheduler.remote.exceptions.RemoteException;
import org.apache.dolphinscheduler.remote.handler.NettyServerHandler;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants;
......@@ -93,6 +94,11 @@ public class NettyRemotingServer {
*/
private final AtomicBoolean isStarted = new AtomicBoolean(false);
/**
* Netty server bind fail message
*/
private static final String NETTY_BIND_FAILURE_MSG = "NettyRemotingServer bind %s fail";
/**
* server init
*
......@@ -102,7 +108,7 @@ public class NettyRemotingServer {
this.serverConfig = serverConfig;
if (NettyUtils.useEpoll()) {
this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -111,7 +117,7 @@ public class NettyRemotingServer {
});
this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -120,7 +126,7 @@ public class NettyRemotingServer {
});
} else {
this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -129,7 +135,7 @@ public class NettyRemotingServer {
});
this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() {
private AtomicInteger threadIndex = new AtomicInteger(0);
private final AtomicInteger threadIndex = new AtomicInteger(0);
@Override
public Thread newThread(Runnable r) {
......@@ -145,35 +151,35 @@ public class NettyRemotingServer {
public void start() {
if (isStarted.compareAndSet(false, true)) {
this.serverBootstrap
.group(this.bossGroup, this.workGroup)
.channel(NettyUtils.getServerSocketChannelClass())
.option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog())
.childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive())
.childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay())
.childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize())
.childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize())
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
initNettyChannel(ch);
}
});
.group(this.bossGroup, this.workGroup)
.channel(NettyUtils.getServerSocketChannelClass())
.option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog())
.childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive())
.childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay())
.childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize())
.childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize())
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) {
initNettyChannel(ch);
}
});
ChannelFuture future;
try {
future = serverBootstrap.bind(serverConfig.getListenPort()).sync();
} catch (Exception e) {
logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e);
throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()));
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()));
}
if (future.isSuccess()) {
logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort());
} else if (future.cause() != null) {
throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause());
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()), future.cause());
} else {
throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()));
throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()));
}
}
}
......@@ -185,17 +191,17 @@ public class NettyRemotingServer {
*/
private void initNettyChannel(SocketChannel ch) {
ch.pipeline()
.addLast("encoder", encoder)
.addLast("decoder", new NettyDecoder())
.addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS))
.addLast("handler", serverHandler);
.addLast("encoder", encoder)
.addLast("decoder", new NettyDecoder())
.addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS))
.addLast("handler", serverHandler);
}
/**
* register processor
*
* @param commandType command type
* @param processor processor
* @param processor processor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) {
this.registerProcessor(commandType, processor, null);
......@@ -205,8 +211,8 @@ public class NettyRemotingServer {
* register processor
*
* @param commandType command type
* @param processor processor
* @param executor thread executor
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
this.serverHandler.registerProcessor(commandType, processor, executor);
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java
浏览文件 @ dad94369
......@@ -14,16 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.codec;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.MessageToByteEncoder;
import org.apache.dolphinscheduler.remote.command.Command;
/**
* netty encoder
* netty encoder
*/
@Sharable
public class NettyEncoder extends MessageToByteEncoder<Command> {
......@@ -34,12 +37,11 @@ public class NettyEncoder extends MessageToByteEncoder<Command> {
* @param ctx channel handler context
* @param msg command
* @param out byte buffer
* @throws Exception
*/
@Override
protected void encode(ChannelHandlerContext ctx, Command msg, ByteBuf out) throws Exception {
if(msg == null){
throw new Exception("encode msg is null");
if (msg == null) {
throw new RemotingException("encode msg is null");
}
out.writeByte(Command.MAGIC);
out.writeByte(Command.VERSION);
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java
浏览文件 @ dad94369
......@@ -23,7 +23,7 @@ import java.io.Serializable;
import java.util.List;
/**
* kill task response command
* kill task response command
*/
public class TaskKillResponseCommand implements Serializable {
......@@ -51,7 +51,7 @@ public class TaskKillResponseCommand implements Serializable {
/**
* other resource manager appId , for example : YARN etc
*/
protected List<String> appIds;
private List<String> appIds;
public int getTaskInstanceId() {
return taskInstanceId;
......@@ -94,7 +94,7 @@ public class TaskKillResponseCommand implements Serializable {
}
/**
* package request command
* package request command
*
* @return command
*/
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemoteException.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.exceptions;
/**
* Custom runtime exception
*/
public class RemoteException extends RuntimeException {
/**
* Construct a new runtime exception with the detail message
*
* @param message detail message
*/
public RemoteException(String message) {
super(message);
}
/**
* Construct a new runtime exception with the detail message and cause
*
* @param message detail message
*/
public RemoteException(String message, Throwable cause) {
super(message, cause);
}
}
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java
浏览文件 @ dad94369
......@@ -18,14 +18,17 @@
package org.apache.dolphinscheduler.remote.future;
import org.apache.dolphinscheduler.remote.command.Command;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* response future
......@@ -34,25 +37,25 @@ public class ResponseFuture {
private static final Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class);
private static final ConcurrentHashMap<Long,ResponseFuture> FUTURE_TABLE = new ConcurrentHashMap<>(256);
private static final ConcurrentHashMap<Long, ResponseFuture> FUTURE_TABLE = new ConcurrentHashMap<>(256);
/**
* request unique identification
* request unique identification
*/
private final long opaque;
/**
* timeout
* timeout
*/
private final long timeoutMillis;
/**
* invokeCallback function
* invokeCallback function
*/
private final InvokeCallback invokeCallback;
/**
* releaseSemaphore
* releaseSemaphore
*/
private final ReleaseSemaphore releaseSemaphore;
......@@ -61,7 +64,7 @@ public class ResponseFuture {
private final long beginTimestamp = System.currentTimeMillis();
/**
* response command
* response command
*/
private Command responseCommand;
......@@ -78,10 +81,9 @@ public class ResponseFuture {
}
/**
* wait for response
* wait for response
*
* @return command
* @throws InterruptedException
*/
public Command waitResponse() throws InterruptedException {
this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS);
......@@ -89,7 +91,7 @@ public class ResponseFuture {
}
/**
* put response
* put response
*
* @param responseCommand responseCommand
*/
......@@ -99,12 +101,13 @@ public class ResponseFuture {
FUTURE_TABLE.remove(opaque);
}
public static ResponseFuture getFuture(long opaque){
public static ResponseFuture getFuture(long opaque) {
return FUTURE_TABLE.get(opaque);
}
/**
* whether timeout
* whether timeout
*
* @return timeout
*/
public boolean isTimeout() {
......@@ -113,7 +116,7 @@ public class ResponseFuture {
}
/**
* execute invoke callback
* execute invoke callback
*/
public void executeInvokeCallback() {
if (invokeCallback != null) {
......@@ -162,10 +165,10 @@ public class ResponseFuture {
}
/**
* release
* release
*/
public void release() {
if(this.releaseSemaphore != null){
if (this.releaseSemaphore != null) {
this.releaseSemaphore.release();
}
}
......@@ -173,7 +176,7 @@ public class ResponseFuture {
/**
* scan future table
*/
public static void scanFutureTable(){
public static void scanFutureTable() {
final List<ResponseFuture> futureList = new LinkedList<>();
Iterator<Map.Entry<Long, ResponseFuture>> it = FUTURE_TABLE.entrySet().iterator();
while (it.hasNext()) {
......@@ -189,7 +192,7 @@ public class ResponseFuture {
try {
future.release();
future.executeInvokeCallback();
} catch (Throwable ex) {
} catch (Exception ex) {
LOGGER.warn("scanFutureTable, execute callback error", ex);
}
}
......@@ -197,16 +200,16 @@ public class ResponseFuture {
@Override
public String toString() {
return "ResponseFuture{" +
"opaque=" + opaque +
", timeoutMillis=" + timeoutMillis +
", invokeCallback=" + invokeCallback +
", releaseSemaphore=" + releaseSemaphore +
", latch=" + latch +
", beginTimestamp=" + beginTimestamp +
", responseCommand=" + responseCommand +
", sendOk=" + sendOk +
", cause=" + cause +
'}';
return "ResponseFuture{"
+ "opaque=" + opaque
+ ", timeoutMillis=" + timeoutMillis
+ ", invokeCallback=" + invokeCallback
+ ", releaseSemaphore=" + releaseSemaphore
+ ", latch=" + latch
+ ", beginTimestamp=" + beginTimestamp
+ ", responseCommand=" + responseCommand
+ ", sendOk=" + sendOk
+ ", cause=" + cause
+ '}';
}
}
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java
浏览文件 @ dad94369
......@@ -74,7 +74,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) {
this.nettyRemotingClient = nettyRemotingClient;
this.callbackExecutor = callbackExecutor;
this.processors = new ConcurrentHashMap();
this.processors = new ConcurrentHashMap<>();
}
/**
......@@ -82,10 +82,9 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
* the current channel has reached the end of its life cycle
*
* @param ctx channel handler context
* @throws Exception
*/
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
public void channelInactive(ChannelHandlerContext ctx) {
nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
ctx.channel().close();
}
......@@ -95,10 +94,9 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
*
* @param ctx channel handler context
* @param msg message
* @throws Exception
*/
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
public void channelRead(ChannelHandlerContext ctx, Object msg) {
processReceived(ctx.channel(), (Command) msg);
}
......@@ -106,7 +104,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
* register processor
*
* @param commandType command type
* @param processor processor
* @param processor processor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) {
this.registerProcessor(commandType, processor, null);
......@@ -116,8 +114,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
* register processor
*
* @param commandType command type
* @param processor processor
* @param executor thread executor
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
ExecutorService executorRef = executor;
......@@ -138,12 +136,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
future.setResponseCommand(command);
future.release();
if (future.getInvokeCallback() != null) {
this.callbackExecutor.submit(new Runnable() {
@Override
public void run() {
future.executeInvokeCallback();
}
});
this.callbackExecutor.submit(future::executeInvokeCallback);
} else {
future.putResponse(command);
}
......@@ -158,7 +151,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
Runnable run = () -> {
try {
pair.getLeft().process(channel, command);
} catch (Throwable e) {
} catch (Exception e) {
logger.error(String.format("process command %s exception", command), e);
}
};
......@@ -175,13 +168,12 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
/**
* caught exception
*
* @param ctx channel handler context
* @param ctx channel handler context
* @param cause cause
* @throws Exception
*/
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
logger.error("exceptionCaught : {}", cause);
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
logger.error("exceptionCaught : {}", cause.getMessage(), cause);
nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
ctx.channel().close();
}
......@@ -193,11 +185,11 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
heartBeat.setType(CommandType.HEART_BEAT);
heartBeat.setBody(heartBeatData);
ctx.writeAndFlush(heartBeat)
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
} else {
super.userEventTriggered(ctx, evt);
}
}
}
\ No newline at end of file
}
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java
浏览文件 @ dad94369
......@@ -55,7 +55,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
/**
* server processors queue
*/
private final ConcurrentHashMap<CommandType, Pair<NettyRequestProcessor, ExecutorService>> processors = new ConcurrentHashMap();
private final ConcurrentHashMap<CommandType, Pair<NettyRequestProcessor, ExecutorService>> processors = new ConcurrentHashMap<>();
public NettyServerHandler(NettyRemotingServer nettyRemotingServer) {
this.nettyRemotingServer = nettyRemotingServer;
......@@ -66,10 +66,9 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
* the current channel has reached the end of its life cycle
*
* @param ctx channel handler context
* @throws Exception
*/
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
public void channelInactive(ChannelHandlerContext ctx) {
ctx.channel().close();
}
......@@ -78,10 +77,9 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
*
* @param ctx channel handler context
* @param msg message
* @throws Exception
*/
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
public void channelRead(ChannelHandlerContext ctx, Object msg) {
processReceived(ctx.channel(), (Command) msg);
}
......@@ -89,7 +87,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
* register processor
*
* @param commandType command type
* @param processor processor
* @param processor processor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) {
this.registerProcessor(commandType, processor, null);
......@@ -99,8 +97,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
* register processor
*
* @param commandType command type
* @param processor processor
* @param executor thread executor
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
ExecutorService executorRef = executor;
......@@ -114,21 +112,17 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
* process received logic
*
* @param channel channel
* @param msg message
* @param msg message
*/
private void processReceived(final Channel channel, final Command msg) {
final CommandType commandType = msg.getType();
final Pair<NettyRequestProcessor, ExecutorService> pair = processors.get(commandType);
if (pair != null) {
Runnable r = new Runnable() {
@Override
public void run() {
try {
pair.getLeft().process(channel, msg);
} catch (Throwable ex) {
logger.error("process msg {} error", msg, ex);
}
Runnable r = () -> {
try {
pair.getLeft().process(channel, msg);
} catch (Exception ex) {
logger.error("process msg {} error", msg, ex);
}
};
try {
......@@ -144,9 +138,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
/**
* caught exception
*
* @param ctx channel handler context
* @param ctx channel handler context
* @param cause cause
* @throws Exception
*/
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
......@@ -158,7 +151,6 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
* channel write changed
*
* @param ctx channel handler context
* @throws Exception
*/
@Override
public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
......@@ -168,14 +160,14 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
if (!ch.isWritable()) {
if (logger.isWarnEnabled()) {
logger.warn("{} is not writable, over high water level : {}",
ch, config.getWriteBufferHighWaterMark());
ch, config.getWriteBufferHighWaterMark());
}
config.setAutoRead(false);
} else {
if (logger.isWarnEnabled()) {
logger.warn("{} is writable, to low water : {}",
ch, config.getWriteBufferLowWaterMark());
ch, config.getWriteBufferLowWaterMark());
}
config.setAutoRead(true);
}
......@@ -189,4 +181,4 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
super.userEventTriggered(ctx, evt);
}
}
}
\ No newline at end of file
}
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
浏览文件 @ dad94369
......@@ -14,43 +14,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
import io.netty.channel.Channel;
package org.apache.dolphinscheduler.remote.utils;
import java.net.InetSocketAddress;
import io.netty.channel.Channel;
/**
* channel utils
* channel utils
*/
public class ChannelUtils {
private ChannelUtils() {
throw new IllegalStateException(ChannelUtils.class.getName());
}
/**
* get local address
* get local address
*
* @param channel channel
* @return local address
*/
public static String getLocalAddress(Channel channel){
return ((InetSocketAddress)channel.localAddress()).getAddress().getHostAddress();
public static String getLocalAddress(Channel channel) {
return ((InetSocketAddress) channel.localAddress()).getAddress().getHostAddress();
}
/**
* get remote address
* get remote address
*
* @param channel channel
* @return remote address
*/
public static String getRemoteAddress(Channel channel){
return ((InetSocketAddress)channel.remoteAddress()).getAddress().getHostAddress();
public static String getRemoteAddress(Channel channel) {
return ((InetSocketAddress) channel.remoteAddress()).getAddress().getHostAddress();
}
/**
* channel to address
* channel to address
*
* @param channel channel
* @return address
*/
public static Host toAddress(Channel channel){
InetSocketAddress socketAddress = ((InetSocketAddress)channel.remoteAddress());
public static Host toAddress(Channel channel) {
InetSocketAddress socketAddress = ((InetSocketAddress) channel.remoteAddress());
return new Host(socketAddress.getAddress().getHostAddress(), socketAddress.getPort());
}
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java
浏览文件 @ dad94369
......@@ -25,6 +25,10 @@ import java.nio.charset.StandardCharsets;
*/
public class Constants {
private Constants() {
throw new IllegalStateException(Constants.class.getName());
}
public static final String COMMA = ",";
public static final String SLASH = "/";
......
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java
浏览文件 @ dad94369
......@@ -16,6 +16,8 @@
*/
package org.apache.dolphinscheduler.remote.utils;
import org.apache.dolphinscheduler.remote.exceptions.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -28,22 +30,24 @@ import java.util.regex.Pattern;
public class IPUtils {
private static final Logger logger = LoggerFactory.getLogger(IPUtils.class);
private IPUtils() {
throw new IllegalStateException(IPUtils.class.getName());
}
private static String IP_REGEX = "([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}";
private static final Logger logger = LoggerFactory.getLogger(IPUtils.class);
private static String LOCAL_HOST = "unknown";
private static String localHost = "unknown";
static {
String host = System.getenv("HOSTNAME");
if (isNotEmpty(host)) {
LOCAL_HOST = host;
localHost = host;
} else {
try {
String hostName = InetAddress.getLocalHost().getHostName();
if (isNotEmpty(hostName)) {
LOCAL_HOST = hostName;
localHost = hostName;
}
} catch (UnknownHostException e) {
logger.error("get hostName error!", e);
......@@ -52,7 +56,7 @@ public class IPUtils {
}
public static String getLocalHost() {
return LOCAL_HOST;
return localHost;
}
......@@ -100,7 +104,7 @@ public class IPUtils {
return addresses;
} catch (SocketException e) {
throw new RuntimeException(e.getMessage(), e);
throw new RemoteException(e.getMessage(), e);
}
}
......@@ -131,12 +135,11 @@ public class IPUtils {
return false;
}
Pattern pat = Pattern.compile(IP_REGEX);
String ipRegex = "([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}";
Pattern pat = Pattern.compile(ipRegex);
Matcher mat = pat.matcher(addr);
boolean ipAddress = mat.find();
return ipAddress;
return mat.find();
}
}
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java
浏览文件 @ dad94369
......@@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.remote.utils;
/**
* key value pair
*
......@@ -51,7 +50,7 @@ public class Pair<L, R> {
this.right = right;
}
public static <L, R> Pair of(L left, R right){
return new Pair(left, right);
public static <L, R> Pair<L, R> of(L left, R right) {
return new Pair<>(left, right);
}
}
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
浏览文件 @ dad94369
......@@ -103,11 +103,14 @@ public class WorkerServer {
public void run() {
logger.info("start worker server...");
//alert-server client registry
alertClientService = new AlertClientService(workerConfig.getAlertListenHost(),Constants.ALERT_RPC_PORT);
//init remoting server
NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(workerConfig.getListenPort());
this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, new TaskExecuteProcessor());
this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, new TaskExecuteProcessor(alertClientService));
this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_REQUEST, new TaskKillProcessor());
this.nettyRemotingServer.registerProcessor(CommandType.DB_TASK_ACK, new DBTaskAckProcessor());
this.nettyRemotingServer.registerProcessor(CommandType.DB_TASK_RESPONSE, new DBTaskResponseProcessor());
......@@ -116,9 +119,6 @@ public class WorkerServer {
// worker registry
this.workerRegistry.registry();
//alert-server client registry
alertClientService = new AlertClientService(workerConfig.getAlertListenHost(),Constants.ALERT_RPC_PORT);
// retry report task status
this.retryReportTaskStatusThread.start();
......
dolphinscheduler-server/src/main/resources/logback-master.xml
浏览文件 @ dad94369
......@@ -32,9 +32,9 @@
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
</filter> -->
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
......
dolphinscheduler-server/src/main/resources/logback-worker.xml
浏览文件 @ dad94369
......@@ -33,9 +33,9 @@
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
</filter> -->
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
......@@ -56,10 +56,9 @@
</appender>
<appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-worker.log</file>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter"/>
</filter> -->
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
......
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
浏览文件 @ dad94369
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.bean;
import org.springframework.beans.BeansException;
......@@ -31,9 +32,7 @@ public class SpringApplicationContext implements ApplicationContextAware {
SpringApplicationContext.applicationContext = applicationContext;
}
public static <T> T getBean(Class<T> requiredType){
public static <T> T getBean(Class<T> requiredType) {
return applicationContext.getBean(requiredType);
}
}
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/exceptions/ServiceException.java 0 → 100644
浏览文件 @ dad94369
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.exceptions;
/**
* Custom ZKServerException exception
*/
public class ServiceException extends RuntimeException {
/**
* Construct a new runtime exception with the error message
*
* @param errMsg Error message
*/
public ServiceException(String errMsg) {
super(errMsg);
}
/**
* Construct a new runtime exception with the cause
*
* @param cause cause
*/
public ServiceException(Throwable cause) {
super(cause);
}
/**
* Construct a new runtime exception with the detail message and cause
*
* @param errMsg message
* @param cause cause
*/
public ServiceException(String errMsg, Throwable cause) {
super(errMsg, cause);
}
}
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
浏览文件 @ dad94369
......@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.permission;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
......@@ -21,11 +22,13 @@ import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.List;
import org.slf4j.Logger;
public class PermissionCheck<T> {
/**
* logger
......@@ -58,8 +61,9 @@ public class PermissionCheck<T> {
/**
* permission check
*
* @param authorizationType authorization type
* @param processService process dao
* @param processService process dao
*/
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService) {
this.authorizationType = authorizationType;
......@@ -68,10 +72,6 @@ public class PermissionCheck<T> {
/**
* permission check
* @param authorizationType
* @param processService
* @param needChecks
* @param userId
*/
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) {
this.authorizationType = authorizationType;
......@@ -82,11 +82,6 @@ public class PermissionCheck<T> {
/**
* permission check
* @param authorizationType
* @param processService
* @param needChecks
* @param userId
* @param logger
*/
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) {
this.authorizationType = authorizationType;
......@@ -98,13 +93,8 @@ public class PermissionCheck<T> {
/**
* permission check
* @param logger
* @param authorizationType
* @param processService
* @param resourceList
* @param userId
*/
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List<ResourceInfo> resourceList, int userId,Logger logger) {
public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List<ResourceInfo> resourceList, int userId, Logger logger) {
this.authorizationType = authorizationType;
this.processService = processService;
this.resourceList = resourceList;
......@@ -154,9 +144,10 @@ public class PermissionCheck<T> {
/**
* has permission
*
* @return true if has permission
*/
public boolean hasPermission(){
public boolean hasPermission() {
try {
checkPermission();
return true;
......@@ -167,23 +158,24 @@ public class PermissionCheck<T> {
/**
* check permission
* @throws Exception exception
*
* @throws ServiceException exception
*/
public void checkPermission() throws Exception{
if(this.needChecks.length > 0){
public void checkPermission() throws ServiceException {
if (this.needChecks.length > 0) {
// get user type in order to judge whether the user is admin
User user = processService.getUserById(userId);
if (user == null) {
logger.error("user id {} didn't exist",userId);
throw new RuntimeException(String.format("user %s didn't exist",userId));
logger.error("user id {} doesn't exist", userId);
throw new ServiceException(String.format("user %s doesn't exist", userId));
}
if (user.getUserType() != UserType.ADMIN_USER){
List<T> unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType);
if (user.getUserType() != UserType.ADMIN_USER) {
List<T> unauthorizedList = processService.listUnauthorized(userId, needChecks, authorizationType);
// if exist unauthorized resource
if(CollectionUtils.isNotEmpty(unauthorizedList)){
logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList);
throw new RuntimeException(String.format("user %s didn't has permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0)));
if (CollectionUtils.isNotEmpty(unauthorizedList)) {
logger.error("user {} doesn't have permission of {}: {}", user.getUserName(), authorizationType.getDescp(), unauthorizedList);
throw new ServiceException(String.format("user %s doesn't have permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0)));
}
}
}
......
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
浏览文件 @ dad94369
......@@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.quartz;
package org.apache.dolphinscheduler.service.quartz;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
......@@ -25,6 +25,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.Date;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
......@@ -34,8 +37,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.util.Date;
/**
* process schedule job
*/
......@@ -46,7 +47,7 @@ public class ProcessScheduleJob implements Job {
*/
private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class);
public ProcessService getProcessService(){
public ProcessService getProcessService() {
return SpringApplicationContext.getBean(ProcessService.class);
}
......@@ -66,10 +67,8 @@ public class ProcessScheduleJob implements Job {
int projectId = dataMap.getInt(Constants.PROJECT_ID);
int scheduleId = dataMap.getInt(Constants.SCHEDULE_ID);
Date scheduledFireTime = context.getScheduledFireTime();
Date fireTime = context.getFireTime();
logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId);
......@@ -82,11 +81,10 @@ public class ProcessScheduleJob implements Job {
return;
}
ProcessDefinition processDefinition = getProcessService().findProcessDefineById(schedule.getProcessDefinitionId());
// release state : online/offline
ReleaseState releaseState = processDefinition.getReleaseState();
if (processDefinition == null || releaseState == ReleaseState.OFFLINE) {
if (releaseState == ReleaseState.OFFLINE) {
logger.warn("process definition does not exist in db or offline，need not to create command, projectId:{}, processId:{}", projectId, scheduleId);
return;
}
......@@ -107,7 +105,6 @@ public class ProcessScheduleJob implements Job {
getProcessService().createCommand(command);
}
/**
* delete job
*/
......
dolphinscheduler-ui/package.json
浏览文件 @ dad94369
此差异已折叠。
pom.xml
浏览文件 @ dad94369
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或