From 6a48b985138cbb0f03a3cba29393e9c742dd5880 Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Wed, 9 Oct 2019 18:47:42 +0800 Subject: [PATCH] add multiply settings for application.yml (#973) * update english documents * refactor zk client * update documents * update zkclient * update zkclient * update documents * add architecture-design * change i18n * update i18n * update english documents * add architecture-design * update english documents * update en-US documents * add architecture-design * update demo site * add mybatis plus model * modify mybatisplus * modify mybatisplus * change interface by mybatisplus * add unit test * refactor dao interface. * add unit test for dao... * add unit test for dao... * add unit test for dao... * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml # escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java # escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update some dao bugs * update for some bugs * update some bugs * Merge remote-tracking branch 'upstream/dev-db' into dev-db # Conflicts: # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml * update * update * add multiply settings for application.yml * add multiply settings for application.yml * revert --- .../api/service/TenantService.java | 12 ++-- .../common/utils/DateUtils.java | 3 + .../src/main/resources/quartz.properties | 6 +- .../dolphinscheduler/dao/MonitorDBDao.java | 26 ++++---- .../dolphinscheduler/dao/ProcessDao.java | 20 +++--- .../dao/mapper/TenantMapper.java | 2 +- .../src/main/resources/application.yml | 65 ++++++++++++++++--- .../dao/mapper/CommandMapper.xml | 2 +- .../dao/mapper/AccessTokenMapperTest.java | 2 +- .../dao/mapper/ProcessInstanceMapperTest.java | 3 +- .../master/runner/MasterExecThread.java | 1 + .../src/main/resources/master.properties | 4 +- dolphinscheduler-ui/.env | 2 +- .../build/webpack.config.dev.js | 2 +- dolphinscheduler-ui/src/js/module/io/index.js | 2 +- script/dolphinscheduler-daemon.sh | 6 +- 16 files changed, 106 insertions(+), 52 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index fa70cc790..40eb099ab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -74,7 +74,7 @@ public class TenantService extends BaseService{ return result; } - if (!checkTenant(tenantCode)){ + if (checkTenantExists(tenantCode)){ putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); return result; } @@ -168,7 +168,7 @@ public class TenantService extends BaseService{ * if the tenant code is modified, the original resource needs to be copied to the new tenant. */ if (!tenant.getTenantCode().equals(tenantCode)){ - if (checkTenant(tenantCode)){ + if (checkTenantExists(tenantCode)){ // if hdfs startup if (PropertyUtils.getResUploadStartupState()){ String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; @@ -278,8 +278,7 @@ public class TenantService extends BaseService{ */ public Result verifyTenantCode(String tenantCode) { Result result=new Result(); - Tenant tenant = tenantMapper.queryByTenantCode(tenantCode); - if (tenant != null) { + if (checkTenantExists(tenantCode)) { logger.error("tenant {} has exist, can't create again.", tenantCode); putMsg(result, Status.TENANT_NAME_EXIST); }else{ @@ -295,7 +294,8 @@ public class TenantService extends BaseService{ * @param tenantCode * @return */ - private boolean checkTenant(String tenantCode) { - return tenantMapper.queryByTenantCode(tenantCode) == null ? true : false; + private boolean checkTenantExists(String tenantCode) { + List tenants = tenantMapper.queryByTenantCode(tenantCode); + return (tenants != null && tenants.size() > 0); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index c927a226e..b311e7a9a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -132,6 +132,9 @@ public class DateUtils { * @return */ public static long differSec(Date d1, Date d2) { + if(d1 == null || d2 == null){ + return 0; + } return (long) Math.ceil(differMs(d1, d2) / 1000.0); } diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-common/src/main/resources/quartz.properties index 22529b8e8..d26e77dbf 100644 --- a/dolphinscheduler-common/src/main/resources/quartz.properties +++ b/dolphinscheduler-common/src/main/resources/quartz.properties @@ -32,8 +32,8 @@ org.quartz.jobStore.dataSource = myDs #============================================================================ org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver -org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.user = xx -org.quartz.dataSource.myDs.password = xx +org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.220.188:3306/dolphinscheduler_bl?characterEncoding=utf8 +org.quartz.dataSource.myDs.user = root +org.quartz.dataSource.myDs.password = root@123 org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index e542dfeae..6e2023e52 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.config.YmlConfig; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; @@ -41,25 +42,26 @@ public class MonitorDBDao { /** * 加载配置文件 */ - private static Configuration conf; +// private static Configuration conf; + +// static { +// try { +// conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES); +// }catch (ConfigurationException e){ +// logger.error("load configuration excetpion",e); +// System.exit(1); +// } +// } - static { - try { - conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES); - }catch (ConfigurationException e){ - logger.error("load configuration excetpion",e); - System.exit(1); - } - } /** * create connection * @return */ private static Connection getConn() { - String url = conf.getString(Constants.SPRING_DATASOURCE_URL); - String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME); - String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD); + String url = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_URL); + String username = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_USERNAME); + String password = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_PASSWORD); Connection conn = null; try { //classloader,load driver diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java index f720c5660..04fe11f23 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java @@ -146,17 +146,17 @@ public class ProcessDao extends AbstractBaseDao { delCommandByid(command.getId()); saveErrorCommand(command, "process instance is null"); return null; - }else if(!checkThreadNum(command, validThreadNum)){ - logger.info("there is not enough thread for this command: {}",command.toString() ); - return setWaitingThreadProcess(command, processInstance); - }else{ - processInstance.setCommandType(command.getCommandType()); - processInstance.addHistoryCmd(command.getCommandType()); - saveProcessInstance(processInstance); - this.setSubProcessParam(processInstance); - delCommandByid(command.getId()); - return processInstance; } + if(!checkThreadNum(command, validThreadNum)){ + logger.info("there is not enough thread for this command: {}",command.toString() ); + return setWaitingThreadProcess(command, processInstance); + } + processInstance.setCommandType(command.getCommandType()); + processInstance.addHistoryCmd(command.getCommandType()); + saveProcessInstance(processInstance); + this.setSubProcessParam(processInstance); + delCommandByid(command.getId()); + return processInstance; }catch (Exception e){ logger.error("scan command error ", e); saveErrorCommand(command, e.toString()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java index 5c8806988..2cf90520a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java @@ -27,7 +27,7 @@ public interface TenantMapper extends BaseMapper { Tenant queryById(@Param("tenantId") int tenantId); - Tenant queryByTenantCode(@Param("tenantCode") String tenantCode); + List queryByTenantCode(@Param("tenantCode") String tenantCode); IPage queryTenantPaging(IPage page, @Param("searchVal") String searchVal); diff --git a/dolphinscheduler-dao/src/main/resources/application.yml b/dolphinscheduler-dao/src/main/resources/application.yml index 52af23c0d..6c3c258e3 100644 --- a/dolphinscheduler-dao/src/main/resources/application.yml +++ b/dolphinscheduler-dao/src/main/resources/application.yml @@ -1,14 +1,38 @@ # mysql spring: - datasource: - driver-class-name: org.postgresql.Driver -# driver-class-name: com.mysql.jdbc.Driver - url: jdbc:postgresql://192.168.220.154:5432/dolphinscheduler -# url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - username: root - password: root@123 -# platform: - continue-on-error: true + + datasource: + url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + username: root + password: root@123 + # + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.jdbc.Driver + initialSize: 5 + minIdle: 5 + maxActive: 20 + maxWait: 60000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + maxPoolPreparedStatementPerConnectionSize: 20 + spring.datasource.filters: stat,wall,log4j + connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 + +# jpa: +# hibernate: +# ddl-auto : update +# datasource: +# platform: +# continue-on-error: true +# druid: +# driver-class-name: org.postgresql.Driver +# url: jdbc:postgresql://192.168.220.154:5432/escheduler + #mybatis @@ -42,3 +66,26 @@ logging: level: org.apache.dolphinscheduler.dao: debug +--- +# master settings +spring: + profiles: master + +server: + port: 5566 + +--- +# worker settings +spring: + profiles: worker + +server: + port: 7788 + +--- +# alert settings +spring: + profiles: alert + +server: + port: 7789 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index 9abb6a959..e3ab6b23c 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -5,7 +5,7 @@ select command.* from t_ds_command command join t_ds_process_definition definition on command.process_definition_id = definition.id where definition.release_state = 1 AND definition.flag = 1 - order by update_time asc + order by command.update_time asc limit 1