diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 0f1df00372fedb2ec09616a83ee7044afdaa8e96..37718319b906afc71267c1d329e2647e6a18c270 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -246,7 +246,8 @@ worker.heartbeat.interval|10|worker-service heartbeat interval, the unit is seco worker.max.cpuload.avg|-1|worker max CPU load avg, only higher than the system CPU load average, worker server can be dispatched tasks. default value -1: the number of CPU cores * 2 worker.reserved.memory|0.3|worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G worker.groups|default|worker groups separated by comma, e.g., 'worker.groups=default,test'
worker will join corresponding group according to this config when startup - +worker.tenant.auto.create|true|tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. +worker.tenant.distributed.user|false|Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. ### alert.properties [alert-service log config] diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 14f79f1344c8547dfa66debb90522087c084ec42..94cfa57c9e664ca1a51fe7c31a5bab5c192e7c1a 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -235,6 +235,8 @@ worker.heartbeat.interval|10|worker心跳间隔,单位为秒 worker.max.cpuload.avg|-1|worker最大cpuload均值,只有高于系统cpuload均值时,worker服务才能被派发任务. 默认值为-1: cpu cores * 2 worker.reserved.memory|0.3|worker预留内存,只有低于系统可用内存时,worker服务才能被派发任务,单位为G worker.groups|default|worker分组配置,逗号分隔,例如'worker.groups=default,test'
worker启动时会根据该配置自动加入对应的分组 +worker.tenant.auto.create|true|租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 +worker.tenant.distributed.user|false|使用场景为分布式用户例如使用FreeIpa创建的用户存于LDAP中.该参数只适用于Linux,当该参数为true时worker.tenant.auto.create将不生效,不会自动去创建租户 ## 8.alert.properties [Alert 告警服务配置] diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index a4e79236ca03b2fd5f594de4d980d1c444c6ac5b..79135aaac481597cbbdcaf48be89fdf35ceb79b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -249,6 +249,25 @@ public class OSUtils { return users; } + /** + * whether the user exists in linux + * + * @return boolean + */ + public static boolean existTenantCodeInLinux(String tenantCode) { + try{ + String result = exeCmd("id "+ tenantCode); + if (!StringUtils.isEmpty(result)){ + return result.contains("uid="); + } + }catch (Exception e){ + //because ShellExecutor method throws exception to the linux return status is not 0 + //not exist user return status is 1 + logger.error(e.getMessage(), e); + } + return false; + } + /** * create user * diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 64ef52ed506a905559c1e44d12ebb1275f11a14f..5300a64d82798fe503f71e5f27b030b861b2580f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.os; +import org.apache.commons.lang.SystemUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.junit.Assert; @@ -24,6 +25,8 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * OSUtilsTest @@ -52,4 +55,41 @@ public class OSUtilsTest { logger.info("cpuUsage : {}", cpuUsage); Assert.assertTrue(cpuUsage >= 0.0); } + + @Test + public void availablePhysicalMemorySize(){ + double physicalMemorySize = OSUtils.availablePhysicalMemorySize(); + logger.info("physicalMemorySize : {}", physicalMemorySize); + Assert.assertTrue(physicalMemorySize >= 0.0); + + } + + @Test + public void existTenantCodeInLinux(){ + if (SystemUtils.IS_OS_LINUX){ + boolean test = OSUtils.existTenantCodeInLinux("root"); + Assert.assertTrue(test); + boolean test1 = OSUtils.existTenantCodeInLinux("xxxtt"); + Assert.assertFalse(test1); + }else{ + Assert.assertFalse("system must be linux",false); + } + + } + + @Test + public void existOSTenandCode(){ + if (SystemUtils.IS_OS_LINUX){ + List userList = OSUtils.getUserList(); + Assert.assertTrue(userList.contains("root")); + Assert.assertFalse(userList.contains("xxxtt")); + }else{ + Assert.assertFalse("system must be linux",false); + + } + + } + + + } diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 980a08826742289a1e704c253788773bca12bbfd..6b58c5045033f6343134c46787572b42644ff2ff 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -123,8 +123,10 @@ worker: heartbeat-interval: 10 # worker host weight to dispatch tasks, default value 100 host-weight: 100 - # worker tenant auto create + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. tenant-auto-create: true + #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. + tenant-distributed-user: false # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 max-cpu-load-avg: -1 # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 5192e27d61c9d8233c7a4e522163422a3163253d..3eb112bb07089c72c75718149a5130c092d9c0b7 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -32,6 +32,7 @@ public class WorkerConfig { private int heartbeatInterval; private int hostWeight; private boolean tenantAutoCreate; + private boolean tenantDistributedUser; private int maxCpuLoadAvg; private double reservedMemory; private Set groups; @@ -117,4 +118,12 @@ public class WorkerConfig { public void setAlertListenPort(final int alertListenPort) { this.alertListenPort = alertListenPort; } + + public boolean isTenantDistributedUser() { + return tenantDistributedUser; + } + + public void setTenantDistributedUser(boolean tenantDistributedUser) { + this.tenantDistributedUser = tenantDistributedUser; + } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index a376587aad915a3525efa2ae299b4e4354fff902..cd3d940f001c9137cfa14643a8ac2f397e6864c6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.processor; +import org.apache.commons.lang.SystemUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -116,12 +117,22 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) { - if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) { + boolean osUserExistFlag ; + //if Using distributed is true and Currently supported systems are linux,Should not let it automatically + //create tenants,so TenantAutoCreate has no effect + if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX){ + //use the id command to judge in linux + osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode()); + }else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()){ + // if not exists this user, then create OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode()); + osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); + }else { + osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); } // check if the OS user exists - if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) { + if (!osUserExistFlag) { logger.error("tenantCode: {} does not exist, taskInstanceId: {}", taskExecutionContext.getTenantCode(), taskExecutionContext.getTaskInstanceId()); TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index dd353f049508a2713dc3c895ea8af25affcdee48..80e650c7ed8fff296e29a8cc30fb4b4f10755019 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -62,8 +62,10 @@ worker: heartbeat-interval: 10 # worker host weight to dispatch tasks, default value 100 host-weight: 100 - # worker tenant auto create + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. tenant-auto-create: true + #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. + tenant-distributed-user: false # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 max-cpu-load-avg: -1 # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G