From a0771541e530e1d269745a43a7af3db165007cfc Mon Sep 17 00:00:00 2001 From: lugela <374029008@qq.com> Date: Thu, 26 May 2022 14:58:07 +0800 Subject: [PATCH] [Fix-10181] Fix the logic of judging that the tenant does not exist (#10185) * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac * [Fix-10181] Fix the logic of judging that the tenant does not exist Add parameter description to configuration.md Add test method The configuration item adds 'tenant-distributed-user' in worker application.yaml to make it suitable for distributed users. If it is false, the original logic remains unchanged. At present, considering that it is a distributed user, it should not be allowed to create users in linux Use the linux command as id to get the user information that exists in /etc/passwd file and the cached sssd user. for example: id test 1. exist in /etc/passwd file or ldap : uid=1030(test) gid=1030(test) groups=1030(test) 2. no exist in /etc/passwd file and ldap: id: test: no such user Temporarily unable to test the system for windows and mac Co-authored-by: ouyangl --- docs/docs/en/architecture/configuration.md | 3 +- docs/docs/zh/architecture/configuration.md | 2 + .../common/utils/OSUtils.java | 19 +++++++++ .../common/os/OSUtilsTest.java | 40 +++++++++++++++++++ .../src/main/resources/application.yaml | 4 +- .../server/worker/config/WorkerConfig.java | 9 +++++ .../processor/TaskExecuteProcessor.java | 15 ++++++- .../src/main/resources/application.yaml | 4 +- 8 files changed, 91 insertions(+), 5 deletions(-) diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 0f1df0037..37718319b 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -246,7 +246,8 @@ worker.heartbeat.interval|10|worker-service heartbeat interval, the unit is seco worker.max.cpuload.avg|-1|worker max CPU load avg, only higher than the system CPU load average, worker server can be dispatched tasks. default value -1: the number of CPU cores * 2 worker.reserved.memory|0.3|worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G worker.groups|default|worker groups separated by comma, e.g., 'worker.groups=default,test'
worker will join corresponding group according to this config when startup - +worker.tenant.auto.create|true|tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. +worker.tenant.distributed.user|false|Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. ### alert.properties [alert-service log config] diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 14f79f134..94cfa57c9 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -235,6 +235,8 @@ worker.heartbeat.interval|10|worker心跳间隔,单位为秒 worker.max.cpuload.avg|-1|worker最大cpuload均值,只有高于系统cpuload均值时,worker服务才能被派发任务. 默认值为-1: cpu cores * 2 worker.reserved.memory|0.3|worker预留内存,只有低于系统可用内存时,worker服务才能被派发任务,单位为G worker.groups|default|worker分组配置,逗号分隔,例如'worker.groups=default,test'
worker启动时会根据该配置自动加入对应的分组 +worker.tenant.auto.create|true|租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 +worker.tenant.distributed.user|false|使用场景为分布式用户例如使用FreeIpa创建的用户存于LDAP中.该参数只适用于Linux,当该参数为true时worker.tenant.auto.create将不生效,不会自动去创建租户 ## 8.alert.properties [Alert 告警服务配置] diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index a4e79236c..79135aaac 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -249,6 +249,25 @@ public class OSUtils { return users; } + /** + * whether the user exists in linux + * + * @return boolean + */ + public static boolean existTenantCodeInLinux(String tenantCode) { + try{ + String result = exeCmd("id "+ tenantCode); + if (!StringUtils.isEmpty(result)){ + return result.contains("uid="); + } + }catch (Exception e){ + //because ShellExecutor method throws exception to the linux return status is not 0 + //not exist user return status is 1 + logger.error(e.getMessage(), e); + } + return false; + } + /** * create user * diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 64ef52ed5..5300a64d8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.os; +import org.apache.commons.lang.SystemUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.junit.Assert; @@ -24,6 +25,8 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * OSUtilsTest @@ -52,4 +55,41 @@ public class OSUtilsTest { logger.info("cpuUsage : {}", cpuUsage); Assert.assertTrue(cpuUsage >= 0.0); } + + @Test + public void availablePhysicalMemorySize(){ + double physicalMemorySize = OSUtils.availablePhysicalMemorySize(); + logger.info("physicalMemorySize : {}", physicalMemorySize); + Assert.assertTrue(physicalMemorySize >= 0.0); + + } + + @Test + public void existTenantCodeInLinux(){ + if (SystemUtils.IS_OS_LINUX){ + boolean test = OSUtils.existTenantCodeInLinux("root"); + Assert.assertTrue(test); + boolean test1 = OSUtils.existTenantCodeInLinux("xxxtt"); + Assert.assertFalse(test1); + }else{ + Assert.assertFalse("system must be linux",false); + } + + } + + @Test + public void existOSTenandCode(){ + if (SystemUtils.IS_OS_LINUX){ + List userList = OSUtils.getUserList(); + Assert.assertTrue(userList.contains("root")); + Assert.assertFalse(userList.contains("xxxtt")); + }else{ + Assert.assertFalse("system must be linux",false); + + } + + } + + + } diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 980a08826..6b58c5045 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -123,8 +123,10 @@ worker: heartbeat-interval: 10 # worker host weight to dispatch tasks, default value 100 host-weight: 100 - # worker tenant auto create + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. tenant-auto-create: true + #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. + tenant-distributed-user: false # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 max-cpu-load-avg: -1 # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 5192e27d6..3eb112bb0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -32,6 +32,7 @@ public class WorkerConfig { private int heartbeatInterval; private int hostWeight; private boolean tenantAutoCreate; + private boolean tenantDistributedUser; private int maxCpuLoadAvg; private double reservedMemory; private Set groups; @@ -117,4 +118,12 @@ public class WorkerConfig { public void setAlertListenPort(final int alertListenPort) { this.alertListenPort = alertListenPort; } + + public boolean isTenantDistributedUser() { + return tenantDistributedUser; + } + + public void setTenantDistributedUser(boolean tenantDistributedUser) { + this.tenantDistributedUser = tenantDistributedUser; + } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index a376587aa..cd3d940f0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.processor; +import org.apache.commons.lang.SystemUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -116,12 +117,22 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) { - if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) { + boolean osUserExistFlag ; + //if Using distributed is true and Currently supported systems are linux,Should not let it automatically + //create tenants,so TenantAutoCreate has no effect + if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX){ + //use the id command to judge in linux + osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode()); + }else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()){ + // if not exists this user, then create OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode()); + osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); + }else { + osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); } // check if the OS user exists - if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) { + if (!osUserExistFlag) { logger.error("tenantCode: {} does not exist, taskInstanceId: {}", taskExecutionContext.getTenantCode(), taskExecutionContext.getTaskInstanceId()); TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index dd353f049..80e650c7e 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -62,8 +62,10 @@ worker: heartbeat-interval: 10 # worker host weight to dispatch tasks, default value 100 host-weight: 100 - # worker tenant auto create + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. tenant-auto-create: true + #Scenes to be used for distributed users.For example,users created by FreeIpa are stored in LDAP.This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. + tenant-distributed-user: false # worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 max-cpu-load-avg: -1 # worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G -- GitLab