From 2d8c4ec86bdc8d00baee4066f40699078735c5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E4=BE=A0?= Date: Mon, 15 Mar 2021 10:22:13 +0800 Subject: [PATCH] [Improvement][Worker-Task] Python Command (#5036) * create the interpreter for python command * Update README.md Co-authored-by: zhouhong --- docker/build/README.md | 2 +- docker/build/README_zh_CN.md | 2 +- .../env/dolphinscheduler_env.sh.tpl | 2 +- docker/build/startup-init-conf.sh | 2 +- docker/docker-swarm/docker-compose.yml | 2 +- docker/docker-swarm/docker-stack.yml | 2 +- docker/kubernetes/dolphinscheduler/values.yaml | 4 ++-- .../server/worker/task/PythonCommandExecutor.java | 14 +++++--------- .../server/worker/EnvFileTest.java | 3 ++- script/env/dolphinscheduler_env.sh | 2 +- 10 files changed, 16 insertions(+), 19 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index 69273f9da..4d3ac8767 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -193,7 +193,7 @@ This environment variable sets `SPARK_HOME2`. The default value is `/opt/soft/sp **`PYTHON_HOME`** -This environment variable sets `PYTHON_HOME`. The default value is `/usr/bin/python`. +This environment variable sets `PYTHON_HOME`. The default value is `/usr`. **`JAVA_HOME`** diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 96bfb80dd..1dfc69c5a 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -193,7 +193,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会 **`PYTHON_HOME`** -配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr/bin/python`。 +配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr`。 **`JAVA_HOME`** diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl index b09e4972c..4463a3f22 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl @@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME export FLINK_HOME=$FLINK_HOME export DATAX_HOME=$DATAX_HOME -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 95491268a..b16100c40 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -42,7 +42,7 @@ export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"} export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"} export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"} -export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"} +export PYTHON_HOME=${PYTHON_HOME:-"/usr"} export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index e8601708e..5b2cf46b1 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -177,7 +177,7 @@ services: HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" SPARK_HOME1: "/opt/soft/spark1" SPARK_HOME2: "/opt/soft/spark2" - PYTHON_HOME: "/usr/bin/python" + #PYTHON_HOME: "/opt/soft/python" JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 094890dfa..aab44a388 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -171,7 +171,7 @@ services: HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" SPARK_HOME1: "/opt/soft/spark1" SPARK_HOME2: "/opt/soft/spark2" - PYTHON_HOME: "/usr/bin/python" + #PYTHON_HOME: "/opt/soft/python" JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 896ae772d..ba6b8eb8f 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -76,12 +76,12 @@ common: - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" - "export SPARK_HOME1=/opt/soft/spark1" - "export SPARK_HOME2=/opt/soft/spark2" - - "export PYTHON_HOME=/usr/bin/python" + #- "export PYTHON_HOME=/opt/soft/python" - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" - "export HIVE_HOME=/opt/soft/hive" - "export FLINK_HOME=/opt/soft/flink" - "export DATAX_HOME=/opt/soft/datax/bin/datax.py" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" RESOURCE_STORAGE_TYPE: "HDFS" RESOURCE_UPLOAD_PATH: "/dolphinscheduler" diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java index 344d00fa8..8340b788b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java @@ -109,8 +109,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { } /** - * get python home - * @return python home + * Gets the command path to which Python can execute + * @return python command path */ @Override protected String commandInterpreter() { @@ -118,21 +118,17 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { if (StringUtils.isEmpty(pythonHome)){ return PYTHON; } - return pythonHome; + return pythonHome + "/bin/python"; } /** - * get the absolute path of the Python command + * get the absolute path of the Python are installed * note : - * common.properties - * PYTHON_HOME configured under common.properties is Python absolute path, not PYTHON_HOME itself * * for example : - * your PYTHON_HOM is /opt/python3.7/ - * you must set PYTHON_HOME is /opt/python3.7/python under nder common.properties - * dolphinscheduler.env.path file. + * your PYTHON_HOM is /opt/python3.7 * * @param envPath env path * @return python home diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java index dc2a6008f..34f5d2f66 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.StringUtils; import java.io.BufferedReader; @@ -51,7 +52,7 @@ public class EnvFileTest { try { br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); while ((line = br.readLine()) != null){ - if (line.contains("PYTHON_HOME")){ + if (line.contains(Constants.PYTHON_HOME)) { sb.append(line); break; } diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh index 026b62020..9fb9f55d7 100755 --- a/script/env/dolphinscheduler_env.sh +++ b/script/env/dolphinscheduler_env.sh @@ -25,4 +25,4 @@ export HIVE_HOME=/opt/soft/hive export FLINK_HOME=/opt/soft/flink export DATAX_HOME=/opt/soft/datax/bin/datax.py -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH -- GitLab