From 427c58546c93869e0bee7fb853319083c35c8bc4 Mon Sep 17 00:00:00 2001 From: chuxing <923622908@qq.com> Date: Wed, 13 Jul 2022 11:44:13 +0800 Subject: [PATCH] [fix][plugin]: fix spark sql use custom parma. (#10821) * [fix][plugin]: fix spark sql use custom parma. * [fix][plugin] roll back part of the code Co-authored-by: huaangcg --- .../plugin/task/api/parameters/AbstractParameters.java | 4 ++-- .../dolphinscheduler/plugin/task/spark/SparkTask.java | 10 +++++++++- .../projects/task/components/node/fields/use-spark.ts | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java index b575ece49..f01198d3b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -90,9 +91,8 @@ public abstract class AbstractParameters implements IParameters { public Map getInputLocalParametersMap() { Map localParametersMaps = new LinkedHashMap<>(); if (localParams != null) { - for (Property property : localParams) { - if (property.getDirect().equals(Direct.IN)) { + if (Objects.equals(Direct.IN, property.getDirect())) { localParametersMaps.put(property.getProp(), property); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index b30ac64ae..01db34266 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -229,7 +229,7 @@ public class SparkTask extends AbstractYarnTask { Path path = file.toPath(); if (!Files.exists(path)) { - String script = sparkParameters.getRawScript().replaceAll("\\r\\n", "\n"); + String script = replaceParam(sparkParameters.getRawScript()); sparkParameters.setRawScript(script); logger.info("raw script : {}", sparkParameters.getRawScript()); @@ -255,6 +255,14 @@ public class SparkTask extends AbstractYarnTask { return scriptFileName; } + private String replaceParam(String script) { + script = script.replaceAll("\\r\\n", "\n"); + // replace placeholder, and combining local and global parameters + Map paramsMap = taskExecutionContext.getPrepareParamsMap(); + script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap)); + return script; + } + @Override protected void setMainJarName() { // main jar diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts index fdce98a16..89ac7800d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts @@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { } }, useResources(), - ...useCustomParams({ model, field: 'localParams', isSimple: true }) + ...useCustomParams({ model, field: 'localParams', isSimple: false }) ] } -- GitLab