未验证 提交 427c5854 编写于 作者: a9236229028's avatar a9236229028 提交者: GitHub

[fix][plugin]: fix spark sql use custom parma. (#10821)

* [fix][plugin]: fix spark sql use custom parma.

* [fix][plugin] roll back part of the code
Co-authored-by: Nhuaangcg <huangcg@getui.com>
上级 2397423e
......@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
......@@ -90,9 +91,8 @@ public abstract class AbstractParameters implements IParameters {
public Map<String, Property> getInputLocalParametersMap() {
Map<String, Property> localParametersMaps = new LinkedHashMap<>();
if (localParams != null) {
for (Property property : localParams) {
if (property.getDirect().equals(Direct.IN)) {
if (Objects.equals(Direct.IN, property.getDirect())) {
localParametersMaps.put(property.getProp(), property);
}
}
......
......@@ -229,7 +229,7 @@ public class SparkTask extends AbstractYarnTask {
Path path = file.toPath();
if (!Files.exists(path)) {
String script = sparkParameters.getRawScript().replaceAll("\\r\\n", "\n");
String script = replaceParam(sparkParameters.getRawScript());
sparkParameters.setRawScript(script);
logger.info("raw script : {}", sparkParameters.getRawScript());
......@@ -255,6 +255,14 @@ public class SparkTask extends AbstractYarnTask {
return scriptFileName;
}
private String replaceParam(String script) {
script = script.replaceAll("\\r\\n", "\n");
// replace placeholder, and combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
return script;
}
@Override
protected void setMainJarName() {
// main jar
......
......@@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
}
},
useResources(),
...useCustomParams({ model, field: 'localParams', isSimple: true })
...useCustomParams({ model, field: 'localParams', isSimple: false })
]
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册