#1544 fix bug: workflow import (#1676)

* modify FileUtils.readFile2Str

* #1300 Add right alignment function in sql email content

* cancel formatted for alert_mail_template.ftl

* #747 sql task password Log desensitization

* cancel mail_temple

* edit ExcelUtils

* modify test method name

* #747 sql task password Log desensitization

* #1544 workflow import

* Constants add DATASOURCE_PASSWORD_REGEX

* #747 sql task password Log desensitization

* deal with import project have sub process

* modify export process addTaskNodeParam method name

* add testAddTaskNodeSpecialParam UT

* add ProcessDefinitionServiceTest-ut to pom

* add testImportSubProcess in ProcessDefinitionServiceTest

* add testImportSubProcess in ProcessDefinitionServiceTest

* add testImportProcessDefinition

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java

@@ -460,7 +460,7 @@ public class ProcessDefinitionController extends BaseController{
                 }
             }
 
-            if(deleteFailedIdList.size() > 0){
+            if(!deleteFailedIdList.isEmpty()){
                 putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList.toArray(),","));
             }else{
                 putMsg(result, Status.SUCCESS);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java

@@ -267,11 +267,12 @@ public class ProjectController extends BaseController {
     })
     @PostMapping(value="/import-definition")
     public Result importProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
-                                          @RequestParam("file") MultipartFile file){
+                                          @RequestParam("file") MultipartFile file,
+                                          @RequestParam("projectName") String projectName){
         try{
-            logger.info("import process definition by id, login user:{}",
-                    loginUser.getUserName());
-            Map<String, Object> result = processDefinitionService.importProcessDefinition(loginUser,file);
+            logger.info("import process definition by id, login user:{}, project: {}",
+                    loginUser.getUserName(), projectName);
+            Map<String, Object> result = processDefinitionService.importProcessDefinition(loginUser, file, projectName);
             return returnDataList(result);
         }catch (Exception e){
             logger.error(IMPORT_PROCESS_DEFINE_ERROR.getMsg(),e);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@@ -16,38 +16,80 @@
  */
 package org.apache.dolphinscheduler.api.service;
 
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
 import org.apache.dolphinscheduler.api.ApiApplicationServer;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.DataSource;
+import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
+import org.apache.dolphinscheduler.dao.entity.Project;
 import org.apache.dolphinscheduler.dao.entity.User;
-import com.alibaba.fastjson.JSON;
+import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
+import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
+import org.apache.http.entity.ContentType;
+import org.json.JSONException;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.skyscreamer.jsonassert.JSONAssert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.web.multipart.MultipartFile;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.HashMap;
 import java.util.Map;
 
-@RunWith(SpringRunner.class)
+@RunWith(MockitoJUnitRunner.Silent.class)
 @SpringBootTest(classes = ApiApplicationServer.class)
 public class ProcessDefinitionServiceTest {
     private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionServiceTest.class);
 
-    @Autowired
+    @InjectMocks
     ProcessDefinitionService processDefinitionService;
 
+    @Mock
+    private DataSourceMapper dataSourceMapper;
+
+    @Mock
+    private ProcessDefinitionMapper processDefineMapper;
+
+    @Mock
+    private ProjectMapper projectMapper;
+
+    @Mock
+    private ProjectService projectService;
+
     @Test
     public void queryProccessDefinitionList() throws Exception {
+        String projectName = "project_test1";
+        Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
 
+        Project project = getProject(projectName);
         User loginUser = new User();
         loginUser.setId(-1);
         loginUser.setUserType(UserType.GENERAL_USER);
 
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
+
+        Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
+
         Map<String, Object> map = processDefinitionService.queryProccessDefinitionList(loginUser,"project_test1");
         Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
         logger.info(JSON.toJSONString(map));
@@ -55,10 +97,20 @@ public class ProcessDefinitionServiceTest {
 
     @Test
     public void queryProcessDefinitionListPagingTest() throws Exception {
+        String projectName = "project_test1";
+        Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
+
+        Project project = getProject(projectName);
 
         User loginUser = new User();
         loginUser.setId(-1);
         loginUser.setUserType(UserType.GENERAL_USER);
+
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
+
+        Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
+
         Map<String, Object> map = processDefinitionService.queryProcessDefinitionListPaging(loginUser, "project_test1", "",1, 5,0);
 
         Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
@@ -67,13 +119,243 @@ public class ProcessDefinitionServiceTest {
 
     @Test
     public void deleteProcessDefinitionByIdTest() throws Exception {
+        String projectName = "project_test1";
+        Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
 
+        Project project = getProject(projectName);
         User loginUser = new User();
         loginUser.setId(-1);
         loginUser.setUserType(UserType.GENERAL_USER);
-        Map<String, Object> map = processDefinitionService.deleteProcessDefinitionById(loginUser, "li_sql_test", 6);
+
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
+        Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
+
+        Map<String, Object> map = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 6);
 
         Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
         logger.info(JSON.toJSONString(map));
     }
+
+    /**
+     * add datasource param and dependent when export process
+     * @throws JSONException
+     */
+    @Test
+    public void testAddTaskNodeSpecialParam() throws JSONException {
+
+        Mockito.when(dataSourceMapper.selectById(1)).thenReturn(getDataSource());
+        Mockito.when(processDefineMapper.queryByDefineId(2)).thenReturn(getProcessDefinition());
+
+
+        String sqlDependentJson = "{\"globalParams\":[]," +
+                "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," +
+                "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," +
+                "\"udfs\":\"\",\"sqlType\":\"1\",\"title\":\"\",\"receivers\":\"\",\"receiversCc\":\"\",\"showType\":\"TABLE\"" +
+                ",\"localParams\":[],\"connParams\":\"\"," +
+                "\"preStatements\":[],\"postStatements\":[]}," +
+                "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
+                "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\"," +
+                "\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
+                "\"preTasks\":[\"dependent\"]},{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," +
+                "\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"," +
+                "\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\"," +
+                "\"dependItemList\":[{\"projectId\":2,\"definitionId\":46,\"depTasks\":\"ALL\"," +
+                "\"cycle\":\"day\",\"dateValue\":\"today\"}]}]},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
+                "\"timeout\":{\"strategy\":\"\",\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," +
+                "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
+
+        String corSqlDependentJson = processDefinitionService.addTaskNodeSpecialParam(sqlDependentJson);
+
+
+        JSONAssert.assertEquals(sqlDependentJson,corSqlDependentJson,false);
+
+    }
+
+    /**
+     * import sub process test
+     */
+    @Test
+    public void testImportSubProcess() {
+
+        User loginUser = new User();
+        loginUser.setId(1);
+        loginUser.setUserType(UserType.ADMIN_USER);
+
+        Project testProject = getProject("test");
+
+        //Recursive subprocess sub2 process in sub1 process and sub1process in top process
+        String topProcessJson = "{\"globalParams\":[]," +
+                "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-38634\",\"name\":\"shell1\"," +
+                "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," +
+                "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
+                "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
+                "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}," +
+                "{\"type\":\"SUB_PROCESS\",\"id\":\"tasks-44207\",\"name\":\"shell-4\"," +
+                "\"params\":{\"processDefinitionId\":39},\"description\":\"\",\"runFlag\":\"NORMAL\"," +
+                "\"dependence\":{},\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
+                "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
+                "\"preTasks\":[\"shell1\"]}],\"tenantId\":1,\"timeout\":0}";
+
+        String sub1ProcessJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-84090\"," +
+                "\"name\":\"shell-4\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-4\\\"\"}," +
+                "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
+                "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
+                "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SUB_PROCESS\"," +
+                "\"id\":\"tasks-87364\",\"name\":\"shell-5\"," +
+                "\"params\":{\"processDefinitionId\":46},\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{}," +
+                "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," +
+                "\"workerGroupId\":-1,\"preTasks\":[\"shell-4\"]}],\"tenantId\":1,\"timeout\":0}";
+
+        String sub2ProcessJson = "{\"globalParams\":[]," +
+                "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," +
+                "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," +
+                "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
+                "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
+                "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
+
+
+        JSONObject jsonObject = JSONUtils.parseObject(topProcessJson);
+        JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
+
+        String originSubJson = jsonArray.toString();
+
+        Map<Integer, Integer> subProcessIdMap = new HashMap<>(20);
+
+        ProcessDefinition shellDefinition1 = new ProcessDefinition();
+        shellDefinition1.setId(39);
+        shellDefinition1.setName("shell-4");
+        shellDefinition1.setProjectId(2);
+        shellDefinition1.setProcessDefinitionJson(sub1ProcessJson);
+
+        ProcessDefinition shellDefinition2 = new ProcessDefinition();
+        shellDefinition2.setId(46);
+        shellDefinition2.setName("shell-5");
+        shellDefinition2.setProjectId(2);
+        shellDefinition2.setProcessDefinitionJson(sub2ProcessJson);
+
+        Mockito.when(processDefineMapper.queryByDefineId(39)).thenReturn(shellDefinition1);
+        Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2);
+        Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-5")).thenReturn(null);
+        Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-4")).thenReturn(null);
+        Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "testProject")).thenReturn(shellDefinition2);
+
+        processDefinitionService.importSubProcess(loginUser,testProject,jsonArray,subProcessIdMap);
+
+        String correctSubJson = jsonArray.toString();
+
+        Assert.assertEquals(originSubJson, correctSubJson);
+
+    }
+
+    @Test
+    public void testImportProcessDefinitionById() throws IOException {
+
+        String processJson = "{\"projectName\":\"testProject\",\"processDefinitionName\":\"shell-4\"," +
+                "\"processDefinitionJson\":\"{\\\"tenantId\\\":1,\\\"globalParams\\\":[]," +
+                "\\\"tasks\\\":[{\\\"workerGroupId\\\":-1,\\\"description\\\":\\\"\\\",\\\"runFlag\\\":\\\"NORMAL\\\"," +
+                "\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\":\\\"#!/bin/bash\\\\necho \\\\\\\"shell-4\\\\\\\"\\\"," +
+                "\\\"localParams\\\":[],\\\"resourceList\\\":[]},\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}," +
+                "\\\"maxRetryTimes\\\":\\\"0\\\",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-4\\\"," +
+                "\\\"dependence\\\":{},\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-84090\\\"}," +
+                "{\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-5\\\",\\\"workerGroupId\\\":-1," +
+                "\\\"description\\\":\\\"\\\",\\\"dependence\\\":{},\\\"preTasks\\\":[\\\"shell-4\\\"],\\\"id\\\":\\\"tasks-87364\\\"," +
+                "\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SUB_PROCESS\\\",\\\"params\\\":{\\\"processDefinitionId\\\":46}," +
+                "\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}}],\\\"timeout\\\":0}\"," +
+                "\"processDefinitionDescription\":\"\",\"processDefinitionLocations\":\"{\\\"tasks-84090\\\":{\\\"name\\\":\\\"shell-4\\\"," +
+                "\\\"targetarr\\\":\\\"\\\",\\\"x\\\":128,\\\"y\\\":114},\\\"tasks-87364\\\":{\\\"name\\\":\\\"shell-5\\\"," +
+                "\\\"targetarr\\\":\\\"tasks-84090\\\",\\\"x\\\":266,\\\"y\\\":115}}\"," +
+                "\"processDefinitionConnects\":\"[{\\\"endPointSourceId\\\":\\\"tasks-84090\\\"," +
+                "\\\"endPointTargetId\\\":\\\"tasks-87364\\\"}]\"}";
+
+        String subProcessJson = "{\"globalParams\":[]," +
+                "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," +
+                "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," +
+                "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
+                "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
+                "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
+
+        FileUtils.writeStringToFile(new File("/tmp/task.json"),processJson);
+
+        File file = new File("/tmp/task.json");
+
+        FileInputStream fileInputStream = new FileInputStream("/tmp/task.json");
+
+        MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(),
+                ContentType.APPLICATION_OCTET_STREAM.toString(), fileInputStream);
+
+        User loginUser = new User();
+        loginUser.setId(1);
+        loginUser.setUserType(UserType.ADMIN_USER);
+
+        String currentProjectName = "testProject";
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.SUCCESS, currentProjectName);
+
+        ProcessDefinition shellDefinition2 = new ProcessDefinition();
+        shellDefinition2.setId(46);
+        shellDefinition2.setName("shell-5");
+        shellDefinition2.setProjectId(2);
+        shellDefinition2.setProcessDefinitionJson(subProcessJson);
+
+        Mockito.when(projectMapper.queryByName(currentProjectName)).thenReturn(getProject(currentProjectName));
+        Mockito.when(projectService.checkProjectAndAuth(loginUser, getProject(currentProjectName), currentProjectName)).thenReturn(result);
+        Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2);
+
+        //import process
+        Map<String, Object> importProcessResult = processDefinitionService.importProcessDefinition(loginUser, multipartFile, currentProjectName);
+
+        Assert.assertEquals(Status.SUCCESS, importProcessResult.get(Constants.STATUS));
+
+        boolean delete = file.delete();
+
+        Assert.assertTrue(delete);
+
+
+    }
+
+    /**
+     * get mock datasource
+     * @return DataSource
+     */
+    private DataSource getDataSource(){
+        DataSource dataSource = new DataSource();
+        dataSource.setId(2);
+        dataSource.setName("test");
+        return  dataSource;
+    }
+
+    /**
+     * get mock processDefinition
+     * @return ProcessDefinition
+     */
+    private ProcessDefinition getProcessDefinition(){
+        ProcessDefinition processDefinition = new ProcessDefinition();
+        processDefinition.setId(46);
+        processDefinition.setName("testProject");
+        processDefinition.setProjectId(2);
+        return  processDefinition;
+    }
+
+    /**
+     * get mock Project
+     * @param projectName projectName
+     * @return Project
+     */
+    private Project getProject(String projectName){
+        Project project = new Project();
+        project.setId(1);
+        project.setName(projectName);
+        project.setUserId(1);
+        return  project;
+    }
+
+    private void putMsg(Map<String, Object> result, Status status, Object... statusParams) {
+        result.put(Constants.STATUS, status);
+        if (statusParams != null && statusParams.length > 0) {
+            result.put(Constants.MSG, MessageFormat.format(status.getMsg(), statusParams));
+        } else {
+            result.put(Constants.MSG, status.getMsg());
+        }
+    }
 }
\ No newline at end of file

dolphinscheduler-ui/src/js/module/components/fileUpdate/definitionUpdate.vue

@@ -141,6 +141,7 @@
           let self = this
           let formData = new FormData()
           formData.append('file', this.file)
+          formData.append('projectName',this.store.state.dag.projectName)
           io.post(`projects/import-definition`, res => {
             this.$message.success(res.msg)
             resolve()

pom.xml

@@ -675,6 +675,7 @@
 						<include>**/api/service/WorkerGroupServiceTest.java</include>
 						<include>**/api/service/AlertGroupServiceTest.java</include>
 						<include>**/api/service/ProjectServiceTest.java</include>
+						<include>**/api/service/ProcessDefinitionServiceTest.java</include>
 						<include>**/api/service/UdfFuncServiceTest.java</include>
 						<include>**/alert/utils/ExcelUtilsTest.java</include>
 						<include>**/alert/utils/FuncUtilsTest.java</include>