Merge branch 'dev' into dev

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java

@@ -94,6 +94,30 @@ public class ProcessDefinitionController extends BaseController {
         return returnDataList(result);
     }
 
+    /**
+     * copy process definition
+     *
+     * @param loginUser   login user
+     * @param projectName project name
+     * @param processId   process definition id
+     * @return copy result code
+     */
+    @ApiOperation(value = "copyProcessDefinition", notes= "COPY_PROCESS_DEFINITION_NOTES")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "processId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100")
+    })
+    @PostMapping(value = "/copy")
+    @ResponseStatus(HttpStatus.OK)
+    @ApiException(COPY_PROCESS_DEFINITION_ERROR)
+    public Result copyProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+                                        @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
+                                        @RequestParam(value = "processId", required = true) int processId) throws JsonProcessingException {
+        logger.info("copy process definition, login user:{}, project name:{}, process definition id:{}",
+                loginUser.getUserName(), projectName, processId);
+        Map<String, Object> result = processDefinitionService.copyProcessDefinition(loginUser, projectName, processId);
+        return returnDataList(result);
+    }
+
     /**
      * verify process definition name unique
      *

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -168,15 +168,13 @@ public enum Status {
     PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"),
     PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"),
     SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"),
-    DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"),
-    DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"),
-    DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"),
-
-    DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"),
-
-    QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"),
-    DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"),
-
+    DELETE_TENANT_BY_ID_FAIL(10142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"),
+    DELETE_TENANT_BY_ID_FAIL_DEFINES(10143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"),
+    DELETE_TENANT_BY_ID_FAIL_USERS(10144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"),
+    DELETE_WORKER_GROUP_BY_ID_FAIL(10145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"),
+    QUERY_WORKER_GROUP_FAIL(10146,"query worker group fail ", "查询worker分组失败"),
+    DELETE_WORKER_GROUP_FAIL(10147,"delete worker group fail ", "删除worker分组失败"),
+    COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"),
 
     UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
     UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@@ -112,8 +112,13 @@ public class ProcessDefinitionService extends BaseDAGService {
      * @return create result code
      * @throws JsonProcessingException JsonProcessingException
      */
-    public Map<String, Object> createProcessDefinition(User loginUser, String projectName, String name,
-                                                       String processDefinitionJson, String desc, String locations, String connects) throws JsonProcessingException {
+    public Map<String, Object> createProcessDefinition(User loginUser,
+                                                       String projectName,
+                                                       String name,
+                                                       String processDefinitionJson,
+                                                       String desc,
+                                                       String locations,
+                                                       String connects) throws JsonProcessingException {
 
         Map<String, Object> result = new HashMap<>(5);
         Project project = projectMapper.queryByName(projectName);
@@ -281,6 +286,41 @@ public class ProcessDefinitionService extends BaseDAGService {
         return result;
     }
 
+    /**
+     * copy process definition
+     *
+     * @param loginUser login user
+     * @param projectName project name
+     * @param processId process definition id
+     * @return copy result code
+     */
+    public Map<String, Object> copyProcessDefinition(User loginUser, String projectName, Integer processId) throws JsonProcessingException{
+
+        Map<String, Object> result = new HashMap<>(5);
+        Project project = projectMapper.queryByName(projectName);
+
+        Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
+        Status resultStatus = (Status) checkResult.get(Constants.STATUS);
+        if (resultStatus != Status.SUCCESS) {
+            return checkResult;
+        }
+
+        ProcessDefinition processDefinition = processDefineMapper.selectById(processId);
+        if (processDefinition == null) {
+            putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId);
+            return result;
+        } else {
+           return createProcessDefinition(
+                   loginUser,
+                   projectName,
+                   processDefinition.getName()+"_copy_"+System.currentTimeMillis(),
+                   processDefinition.getProcessDefinitionJson(),
+                   processDefinition.getDescription(),
+                   processDefinition.getLocations(),
+                   processDefinition.getConnects());
+        }
+    }
+
     /**
      * update  process definition
      *

dolphinscheduler-api/src/main/resources/i18n/messages.properties

@@ -173,6 +173,7 @@ PROCESS_DEFINITION_ID=process definition id
 PROCESS_DEFINITION_IDS=process definition ids
 RELEASE_PROCESS_DEFINITION_NOTES=release process definition
 QUERY_PROCESS_DEFINITION_BY_ID_NOTES=query process definition by id
+COPY_PROCESS_DEFINITION_NOTES=copy process definition
 QUERY_PROCESS_DEFINITION_LIST_NOTES=query process definition list
 QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=query process definition list paging
 QUERY_ALL_DEFINITION_LIST_NOTES=query all definition list

dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties

@@ -173,6 +173,7 @@ PROCESS_DEFINITION_ID=process definition id
 PROCESS_DEFINITION_IDS=process definition ids
 RELEASE_PROCESS_DEFINITION_NOTES=release process definition
 QUERY_PROCESS_DEFINITION_BY_ID_NOTES=query process definition by id
+COPY_PROCESS_DEFINITION_NOTES=copy process definition
 QUERY_PROCESS_DEFINITION_LIST_NOTES=query process definition list
 QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=query process definition list paging
 QUERY_ALL_DEFINITION_LIST_NOTES=query all definition list

dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties

@@ -171,6 +171,7 @@ UPDATE_PROCESS_DEFINITION_NOTES=更新流程定义
 PROCESS_DEFINITION_ID=流程定义ID
 RELEASE_PROCESS_DEFINITION_NOTES=发布流程定义
 QUERY_PROCESS_DEFINITION_BY_ID_NOTES=查询流程定义通过流程定义ID
+COPY_PROCESS_DEFINITION_NOTES=复制流程定义
 QUERY_PROCESS_DEFINITION_LIST_NOTES=查询流程定义列表
 QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=分页查询流程定义列表
 QUERY_ALL_DEFINITION_LIST_NOTES=查询所有流程定义

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java

@@ -174,6 +174,21 @@ public class ProcessDefinitionControllerTest{
         Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue());
     }
 
+    @Test
+    public void testCopyProcessDefinition() throws Exception {
+
+        String projectName = "test";
+        int id = 1;
+
+        Map<String, Object> result = new HashMap<>(5);
+        putMsg(result, Status.SUCCESS);
+
+        Mockito.when(processDefinitionService.copyProcessDefinition(user, projectName,id)).thenReturn(result);
+        Result response = processDefinitionController.copyProcessDefinition(user, projectName,id);
+
+        Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue());
+    }
+
 
     @Test
     public void testQueryProcessDefinitionList() throws Exception {

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@@ -198,6 +198,47 @@ public class ProcessDefinitionServiceTest {
         Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
     }
 
+    @Test
+    public void testCopyProcessDefinition()  throws Exception{
+        String projectName = "project_test1";
+        Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
+
+        Project project = getProject(projectName);
+
+        User loginUser = new User();
+        loginUser.setId(-1);
+        loginUser.setUserType(UserType.GENERAL_USER);
+
+        Map<String, Object> result = new HashMap<>(5);
+        //project check auth success, instance not exist
+        putMsg(result, Status.SUCCESS, projectName);
+        Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
+
+        ProcessDefinition definition = getProcessDefinition();
+        definition.setLocations("{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}");
+        definition.setProcessDefinitionJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}");
+        definition.setConnects("[]");
+        //instance exit
+        Mockito.when(processDefineMapper.selectById(46)).thenReturn(definition);
+
+        Map<String, Object> createProcessResult = new HashMap<>(5);
+        putMsg(result, Status.SUCCESS);
+
+        Mockito.when(processDefinitionService.createProcessDefinition(
+                loginUser,
+                definition.getProjectName(),
+                definition.getName(),
+                definition.getProcessDefinitionJson(),
+                definition.getDescription(),
+                definition.getLocations(),
+                definition.getConnects())).thenReturn(createProcessResult);
+
+        Map<String, Object> successRes = processDefinitionService.copyProcessDefinition(loginUser,
+                "project_test1", 46);
+
+        Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
+    }
+
     @Test
     public void deleteProcessDefinitionByIdTest() throws Exception {
         String projectName = "project_test1";
@@ -770,12 +811,14 @@ public class ProcessDefinitionServiceTest {
      * @return ProcessDefinition
      */
     private ProcessDefinition getProcessDefinition(){
+
         ProcessDefinition processDefinition = new ProcessDefinition();
         processDefinition.setId(46);
         processDefinition.setName("test_pdf");
         processDefinition.setProjectId(2);
         processDefinition.setTenantId(1);
         processDefinition.setDescription("");
+
         return  processDefinition;
     }
 

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java

@@ -133,8 +133,6 @@ public class TaskCallbackServiceTest {
         nettyRemotingClient.close();
     }
 
-
-
     @Test(expected = IllegalArgumentException.class)
     public void testSendAckWithIllegalArgumentException(){
         TaskExecuteAckCommand ackCommand = Mockito.mock(TaskExecuteAckCommand.class);
@@ -178,39 +176,40 @@ public class TaskCallbackServiceTest {
         }
     }
 
-    @Test(expected = IllegalStateException.class)
-    public void testSendAckWithIllegalStateException2(){
-        masterRegistry.registry();
-        final NettyServerConfig serverConfig = new NettyServerConfig();
-        serverConfig.setListenPort(30000);
-        NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor);
-        nettyRemotingServer.start();
-
-        final NettyClientConfig clientConfig = new NettyClientConfig();
-        NettyRemotingClient nettyRemotingClient = new NettyRemotingClient(clientConfig);
-        Channel channel = nettyRemotingClient.getChannel(Host.of("localhost:30000"));
-        taskCallbackService.addRemoteChannel(1, new NettyRemoteChannel(channel, 1));
-        channel.close();
-        TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand();
-        ackCommand.setTaskInstanceId(1);
-        ackCommand.setStartTime(new Date());
+//    @Test(expected = IllegalStateException.class)
+//    public void testSendAckWithIllegalStateException2(){
+//        masterRegistry.registry();
+//        final NettyServerConfig serverConfig = new NettyServerConfig();
+//        serverConfig.setListenPort(30000);
+//        NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig);
+//        nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor);
+//        nettyRemotingServer.start();
+//
+//        final NettyClientConfig clientConfig = new NettyClientConfig();
+//        NettyRemotingClient nettyRemotingClient = new NettyRemotingClient(clientConfig);
+//        Channel channel = nettyRemotingClient.getChannel(Host.of("localhost:30000"));
+//        taskCallbackService.addRemoteChannel(1, new NettyRemoteChannel(channel, 1));
+//        channel.close();
+//        TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand();
+//        ackCommand.setTaskInstanceId(1);
+//        ackCommand.setStartTime(new Date());
+//
+//        nettyRemotingServer.close();
+//
+//        taskCallbackService.sendAck(1, ackCommand.convert2Command());
+//        try {
+//            Thread.sleep(5000);
+//        } catch (InterruptedException e) {
+//            e.printStackTrace();
+//        }
+//
+//        Stopper.stop();
+//
+//        try {
+//            Thread.sleep(5000);
+//        } catch (InterruptedException e) {
+//            e.printStackTrace();
+//        }
+//    }
 
-        nettyRemotingServer.close();
-
-        taskCallbackService.sendAck(1, ackCommand.convert2Command());
-        try {
-            Thread.sleep(5000);
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-
-        Stopper.stop();
-
-        try {
-            Thread.sleep(5000);
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
 }

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue

@@ -48,7 +48,7 @@
     <m-list-box>
       <div slot="text">{{$t('Main jar package')}}</div>
       <div slot="content">
-        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :placeholder="$t('Please enter main jar package')">
+        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :disabled="isDetails" :placeholder="$t('Please enter main jar package')">
           <div slot="value-label" slot-scope="{ node }">{{ node.raw.fullName }}</div>
         </treeselect>
       </div>
@@ -557,4 +557,12 @@
       }
     }
   }
+  .vue-treeselect--disabled {
+    .vue-treeselect__control {
+      background-color: #ecf3f8;
+      .vue-treeselect__single-value {
+        color: #6d859e;
+      }
+    }
+  }
 </style>

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue

@@ -44,7 +44,7 @@
     <m-list-box>
       <div slot="text">{{$t('Main jar package')}}</div>
       <div slot="content">
-        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :value-consists-of="valueConsistsOf"  :placeholder="$t('Please enter main jar package')">
+        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :value-consists-of="valueConsistsOf" :disabled="isDetails"  :placeholder="$t('Please enter main jar package')">
           <div slot="value-label" slot-scope="{ node }">{{ node.raw.fullName }}</div>
         </treeselect>
       </div>
@@ -427,4 +427,12 @@
       }
     }
   }
+  .vue-treeselect--disabled {
+    .vue-treeselect__control {
+      background-color: #ecf3f8;
+      .vue-treeselect__single-value {
+        color: #6d859e;
+      }
+    }
+  }
 </style>

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@@ -333,3 +333,13 @@
     components: { mLocalParams, mListBox, mResources,Treeselect }
   }
 </script>
+<style lang="scss" rel="stylesheet/scss" scope>
+  .vue-treeselect--disabled {
+    .vue-treeselect__control {
+      background-color: #ecf3f8;
+      .vue-treeselect__single-value {
+        color: #6d859e;
+      }
+    }
+  }
+</style>
\ No newline at end of file

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@@ -396,5 +396,12 @@
     right: -12px;
     top: -16px;
   }
-
+  .vue-treeselect--disabled {
+    .vue-treeselect__control {
+      background-color: #ecf3f8;
+      .vue-treeselect__single-value {
+        color: #6d859e;
+      }
+    }
+  }
 </style>

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue

@@ -63,7 +63,7 @@
     <m-list-box>
       <div slot="text">{{$t('Main jar package')}}</div>
       <div slot="content">
-        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :placeholder="$t('Please enter main jar package')">
+        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :disabled="isDetails" :placeholder="$t('Please enter main jar package')">
           <div slot="value-label" slot-scope="{ node }">{{ node.raw.fullName }}</div>
         </treeselect>
       </div>
@@ -606,4 +606,12 @@
       }
     }
   }
+  .vue-treeselect--disabled {
+    .vue-treeselect__control {
+      background-color: #ecf3f8;
+      .vue-treeselect__single-value {
+        color: #6d859e;
+      }
+    }
+  }
 </style>

dolphinscheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue

@@ -43,7 +43,7 @@
     props: {},
     methods: {
       ...mapMutations('dag', ['setIsDetails', 'resetParams']),
-      ...mapActions('dag', ['getProcessList','getProjectList', 'getResourcesList', 'getInstancedetail']),
+      ...mapActions('dag', ['getProcessList','getProjectList', 'getResourcesList', 'getInstancedetail','getResourcesListJar']),
       ...mapActions('security', ['getTenantList','getWorkerGroupsAll']),
       /**
        * init
@@ -62,6 +62,8 @@
           this.getProjectList(),
           // get resources
           this.getResourcesList(),
+          // get jar
+          this.getResourcesListJar(),
           // get worker group list
           this.getWorkerGroupsAll(),
           this.getTenantList()

dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue

@@ -46,7 +46,7 @@
           <th scope="col" width="90">
             <span>{{$t('Timing state')}}</span>
           </th>
-          <th scope="col" width="240">
+          <th scope="col" width="300">
             <span>{{$t('Operation')}}</span>
           </th>
         </tr>
@@ -90,6 +90,7 @@
             <x-button type="info" shape="circle" size="xsmall" data-toggle="tooltip" :title="$t('Timing')" @click="_timing(item)" :disabled="item.releaseState !== 'ONLINE' || item.scheduleReleaseState !== null"  icon="ans-icon-timer"><!--{{$t('定时')}}--></x-button>
             <x-button type="warning" shape="circle" size="xsmall" data-toggle="tooltip" :title="$t('online')" @click="_poponline(item)" v-if="item.releaseState === 'OFFLINE'"  icon="ans-icon-upward"><!--{{$t('下线')}}--></x-button>
             <x-button type="error" shape="circle" size="xsmall" data-toggle="tooltip" :title="$t('offline')" @click="_downline(item)" v-if="item.releaseState === 'ONLINE'"  icon="ans-icon-downward"><!--{{$t('上线')}}--></x-button>
+            <x-button type="info" shape="circle" size="xsmall" data-toggle="tooltip" :title="$t('Copy')" @click="_copyProcess(item)" :disabled="item.releaseState === 'ONLINE'"  icon="ans-icon-copy"><!--{{$t('复制')}}--></x-button>
             <x-button type="info" shape="circle" size="xsmall" data-toggle="tooltip" :title="$t('Cron Manage')" @click="_timingManage(item)" :disabled="item.releaseState !== 'ONLINE'"  icon="ans-icon-datetime"><!--{{$t('定时管理')}}--></x-button>
             <x-poptip
               :ref="'poptip-delete-' + $index"
@@ -158,7 +159,7 @@
       pageSize: Number
     },
     methods: {
-      ...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition','exportDefinition']),
+      ...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition','exportDefinition','copyProcess']),
       _rtPublishStatus (code) {
         return _.filter(publishStatus, v => v.code === code)[0].desc
       },
@@ -306,6 +307,21 @@
           releaseState: 1
         })
       },
+      /**
+       * copy
+       */
+      _copyProcess (item) {
+        this.copyProcess({
+          processId: item.id
+        }).then(res => {
+          this.$message.success(res.msg)
+          $('body').find('.tooltip.fade.top.in').remove()
+          this._onUpdate()
+        }).catch(e => {
+          this.$message.error(e.msg || '')
+        })
+      },
+
       _export (item) {
         this.exportDefinition({
           processDefinitionId: item.id,

dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@@ -90,6 +90,7 @@ export default {
       })
     })
   },
+
   /**
    * Get process definition DAG diagram details
    */
@@ -127,6 +128,22 @@ export default {
       })
     })
   },
+
+/**
+   * Get process definition DAG diagram details
+   */
+  copyProcess ({ state }, payload) {
+    return new Promise((resolve, reject) => {
+      io.post(`projects/${state.projectName}/process/copy`, {
+        processId: payload.processId
+      }, res => {
+        resolve(res)
+      }).catch(e => {
+        reject(e)
+      })
+    })
+  },
+
   /**
    * Get the process instance DAG diagram details
    */

dolphinscheduler-ui/src/js/module/components/nav/nav.vue

@@ -387,7 +387,6 @@
       .logo-m {
         width: 36px;
         height: 36px;
-        background: url("./m_logo.png");
         margin: 0 auto;
         position: relative;
         top: 12px;

dolphinscheduler-ui/src/view/home/index.html

@@ -21,7 +21,9 @@
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
-    <meta http-equiv="Cache-Control" content="no-siteapp" />
+    <meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
+    <meta http-equiv="Pragma" content="no-cache" />
+    <meta http-equiv="Expires" content="0" />
     <meta name="apple-mobile-web-app-capable" content="yes">
     <meta name="apple-mobile-web-app-title" content="标题">
     <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent"/>

charts/dolphinscheduler/Chart.yaml → kubernetes/dolphinscheduler/Chart.yaml

@@ -21,8 +21,8 @@ description: Dolphin Scheduler is a distributed and easy-to-expand visual DAG wo
 home: https://dolphinscheduler.apache.org
 icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg
 keywords:
-  - dolphinscheduler
-  - Scheduler
+- dolphinscheduler
+- Scheduler
 # A chart can be either an 'application' or a 'library' chart.
 #
 # Application charts are a collection of templates that can be packaged into versioned archives
@@ -35,18 +35,18 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.1.0
+version: 1.0.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 1.2.1
+appVersion: 1.3.0
 
 dependencies:
-  - name: postgresql
-    version: 8.x.x
-    repository: https://charts.bitnami.com/bitnami
-    condition: postgresql.enabled
-  - name: zookeeper
-    version: 5.x.x
-    repository: https://charts.bitnami.com/bitnami
-    condition: redis.enabled
+- name: postgresql
+  version: 8.x.x
+  repository: https://charts.bitnami.com/bitnami
+  condition: postgresql.enabled
+- name: zookeeper
+  version: 5.x.x
+  repository: https://charts.bitnami.com/bitnami
+  condition: redis.enabled

charts/README.md → kubernetes/dolphinscheduler/README.md

@@ -16,7 +16,9 @@ To install the chart with the release name `my-release`:
 
 ```bash
 $ git clone https://github.com/apache/incubator-dolphinscheduler.git
-$ cd incubator-dolphinscheduler
+$ cd incubator-dolphinscheduler/kubernetes/dolphinscheduler
+$ helm repo add bitnami https://charts.bitnami.com/bitnami
+$ helm dependency update .
 $ helm install --name dolphinscheduler .
 ```
 These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.

kubernetes/dolphinscheduler/requirements.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dependencies:
+- name: postgresql
+  version: 8.x.x
+  repository: https://charts.bitnami.com/bitnami
+  condition: postgresql.enabled
+- name: zookeeper
+  version: 5.x.x
+  repository: https://charts.bitnami.com/bitnami
+  condition: redis.enabled
\ No newline at end of file

charts/dolphinscheduler/templates/_helpers.tpl → kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -130,20 +130,4 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- define "dolphinscheduler.worker.base.dir" -}}
 {{- $name := default "/tmp/dolphinscheduler" .Values.worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}}
 {{- printf "%s" $name | trunc 63 | trimSuffix "/" -}}
-{{- end -}}
-
-{{/*
-Create a default dolphinscheduler worker data download dir.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "dolphinscheduler.worker.data.download.dir" -}}
-{{- printf "%s%s" (include "dolphinscheduler.worker.base.dir" .) "/download" -}}
-{{- end -}}
-
-{{/*
-Create a default dolphinscheduler worker process exec dir.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "dolphinscheduler.worker.process.exec.dir" -}}
-{{- printf "%s%s" (include "dolphinscheduler.worker.base.dir" .) "/exec" -}}
 {{- end -}}
\ No newline at end of file

charts/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml → kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml

@@ -31,4 +31,6 @@ data:
   MASTER_TASK_COMMIT_INTERVAL: {{ .Values.master.configmap.MASTER_TASK_COMMIT_INTERVAL | quote }}
   MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }}
   MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }}
+  MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }}
+  DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }}
 {{- end }}
\ No newline at end of file

charts/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml → kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@@ -29,9 +29,9 @@ data:
   WORKER_FETCH_TASK_NUM: {{ .Values.worker.configmap.WORKER_FETCH_TASK_NUM | quote }}
   WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }}
   WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }}
+  WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }}
+  WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }}
   DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }}
-  DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH: {{ include "dolphinscheduler.worker.data.download.dir" . | quote }}
-  DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH: {{ include "dolphinscheduler.worker.process.exec.dir" . | quote }}
   dolphinscheduler_env.sh: |-
   {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }}
     {{ . }}

charts/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml → kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -195,7 +195,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - worker-server
+                - AlertServer
             initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }}
@@ -208,7 +208,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - worker-server
+                - AlertServer
             initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }}

charts/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml → kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -109,8 +109,8 @@ spec:
           args:
             - "master-server"
           ports:
-            - containerPort: 8888
-              name: unused-tcp-port
+            - containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }}
+              name: "master-port"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
             - name: TZ
@@ -150,6 +150,16 @@ spec:
                 configMapKeyRef:
                   name: {{ include "dolphinscheduler.fullname" . }}-master
                   key: MASTER_RESERVED_MEMORY
+            - name: MASTER_LISTEN_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "dolphinscheduler.fullname" . }}-master
+                  key: MASTER_LISTEN_PORT
+            - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "dolphinscheduler.fullname" . }}-master
+                  key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
             - name: POSTGRESQL_HOST
               {{- if .Values.postgresql.enabled }}
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
@@ -178,12 +188,6 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
                   {{- end }}
-            - name: TASK_QUEUE
-              {{- if .Values.zookeeper.enabled }}
-              value: {{ .Values.zookeeper.taskQueue }}
-              {{- else }}
-              value: {{ .Values.externalZookeeper.taskQueue }}
-              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
@@ -196,7 +200,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - master-server
+                - MasterServer
             initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }}
@@ -209,7 +213,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - master-server
+                - MasterServer
             initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }}

charts/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml → kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -109,6 +109,8 @@ spec:
           args:
             - "worker-server"
           ports:
+            - containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }}
+              name: "worker-port"
             - containerPort: 50051
               name: "logs-port"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -140,6 +142,21 @@ spec:
                 configMapKeyRef:
                   name: {{ include "dolphinscheduler.fullname" . }}-worker
                   key: WORKER_RESERVED_MEMORY
+            - name: WORKER_LISTEN_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "dolphinscheduler.fullname" . }}-worker
+                  key: WORKER_LISTEN_PORT
+            - name: WORKER_GROUP
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "dolphinscheduler.fullname" . }}-worker
+                  key: WORKER_GROUP
+            - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: {{ include "dolphinscheduler.fullname" . }}-master
+                  key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
             - name: POSTGRESQL_HOST
               {{- if .Values.postgresql.enabled }}
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
@@ -168,12 +185,6 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
             {{- end }}
-            - name: TASK_QUEUE
-              {{- if .Values.zookeeper.enabled }}
-              value: {{ .Values.zookeeper.taskQueue }}
-              {{- else }}
-              value: {{ .Values.externalZookeeper.taskQueue }}
-              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
@@ -186,7 +197,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - worker-server
+                - WorkerServer
             initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
@@ -199,7 +210,7 @@ spec:
               command:
                 - sh
                 - /root/checkpoint.sh
-                - worker-server
+                - WorkerServer
             initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
@@ -247,7 +258,7 @@ spec:
           app.kubernetes.io/managed-by: {{ .Release.Service }}
       spec:
         accessModes:
-          {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }}
+        {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }}
           - {{ . | quote }}
         {{- end }}
         storageClassName: {{ .Values.worker.persistentVolumeClaim.dataPersistentVolume.storageClassName | quote }}
@@ -264,7 +275,7 @@ spec:
           app.kubernetes.io/managed-by: {{ .Release.Service }}
       spec:
         accessModes:
-          {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }}
+        {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }}
           - {{ . | quote }}
         {{- end }}
         storageClassName: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storageClassName | quote }}

charts/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml → kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml

@@ -25,10 +25,10 @@ metadata:
 spec:
   clusterIP: "None"
   ports:
-    - port: 8888
-      targetPort: tcp-port
+    - port: {{ .Values.master.configmap.MASTER_LISTEN_PORT }}
+      targetPort: master-port
       protocol: TCP
-      name: unused-tcp-port
+      name: master-port
   selector:
     app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
     app.kubernetes.io/instance: {{ .Release.Name }}

charts/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml → kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml

@@ -25,6 +25,10 @@ metadata:
 spec:
   clusterIP: "None"
   ports:
+    - port: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }}
+      targetPort: worker-port
+      protocol: TCP
+      name: worker-port
     - port: 50051
       targetPort: logs-port
       protocol: TCP

charts/dolphinscheduler/values.yaml → kubernetes/dolphinscheduler/values.yaml

@@ -27,7 +27,7 @@ timezone: "Asia/Shanghai"
 image:
   registry: "docker.io"
   repository: "dolphinscheduler"
-  tag: "1.2.1"
+  tag: "1.3.0"
   pullPolicy: "IfNotPresent"
 
 imagePullSecrets: []
@@ -56,6 +56,8 @@ externalDatabase:
 zookeeper:
   enabled: true
   taskQueue: "zookeeper"
+  service:
+    port: "2181"
   persistence:
     enabled: false
     size: "20Gi"
@@ -91,6 +93,7 @@ master:
     MASTER_TASK_COMMIT_INTERVAL: "1000"
     MASTER_MAX_CPULOAD_AVG: "100"
     MASTER_RESERVED_MEMORY: "0.1"
+    MASTER_LISTEN_PORT: "5678"
   livenessProbe:
     enabled: true
     initialDelaySeconds: "30"
@@ -156,6 +159,8 @@ worker:
     WORKER_FETCH_TASK_NUM: "3"
     WORKER_MAX_CPULOAD_AVG: "100"
     WORKER_RESERVED_MEMORY: "0.1"
+    WORKER_LISTEN_PORT: "1234"
+    WORKER_GROUP: "default"
     DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
     DOLPHINSCHEDULER_ENV:
     - "export HADOOP_HOME=/opt/soft/hadoop"

pom.xml

@@ -764,6 +764,7 @@
                         <include>**/common/utils/HadoopUtilsTest.java</include>
                         <include>**/common/utils/HttpUtilsTest.java</include>
                         <include>**/common/ConstantsTest.java</include>
+                        <include>**/common/utils/HadoopUtils.java</include>
                         <include>**/dao/mapper/AccessTokenMapperTest.java</include>
                         <include>**/dao/mapper/AlertGroupMapperTest.java</include>
                         <include>**/dao/mapper/CommandMapperTest.java</include>