Change the name of the shell node and modify the parameter transmission method...

Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes (#2416)

* Fix the problem of data echo in script edit box

* Optimize resource tree

* Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes
Co-authored-by: Ndailidong <dailidong66@gmail.com>

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue

@@ -211,7 +211,9 @@
           return {
             label: node.name
           }
-        }
+        },
+        allNoResources: [],
+        noRes: [],
       }
     },
     props: {
@@ -300,6 +302,12 @@
           return false
         }
 
+        // noRes
+        if (this.noRes.length>0) {
+          this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`)
+          return false
+        }
+
         // localParams Subcomponent verification
         if (!this.$refs.refLocalParams._verifProp()) {
           return false
@@ -339,6 +347,67 @@
         }
         delete item.children
       },
+      searchTree(element, id) {
+        // 根据id查找节点
+        if (element.id == id) {
+          return element;
+        } else if (element.children != null) {
+          var i;
+          var result = null;
+          for (i = 0; result == null && i < element.children.length; i++) {
+            result = this.searchTree(element.children[i], id);
+          }
+          return result;
+        }
+        return null;
+      },
+      dataProcess(backResource) {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return item.id
+          })
+          Array.prototype.diff = function(a) {
+            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          };
+          let diffSet = this.resourceList.diff(resourceIdArr);
+          let optionsCmp = []
+          if(diffSet.length>0) {
+            diffSet.forEach(item=>{
+              backResource.forEach(item1=>{
+                if(item==item1.id || item==item1.res) {
+                  optionsCmp.push(item1)
+                }
+              })
+            })
+          }
+          let noResources = [{
+            id: -1,
+            name: $t('No resources'),
+            fullName: '/'+$t('No resources'),
+            children: []
+          }]
+          if(optionsCmp.length>0) {
+            this.allNoResources = optionsCmp
+            optionsCmp = optionsCmp.map(item=>{
+              return {id: item.id,name: item.name,fullName: item.res}
+            })
+            optionsCmp.forEach(item=>{
+              item.isNew = true
+            })
+            noResources[0].children = optionsCmp
+            this.mainJarList = this.mainJarList.concat(noResources)
+          }
+        }
+      },
     },
     watch: {
       // Listening type
@@ -354,15 +423,37 @@
     },
     computed: {
       cacheParams () {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return {id: item.id,name: item.name,res: item.fullName}
+          })
+        }
+        let result = []
+        resourceIdArr.forEach(item=>{
+          this.allNoResources.forEach(item1=>{
+            if(item.id==item1.id) {
+              // resultBool = true
+             result.push(item1)
+            }
+          })
+        })
+        this.noRes = result
         return {
           mainClass: this.mainClass,
           mainJar: {
             id: this.mainJar
           },
           deployMode: this.deployMode,
-          resourceList: _.map(this.resourceList, v => {
-            return {id: v}
-          }),
+          resourceList: resourceIdArr,
           localParams: this.localParams,
           slot: this.slot,
           taskManager: this.taskManager,
@@ -404,20 +495,24 @@
           this.programType = o.params.programType || 'SCALA'
 
           // backfill resourceList
+          let backResource = o.params.resourceList || []
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
             _.map(resourceList, v => {
-              if(v.res) {
+              if(!v.id) {
                 this.store.dispatch('dag/getResourceId',{
                   type: 'FILE',
                   fullName: '/'+v.res
                 }).then(res => {
                   this.resourceList.push(res.id)
+                  this.dataProcess(backResource)
                 }).catch(e => {
-                  this.$message.error(e.msg || '')
+                  this.resourceList.push(v.res)
+                  this.dataProcess(backResource)
                 })
               } else {
                 this.resourceList.push(v.id)
+                this.dataProcess(backResource)
               }
             })
             this.cacheResourceList = resourceList

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue

@@ -44,7 +44,7 @@
     <m-list-box>
       <div slot="text">{{$t('Main jar package')}}</div>
       <div slot="content">
-        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :placeholder="$t('Please enter main jar package')">
+        <treeselect v-model="mainJar" :options="mainJarLists" :disable-branch-nodes="true" :normalizer="normalizer" :value-consists-of="valueConsistsOf"  :placeholder="$t('Please enter main jar package')">
           <div slot="value-label" slot-scope="{ node }">{{ node.raw.fullName }}</div>
         </treeselect>
       </div>
@@ -109,6 +109,7 @@
     name: 'mr',
     data () {
       return {
+        valueConsistsOf: 'LEAF_PRIORITY',
         // Main function class
         mainClass: '',
         // Master jar package
@@ -134,7 +135,9 @@
           return {
             label: node.name
           }
-        }
+        },
+        allNoResources: [],
+        noRes: []
       }
     },
     props: {
@@ -176,9 +179,76 @@
       diGuiTree(item) {  // Recursive convenience tree structure
         item.forEach(item => {
           item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　
-            delete item.children : this.diGuiTree(item.children);
+            this.operationTree(item) : this.diGuiTree(item.children);
         })
       },
+      operationTree(item) {
+        if(item.dirctory) {
+          item.isDisabled =true
+        }
+        delete item.children
+      },
+      searchTree(element, id) {
+        // 根据id查找节点
+        if (element.id == id) {
+          return element;
+        } else if (element.children != null) {
+          var i;
+          var result = null;
+          for (i = 0; result == null && i < element.children.length; i++) {
+            result = this.searchTree(element.children[i], id);
+          }
+          return result;
+        }
+        return null;
+      },
+      dataProcess(backResource) {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return item.id
+          })
+          Array.prototype.diff = function(a) {
+            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          };
+          let diffSet = this.resourceList.diff(resourceIdArr);
+          let optionsCmp = []
+          if(diffSet.length>0) {
+            diffSet.forEach(item=>{
+              backResource.forEach(item1=>{
+                if(item==item1.id || item==item1.res) {
+                  optionsCmp.push(item1)
+                }
+              })
+            })
+          }
+          let noResources = [{
+            id: -1,
+            name: $t('No resources'),
+            fullName: '/'+$t('No resources'),
+            children: []
+          }]
+          if(optionsCmp.length>0) {
+            this.allNoResources = optionsCmp
+            optionsCmp = optionsCmp.map(item=>{
+              return {id: item.id,name: item.name,fullName: item.res}
+            })
+            optionsCmp.forEach(item=>{
+              item.isNew = true
+            })
+            noResources[0].children = optionsCmp
+            this.mainJarList = this.mainJarList.concat(noResources)
+          }
+        }
+      },
       /**
        * verification
        */
@@ -193,6 +263,12 @@
           return false
         }
 
+        // noRes
+        if (this.noRes.length>0) {
+          this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`)
+          return false
+        }
+
         // localParams Subcomponent verification
         if (!this.$refs.refLocalParams._verifProp()) {
           return false
@@ -231,14 +307,36 @@
     },
     computed: {
       cacheParams () {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return {id: item.id,name: item.name,res: item.fullName}
+          })
+        }
+        let result = []
+        resourceIdArr.forEach(item=>{
+          this.allNoResources.forEach(item1=>{
+            if(item.id==item1.id) {
+              // resultBool = true
+             result.push(item1)
+            }
+          })
+        })
+        this.noRes = result
         return {
           mainClass: this.mainClass,
           mainJar: {
             id: this.mainJar
           },
-          resourceList: _.map(this.resourceList, v => {
-            return {id: v}
-          }),
+          resourceList: resourceIdArr,
           localParams: this.localParams,
           mainArgs: this.mainArgs,
           others: this.others,
@@ -273,23 +371,27 @@
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
             _.map(resourceList, v => {
-              if(v.res) {
+              if(!v.id) {
                 this.store.dispatch('dag/getResourceId',{
                   type: 'FILE',
                   fullName: '/'+v.res
                 }).then(res => {
                   this.resourceList.push(res.id)
+                  this.dataProcess(backResource)
                 }).catch(e => {
-                  this.$message.error(e.msg || '')
+                  this.resourceList.push(v.res)
+                  this.dataProcess(backResource)
                 })
               } else {
                 this.resourceList.push(v.id)
+                this.dataProcess(backResource)
               }
             })
             this.cacheResourceList = resourceList
           }
 
           // backfill localParams
+          let backResource = o.params.resourceList || []
           let localParams = o.params.localParams || []
           if (localParams.length) {
             this.localParams = localParams

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@@ -80,6 +80,13 @@
         // Cache ResourceList
         cacheResourceList: [],
         resourceOptions: [],
+        normalizer(node) {
+          return {
+            label: node.name
+          }
+        },
+        allNoResources: [],
+        noRes: []
       }
     },
     mixins: [disabledState],
@@ -96,9 +103,9 @@
       /**
        * return resourceList
        */
-      _onResourcesData (a) {
-        this.resourceList = a
-      },
+      // _onResourcesData (a) {
+      //   this.resourceList = a
+      // },
       /**
        * cache resourceList
        */
@@ -120,6 +127,12 @@
           return false
         }
 
+        // noRes
+        if (this.noRes.length>0) {
+          this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`)
+          return false
+        }
+
         // storage
         this.$emit('on-params', {
           resourceList: _.map(this.resourceList, v => {
@@ -166,6 +179,67 @@
           item.isDisabled =true
         }
         delete item.children
+      },
+      searchTree(element, id) {
+        // 根据id查找节点
+        if (element.id == id) {
+          return element;
+        } else if (element.children != null) {
+          var i;
+          var result = null;
+          for (i = 0; result == null && i < element.children.length; i++) {
+            result = this.searchTree(element.children[i], id);
+          }
+          return result;
+        }
+        return null;
+      },
+      dataProcess(backResource) {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.resourceOptions.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return item.id
+          })
+          Array.prototype.diff = function(a) {
+            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          };
+          let diffSet = this.resourceList.diff(resourceIdArr);
+          let optionsCmp = []
+          if(diffSet.length>0) {
+            diffSet.forEach(item=>{
+              backResource.forEach(item1=>{
+                if(item==item1.id || item==item1.res) {
+                  optionsCmp.push(item1)
+                }
+              })
+            })
+          }
+          let noResources = [{
+            id: -1,
+            name: $t('No resources'),
+            fullName: '/'+$t('No resources'),
+            children: []
+          }]
+          if(optionsCmp.length>0) {
+            this.allNoResources = optionsCmp
+            optionsCmp = optionsCmp.map(item=>{
+              return {id: item.id,name: item.name,fullName: item.res}
+            })
+            optionsCmp.forEach(item=>{
+              item.isNew = true
+            })
+            noResources[0].children = optionsCmp
+            this.resourceOptions = this.resourceOptions.concat(noResources)
+          }
+        }
       }
     },
     watch: {
@@ -176,10 +250,32 @@
     },
     computed: {
       cacheParams () {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.resourceOptions.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return {id: item.id,name: item.name,res: item.fullName}
+          })
+        }
+        let result = []
+        resourceIdArr.forEach(item=>{
+          this.allNoResources.forEach(item1=>{
+            if(item.id==item1.id) {
+              // resultBool = true
+             result.push(item1)
+            }
+          })
+        })
+        this.noRes = result
         return {
-          resourceList: _.map(this.resourceList, v => {
-            return {id: v}
-          }),
+          resourceList: resourceIdArr,
           localParams: this.localParams
         }
       }
@@ -187,7 +283,7 @@
     created () {
       let item = this.store.state.dag.resourcesListS
       this.diGuiTree(item)
-      this.options = item
+      this.resourceOptions = item
       let o = this.backfillItem
 
       // Non-null objects represent backfill
@@ -195,17 +291,20 @@
         this.rawScript = o.params.rawScript || ''
 
         // backfill resourceList
+        let backResource = o.params.resourceList || []
         let resourceList = o.params.resourceList || []
         if (resourceList.length) {
           _.map(resourceList, v => {
-            if(v.res) {
+            if(!v.id) {
               this.store.dispatch('dag/getResourceId',{
                 type: 'FILE',
                 fullName: '/'+v.res
               }).then(res => {
                 this.resourceList.push(res.id)
+                this.dataProcess(backResource)
               }).catch(e => {
-                this.$message.error(e.msg || '')
+                this.resourceList.push(v.res)
+                this.dataProcess(backResource)
               })
             } else {
               this.resourceList.push(v.id)
@@ -230,6 +329,6 @@
       editor.toTextArea() // Uninstall
       editor.off($('.code-python-mirror'), 'keypress', this.keypress)
     },
-    components: { mLocalParams, mListBox, mResources }
+    components: { mLocalParams, mListBox, mResources,Treeselect }
   }
 </script>

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@@ -266,8 +266,8 @@
           }
           let noResources = [{
             id: -1,
-            name: $t('No resources'),
-            fullName: '/'+$t('No resources'),
+            name: $t('Unauthorized or deleted resources'),
+            fullName: '/'+$t('Unauthorized or deleted resources'),
             children: []
           }]
           if(optionsCmp.length>0) {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue

@@ -254,7 +254,9 @@
           return {
             label: node.name
           }
-        }
+        },
+        allNoResources: [],
+        noRes: []
       }
     },
     props: {
@@ -305,6 +307,67 @@
         }
         delete item.children
       },
+      searchTree(element, id) {
+        // 根据id查找节点
+        if (element.id == id) {
+          return element;
+        } else if (element.children != null) {
+          var i;
+          var result = null;
+          for (i = 0; result == null && i < element.children.length; i++) {
+            result = this.searchTree(element.children[i], id);
+          }
+          return result;
+        }
+        return null;
+      },
+      dataProcess(backResource) {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return item.id
+          })
+          Array.prototype.diff = function(a) {
+            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          };
+          let diffSet = this.resourceList.diff(resourceIdArr);
+          let optionsCmp = []
+          if(diffSet.length>0) {
+            diffSet.forEach(item=>{
+              backResource.forEach(item1=>{
+                if(item==item1.id || item==item1.res) {
+                  optionsCmp.push(item1)
+                }
+              })
+            })
+          }
+          let noResources = [{
+            id: -1,
+            name: $t('No resources'),
+            fullName: '/'+$t('No resources'),
+            children: []
+          }]
+          if(optionsCmp.length>0) {
+            this.allNoResources = optionsCmp
+            optionsCmp = optionsCmp.map(item=>{
+              return {id: item.id,name: item.name,fullName: item.res}
+            })
+            optionsCmp.forEach(item=>{
+              item.isNew = true
+            })
+            noResources[0].children = optionsCmp
+            this.mainJarList = this.mainJarList.concat(noResources)
+          }
+        }
+      },
       /**
        * verification
        */
@@ -324,6 +387,12 @@
           return false
         }
 
+        // noRes
+        if (this.noRes.length>0) {
+          this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`)
+          return false
+        }
+
         if (!Number.isInteger(parseInt(this.numExecutors))) {
           this.$message.warning(`${i18n.$t('The number of Executors should be a positive integer')}`)
           return false
@@ -400,15 +469,37 @@
     },
     computed: {
       cacheParams () {
+        let isResourceId = []
+        let resourceIdArr = []
+        if(this.resourceList.length>0) {
+          this.resourceList.forEach(v=>{
+            this.mainJarList.forEach(v1=>{
+              if(this.searchTree(v1,v)) {
+                isResourceId.push(this.searchTree(v1,v))
+              }
+            })
+          })
+          resourceIdArr = isResourceId.map(item=>{
+            return {id: item.id,name: item.name,res: item.fullName}
+          })
+        }
+        let result = []
+        resourceIdArr.forEach(item=>{
+          this.allNoResources.forEach(item1=>{
+            if(item.id==item1.id) {
+              // resultBool = true
+             result.push(item1)
+            }
+          })
+        })
+        this.noRes = result
         return {
           mainClass: this.mainClass,
           mainJar: {
             id: this.mainJar
           },
           deployMode: this.deployMode,
-          resourceList: _.map(this.resourceList, v => {
-            return {id: v}
-          }),
+          resourceList: resourceIdArr,
           localParams: this.localParams,
           driverCores: this.driverCores,
           driverMemory: this.driverMemory,
@@ -453,20 +544,24 @@
           this.sparkVersion = o.params.sparkVersion || 'SPARK2'
 
           // backfill resourceList
+          let backResource = o.params.resourceList || []
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
             _.map(resourceList, v => {
-              if(v.res) {
+              if(!v.id) {
                 this.store.dispatch('dag/getResourceId',{
                   type: 'FILE',
                   fullName: '/'+v.res
                 }).then(res => {
                   this.resourceList.push(res.id)
+                  this.dataProcess(backResource)
                 }).catch(e => {
-                  this.$message.error(e.msg || '')
+                  this.resourceList.push(v.res)
+                  this.dataProcess(backResource)
                 })
               } else {
                 this.resourceList.push(v.id)
+                this.dataProcess(backResource)
               }
             })
             this.cacheResourceList = resourceList

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -588,6 +588,6 @@ export default {
   'Branch flow': 'Branch flow',
   'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow',
   'Successful branch flow and failed branch flow are required': 'Successful branch flow and failed branch flow are required',
-  'No resources': 'No resources',
+  'Unauthorized or deleted resources': 'Unauthorized or deleted resources',
   'Please delete all non-existent resources': 'Please delete all non-existent resources',
 }

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -588,6 +588,6 @@ export default {
   'Branch flow': '分支流转',
   'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点',
   'Successful branch flow and failed branch flow are required': '成功分支流转和失败分支流转必填',
-  'No resources': '未授权或已删除资源',
+  'Unauthorized or deleted resources': '未授权或已删除资源',
   'Please delete all non-existent resources': '请删除所有未授权或已删除资源',
 }