file upload check login changed

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@@ -62,6 +62,7 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_T
 import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR;
 import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING;
 import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR;
+import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_CATNOT_BE_EMPTY;
 import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY;
 import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST;
 import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_UDF_FUNCTION_ERROR;
@@ -404,7 +405,7 @@ public class ResourcesController extends BaseController {
                                        @RequestParam(value = "currentDir") String currentDir) {
         if (StringUtils.isEmpty(content)) {
             logger.error("resource file contents are not allowed to be empty");
-            return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
+            return error(RESOURCE_FILE_CATNOT_BE_EMPTY.getCode(), RESOURCE_FILE_CATNOT_BE_EMPTY.getMsg());
         }
         return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content, pid,
                 currentDir);
@@ -431,7 +432,7 @@ public class ResourcesController extends BaseController {
                                         @RequestParam(value = "content") String content) {
         if (StringUtils.isEmpty(content)) {
             logger.error("The resource file contents are not allowed to be empty");
-            return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
+            return error(RESOURCE_FILE_CATNOT_BE_EMPTY.getCode(), RESOURCE_FILE_CATNOT_BE_EMPTY.getMsg());
         }
         return resourceService.updateResourceContent(loginUser, resourceId, content);
     }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@@ -90,7 +90,7 @@ public enum Status {
     VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059, "verify resource by name and type error", "资源名称或类型验证错误"),
     VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060, "view resource file online error", "查看资源文件错误"),
     CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061, "create resource file online error", "创建资源文件错误"),
-    RESOURCE_FILE_IS_EMPTY(10062, "resource file is empty", "资源文件内容不能为空"),
+    RESOURCE_FILE_CATNOT_BE_EMPTY(10062, "resource file is empty", "资源文件内容不能为空"),
     EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063, "edit resource file online error", "更新资源文件错误"),
     DOWNLOAD_RESOURCE_FILE_ERROR(10064, "download resource file error", "下载资源文件错误"),
     CREATE_UDF_FUNCTION_ERROR(10065, "create udf function error", "创建UDF函数错误"),
@@ -288,6 +288,7 @@ public enum Status {
     RESOURCE_HAS_FOLDER(20018, "There are files or folders in the current directory:{0}", "当前目录下有文件或文件夹[{0}]"),
 
     BATCH_RESOURCE_NAME_REPEAT(20019, "duplicate file names in this batch", "此批处理中存在重复的文件名"),
+    RESOURCE_FILE_IS_EMPTY(20020, "resource file [{0}] is empty", "资源文件 [{0}] 内容不能为空"),
 
     USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"),
     USER_NO_OPERATION_PROJECT_PERM(30002, "user {0} is not has project {1} permission", "当前用户[{0}]没有[{1}]项目的操作权限"),
@@ -487,7 +488,7 @@ public enum Status {
     K8S_CLIENT_OPS_ERROR(1300006, "k8s error with exception {0}", "k8s操作报错[{0}]"),
     VERIFY_K8S_NAMESPACE_ERROR(1300007, "verify k8s and namespace error", "验证k8s命名空间信息错误"),
     DELETE_K8S_NAMESPACE_BY_ID_ERROR(1300008, "delete k8s namespace by id error", "删除命名空间错误"),
-    VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed", "文件命名校验失败"),
+    VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name [{0}] verify failed", "文件命名[{0}]校验失败"),
     STORE_OPERATE_CREATE_ERROR(1300010, "create the resource failed", "存储操作失败"),
     GRANT_K8S_NAMESPACE_ERROR(1300011, "grant namespace error", "授权资源错误"),
     QUERY_UNAUTHORIZED_NAMESPACE_ERROR(1300012, "query unauthorized namespace error", "查询未授权命名空间错误"),
@@ -497,6 +498,8 @@ public enum Status {
     TENANT_FULL_NAME_TOO_LONG_ERROR(1300016, "tenant's fullname is too long error", "租户名过长"),
     USER_PASSWORD_LENGTH_ERROR(1300017, "user's password length error", "用户密码长度错误"),
     QUERY_CAN_USE_K8S_NAMESPACE_ERROR(1300018, "login user query can used namespace list error", "查询可用命名空间错误"),
+    FILE_NAME_CONTAIN_RESTRICTIONS(1300019, "The file name [{0}] contain restrictions", "文件命名[{0}]包含限制内容"),
+    FILE_TYPE_IS_RESTRICTIONS(1300020, "The file [{0}] type is restrictions", "文件[{0}]类型为限制类型"),
 
     NO_CURRENT_OPERATING_PERMISSION(1400001, "The current user does not have this permission.", "当前用户无此权限"),
     FUNCTION_DISABLED(1400002, "The current feature is disabled.", "当前功能已被禁用"),

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@@ -17,14 +17,13 @@
 
 package org.apache.dolphinscheduler.api.service.impl;
 
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.common.base.Joiner;
-import com.google.common.io.Files;
-import org.apache.commons.beanutils.BeanMap;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
+import static org.apache.dolphinscheduler.common.Constants.ALIAS;
+import static org.apache.dolphinscheduler.common.Constants.CONTENT;
+import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
+import static org.apache.dolphinscheduler.common.Constants.FORMAT_SS;
+import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S;
+import static org.apache.dolphinscheduler.common.Constants.JAR;
+
 import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
 import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
 import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
@@ -32,6 +31,7 @@ import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor
 import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
 import org.apache.dolphinscheduler.api.service.ResourcesService;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.api.utils.RegexUtils;
@@ -56,15 +56,11 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
 import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
 import org.apache.dolphinscheduler.dao.mapper.UserMapper;
 import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
-import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
 import org.apache.dolphinscheduler.spi.enums.ResourceType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.dao.DuplicateKeyException;
-import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
-import org.springframework.web.multipart.MultipartFile;
+
+import org.apache.commons.beanutils.BeanMap;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.rmi.ServerException;
@@ -77,18 +73,24 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.regex.Matcher;
 import java.util.stream.Collectors;
 
-import static org.apache.dolphinscheduler.common.Constants.ALIAS;
-import static org.apache.dolphinscheduler.common.Constants.CONTENT;
-import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
-import static org.apache.dolphinscheduler.common.Constants.FORMAT_SS;
-import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S;
-import static org.apache.dolphinscheduler.common.Constants.JAR;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.DuplicateKeyException;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.base.Joiner;
+import com.google.common.io.Files;
 
 /**
  * resources service impl
@@ -735,98 +737,104 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
         Result<Object> result = new Result<>();
         putMsg(result, Status.SUCCESS);
 
-        if (FileUtils.directoryTraversal(name)) {
-            logger.error("file alias name {} verify failed", name);
-            putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
-            return result;
+        Result<Object> emptyCheck = fileEmptyCheck(result, file);
+        if (!emptyCheck.isSuccess()) {
+            return emptyCheck;
         }
+        fileNameCheck(result, name);
+        fileNameCheck(result, file.getOriginalFilename());
+        String nameSuffix = Files.getFileExtension(name);
+        String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
+        fileSuffixCheck(result, type, name, nameSuffix, fileSuffix);
+        fileSizeCheck(result, file);
+        return result;
+    }
 
-        if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) {
-            logger.error("file original name {} verify failed", file.getOriginalFilename());
-            putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+    private Result<Object> verifyFiles(ResourceType type, List<MultipartFile> files) {
+        Result<Object> result = new Result<>();
+        putMsg(result, Status.SUCCESS);
+        for (MultipartFile file : files) {
+            Result<Object> emptyCheck = fileEmptyCheck(result, file);
+            if (!emptyCheck.isSuccess()) {
+                return result;
+            }
+            fileNameCheck(result, file.getName());
+            fileNameCheck(result, file.getOriginalFilename());
+            String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
+            fileSuffixCheck(result, type, null, null, fileSuffix);
+            fileSizeCheck(result, file);
+        }
+        return result;
+    }
+
+    private Result<Object> fileEmptyCheck(Result<Object> result, MultipartFile file) {
+        if (file == null || file.isEmpty()) {
+            logger.error("file is empty: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
+            putMsg(result, Status.RESOURCE_FILE_IS_EMPTY, file.getOriginalFilename());
             return result;
         }
+        return result;
+    }
 
-        if (file != null) {
-            // file is empty
-            if (file.isEmpty()) {
-                logger.error("file is empty: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
-                putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
+    private Result<Object> fileNameCheck(Result<Object> result, String name) {
+        // file name check
+        if (StringUtils.isEmpty(name) || FileUtils.directoryTraversal(name)) {
+            logger.error("file original name {} verify failed", name);
+            putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED, name);
+            return result;
+        }
+        String restrictedContents = PropertyUtils.getString(Constants.FILE_NAME_RESTRICTED_CONTENT, null);
+        if (restrictedContents != null) {
+            Set<String> containSet = Arrays.stream(restrictedContents.split(Constants.COMMA))
+                    .filter(name::contains).collect(Collectors.toSet());
+            if (!containSet.isEmpty()) {
+                putMsg(result, Status.FILE_NAME_CONTAIN_RESTRICTIONS, name);
                 return result;
             }
+        }
+        return result;
+    }
 
-            // file suffix
-            String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
-            String nameSuffix = Files.getFileExtension(name);
-
-            // determine file suffix
+    private Result<Object> fileSuffixCheck(Result<Object> result, ResourceType type, String fileName,
+                                           String nameSuffix, String fileSuffix) {
+        // // determine file suffix
+        if (StringUtils.isNotEmpty(nameSuffix)) {
             if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
                 // rename file suffix and original suffix must be consistent
                 logger.error("rename file suffix and original suffix must be consistent: {}",
-                        RegexUtils.escapeNRT(file.getOriginalFilename()));
+                        RegexUtils.escapeNRT(fileName));
                 putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
                 return result;
             }
-
-            // If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
-            if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) {
-                logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
-                putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
-                return result;
-            }
-            if (file.getSize() > Constants.MAX_FILE_SIZE) {
-                logger.error("file size is too large: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
-                putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
+        }
+        String fileAllowTypes = PropertyUtils.getString(Constants.FILE_TYPE_RESTRICTED_LIST, null);
+        if (fileAllowTypes != null) {
+            Set<String> containSet = Arrays.stream(fileAllowTypes.split(Constants.COMMA))
+                    .filter(fileSuffix::contains).collect(Collectors.toSet());
+            if (!containSet.isEmpty()) {
+                putMsg(result, Status.FILE_TYPE_IS_RESTRICTIONS, fileName);
                 return result;
             }
         }
+        // If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
+        if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) {
+            logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
+            putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
+            return result;
+        }
         return result;
     }
 
-    private Result<Object> verifyFiles(ResourceType type, List<MultipartFile> files) {
-        Result<Object> result = new Result<>();
-        putMsg(result, Status.SUCCESS);
-        for (MultipartFile file : files) {
-            if (FileUtils.directoryTraversal(file.getName())) {
-                logger.error("file alias name {} verify failed", file.getName());
-                putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
-                return result;
-            }
-
-            if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) {
-                logger.error("file original name {} verify failed", file.getOriginalFilename());
-                putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
-                return result;
-            }
-
-            if (file != null) {
-                // file is empty
-                if (file.isEmpty()) {
-                    logger.error("file is empty: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
-                    putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
-                    return result;
-                }
-
-                // file suffix
-                String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
-
-                // If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
-                if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) {
-                    logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
-                    putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
-                    return result;
-                }
-                if (file.getSize() > Constants.MAX_FILE_SIZE) {
-                    logger.error("file size is too large: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
-                    putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
-                    return result;
-                }
-            }
+    private Result<Object> fileSizeCheck(Result<Object> result, MultipartFile file) {
+        // file size check
+        if (file.getSize() > Constants.MAX_FILE_SIZE) {
+            logger.error("file size is too large: {}", RegexUtils.escapeNRT(file.getOriginalFilename()));
+            putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
+            return result;
         }
         return result;
     }
 
-
     /**
      * query resources list paging
      *

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@@ -154,7 +154,7 @@ public class ResourcesServiceTest {
         PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
         Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
         Mockito.when(tenantMapper.queryById(1)).thenReturn(null);
-        Result result = resourcesService.createBatchResource(user,
+        Result result = resourcesService.createBatchResources(user,
                 ResourceType.FILE, mockMultipartFiles, -1, "/");
         Assert.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getCode(), (long)result.getCode());
 
@@ -166,7 +166,7 @@ public class ResourcesServiceTest {
         MockMultipartFile mockMultipartFile2 = new MockMultipartFile("test1.pdf", "test1.pdf", "pdf", "test1".getBytes());
         mockMultipartFiles.add(mockMultipartFile2);
         PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
-        result = resourcesService.createBatchResource(user,
+        result = resourcesService.createBatchResources(user,
                 ResourceType.FILE, mockMultipartFiles, -1, "/");
         Assert.assertEquals(Status.BATCH_RESOURCE_NAME_REPEAT.getCode(), (long)result.getCode());
 
@@ -174,7 +174,7 @@ public class ResourcesServiceTest {
         mockMultipartFiles.remove(mockMultipartFile2);
         List<String> repeatFileNames = Arrays.asList("test1.pdf");
         Mockito.when(resourcesMapper.existResources(any(List.class), eq(ResourceType.FILE.ordinal()))).thenReturn(repeatFileNames);
-        result = resourcesService.createBatchResource(user,
+        result = resourcesService.createBatchResources(user,
                 ResourceType.FILE, mockMultipartFiles, -1, "/");
         Assert.assertEquals(Status.RESOURCE_EXIST.getCode(), (long)result.getCode());
 
@@ -183,7 +183,7 @@ public class ResourcesServiceTest {
                 "pdf", "test1".getBytes());
         mockMultipartFiles.add(mockMultipartFile3);
         Mockito.when(resourcesMapper.existResources(any(List.class), eq(ResourceType.FILE.ordinal()))).thenReturn(new ArrayList());
-        result = resourcesService.createBatchResource(user,
+        result = resourcesService.createBatchResources(user,
                 ResourceType.FILE, mockMultipartFiles, -1, "/");
         Assert.assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getCode(), (long)result.getCode());
     }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -852,4 +852,10 @@ public final class Constants {
     public static final String TYPE_DATA_QUALITY = "DataQuality";
     public static final String TYPE_OTHER = "Other";
     public static final String TYPE_MACHINE_LEARNING = "MachineLearning";
+
+    /**
+     * file upload verify
+     */
+    public static final String FILE_TYPE_RESTRICTED_LIST = "file.type.restricted.list";
+    public static final String FILE_NAME_RESTRICTED_CONTENT= "file.name.restricted.content";
 }

dolphinscheduler-common/src/main/resources/common.properties

@@ -91,3 +91,7 @@ development.state=false
 # rpc port
 alert.rpc.port=50052
 
+# file verify
+file.type.restricted.list=html,HTML,php,PHP
+file.name.restricted.content=XSS
+