diff --git a/.github/workflows/SonarCloud.yml b/.github/workflows/SonarCloud.yml deleted file mode 100644 index fd5b0585db9554cdc8a4c92f7836591e86e9ed88..0000000000000000000000000000000000000000 --- a/.github/workflows/SonarCloud.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: SonarCloud -on: [push, pull_request] -jobs: - sonarCloudTrigger: - name: SonarCloud Trigger - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 - with: - java-version: 8 - - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube - restore-keys: | - ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube - ${{ runner.os }}-maven- - - name: Maven clean - run: mvn clean - - name: Run SonarCloud analyse - run: > - mvn clean --batch-mode - org.jacoco:jacoco-maven-plugin:prepare-agent - verify - org.sonarsource.scanner.maven:sonar-maven-plugin:sonar - -Dmaven.test.skip=true - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.organization=apache - -Dsonar.projectKey=apache-dolphinscheduler - -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index c415d35619e8ae13b0efd946fc8711bd502fa00c..f76c403a8c4041fb106c8ecd450ccf5ad0d2a1e9 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -49,6 +49,19 @@ jobs: export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -Dmaven.test.skip=false cobertura:cobertura CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Run SonarCloud analysis + run: > + mvn clean --batch-mode + org.jacoco:jacoco-maven-plugin:prepare-agent + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - name: Collect logs run: | mkdir -p ${LOG_DIR} diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 1bfac571bae1eccfcd174cbc2e36228f12cb0968..10cede7988f67dbbb0066fcef22bfa94d755eca5 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -330,11 +330,6 @@ - - org.apache.commons - commons-lang3 - - org.postgresql postgresql diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java index c2e1f86d30949d119c68566f303cf586d7d990ce..dfcda991418fa1d8a03703b21aaf4004b60a27c2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java index 8783c8ebbecf5c9729838005828f6e074cdc2a7f..e6d7b94b51c96d2a977355d253a2c52ec70149fa 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java index fc1be356f5be9133a3fba697c134f7dc698f3ef2..2970354e6bc590bdcb6871cbd1791a92c2026d52 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,9 +29,6 @@ public class HiveDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); - - - /** * gets the JDBC url for the data source connection * @return @@ -49,8 +46,6 @@ public class HiveDataSource extends BaseDataSource { jdbcUrl += ";principal=" + getPrincipal(); } - - if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java index 0e850ee2de8ab84700b611b308b635b3f3e1ef7e..f50de0dee38b79513c67931a466306c1b2325544 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java index 67c035d5be623a9b56ffb8f6c34c4c84e38e03ea..ddc30d939abd7e29109821194cd08535d3539b48 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java index 332a9cb9369227e24920a828c178e912be3d03fb..ecc29c73e8ebb7d866fb70592d4df59ba83c6ec4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java index 084b10d42538fb5b29da01a8224a6d43cb292547..7a51ff7a45406c51ad2ffa473efa64a054da1804 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java index a449e398c676b0c380d319d26e7dad8e66eb6573..638664c47d179156b74e20c34e9140e734f674b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 076966ef72cba64f6f8bd46bb6899eb3ac75b6ca..40efd0a24f12d9d6d3e777dba51d72ccf7748a82 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -26,7 +26,7 @@ import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import java.io.IOException; import java.util.List; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 1203cbb26b36bde1662f94bf7373bb0167943b8d..842c74edc09d5b176c58c5dfb817ac0ac682062d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java index dd4d176b3a6e6c9160deff4f81d9013be33eb82a..c153ec817a574998f9b30999dca0b72b874eeb54 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; /** * encryption utils diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..dc2e3a0750f13502959d440138f3dc1047a2b2dc --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + + + +public class EnumUtils { + + public static > E getEnum(final Class enumClass, final String enumName) { + if (enumName == null) { + return null; + } + try { + return Enum.valueOf(enumClass, enumName); + } catch (final IllegalArgumentException ex) { + return null; + } + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6a10a87b513c925ef869f92f63034173de884d5a..072266fb7ff666c7b26c00fcf0b8cf317df3cf01 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -23,7 +23,6 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 20336afd676f28df393362dca6cb070bf2c27a49..0c061fc0ba1e5d272124e7e07f92204431c37aa4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; import org.apache.commons.configuration.Configuration; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import oshi.SystemInfo; @@ -180,7 +179,7 @@ public class OSUtils { private static List getUserListFromMac() throws IOException { String result = exeCmd("dscl . list /users"); if (StringUtils.isNotEmpty(result)) { - return Arrays.asList(StringUtils.split(result, "\n")); + return Arrays.asList(result.split( "\n")); } return Collections.emptyList(); @@ -251,9 +250,8 @@ public class OSUtils { */ public static String getGroup() throws IOException { String result = exeCmd("groups"); - if (StringUtils.isNotEmpty(result)) { - String[] groupInfo = StringUtils.split(result); + String[] groupInfo = result.split(" "); return groupInfo[0]; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java index cd3a2dfc21b328963f9254dbf474807dfd6484e7..a4e16ba8b60c2c4adada424255854518902384f1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..64dd4f3dcdd93d943c0efa00d99180b0834983d8 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + + +public class StringUtils { + + public static final int INDEX_NOT_FOUND = -1; + + public static final String EMPTY = ""; + + public static boolean isEmpty(final CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static boolean isNotEmpty(final CharSequence cs) { + return !isEmpty(cs); + } + + public static boolean isBlank(CharSequence cs){ + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } + + public static boolean isNotBlank(CharSequence str){ + return !isBlank(str); + } + + public static String substringBefore(final String str, final String separator) { + if (isBlank(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringAfter(final String str, final String separator) { + if (isBlank(str)) { + return str; + } + if (separator == null) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return EMPTY; + } + return str.substring(pos + separator.length()); + } + + public static String substringAfterLast(final String str, final String separator) { + if (isEmpty(str)) { + return str; + } + if (isEmpty(separator)) { + return EMPTY; + } + final int pos = str.lastIndexOf(separator); + if (pos == INDEX_NOT_FOUND || pos == str.length() - separator.length()) { + return EMPTY; + } + return str.substring(pos + separator.length()); + } + + public static String getUtf8String(byte[] bytes){ + return new String(bytes, StandardCharsets.UTF_8); + } + + public static byte[] getUtf8Bytes(String str){ + return str.getBytes(StandardCharsets.UTF_8); + } + + public static boolean hasChinese(String str) { + if (str == null) { + return false; + } + Pattern pattern = Pattern.compile("[\\u4E00-\\u9FBF]+"); + return pattern.matcher(str).find(); + } + + public static boolean hasSpace(String str) { + if (str == null) { + return false; + } + int len = str.length(); + for (int i = 0; i < len; i++) { + if (str.charAt(i) == ' ') { + return true; + } + } + return false; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 757d205deaf1399ba0590fd344ed15097997d198..28e2593359fb9e26f1aa203a42fee08b4c07d9d2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.commons.lang3.EnumUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index c3ba7182708482118b8edd3367788c40e7a7e796..b03b88ae6530da0114ad29896a3d0e311ec94721 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -28,7 +28,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.locks.InterProcessMutex; @@ -39,6 +38,7 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..52c8031e7518bd571197b0aea550837b33b1d034 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.threadutils; + +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.concurrent.*; + +import static org.junit.Assert.*; + + +public class ThreadUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class); + /** + * create a naming thread + */ + @Test + public void testNewDaemonFixedThreadExecutor() { + // create core size and max size are all 3 + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10); + + for (int i = 0; i < 19; i++) { + final int index = i; + testExec.submit(() -> { + System.out.println("do some work index " + index); + }); + } + assertFalse(testExec.isShutdown()); + testExec.shutdownNow(); + assertTrue(testExec.isShutdown()); + + } + + /** + * test schedulerThreadExecutor as for print time in scheduler + * default check thread is 1 + */ + @Test + public void testNewDaemonScheduleThreadExecutor() { + + ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1); + Calendar start = Calendar.getInstance(); + Calendar globalTimer = Calendar.getInstance(); + globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); + // current + Calendar end = Calendar.getInstance(); + end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3); + Runnable schedulerTask = new Runnable() { + @Override + public void run() { + start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); + int index = 0; + // send heart beat work + while (start.getTime().getTime() <= end.getTime().getTime()) { + System.out.println("worker here"); + System.out.println(index ++); + start.add(Calendar.SECOND, 1); + globalTimer.add(Calendar.SECOND, 1); + } + System.out.println("time is " + System.currentTimeMillis()); + } + }; + scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS); + assertFalse(scheduleService.isShutdown()); + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + scheduleService.shutdownNow(); + assertTrue(scheduleService.isShutdown()); + } + + /** + * test stopper is working normal + */ + @Test + public void testStopper() { + assertTrue(Stopper.isRunning()); + Stopper.stop(); + assertTrue(Stopper.isStoped()); + } + + /** + * test threadPoolExecutors with 3 workers and current each 5 tasks + * @throws InterruptedException + */ + @Test + public void testThreadInfo() throws InterruptedException { + ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3); + for (int i = 0; i < 5; ++i ) { + int index = i; + workers.execute(() -> { + for (int j = 0; j < 10; ++j) { + try { + Thread.sleep(1000); + System.out.printf("worker %d is doing the task", index); + System.out.println(); + workers.printStatus(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + workers.submit(() -> { + for (int j = 0; j < 10; ++j) { + try { + Thread.sleep(1000); + System.out.printf("worker_2 %d is doing the task", index); + System.out.println(); + workers.printStatus(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + } + Thread.sleep(50001); + workers.shutdown(); + } + + /** + * test a single daemon thread pool + */ + @Test + public void testNewDaemonSingleThreadExecutor() { + ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); + threadTest.execute(() -> { + for (int i = 0; i < 100; ++i) { + System.out.println("daemon working "); + } + + }); + assertFalse(threadTest.isShutdown()); + threadTest.shutdownNow(); + assertTrue(threadTest.isShutdown()); + } + + @Test + public void testNewDaemonCachedThreadPool() { + + ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); + Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { + for (int i = 0; i < 10; ++i) { + System.out.println("this task is with index " + i ); + } + }); + assertTrue(thread1.getName().startsWith("threadTest-")); + assertFalse(threadPoolExecutor.isShutdown()); + threadPoolExecutor.shutdown(); + assertTrue(threadPoolExecutor.isShutdown()); + } + + @Test + public void testNewDaemonCachedThreadPoolWithThreadNumber() { + ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); + for (int i = 0; i < 10; ++ i) { + threadPoolExecutor.getThreadFactory().newThread(() -> { + assertEquals(3, threadPoolExecutor.getActiveCount()); + System.out.println("this task is first work to do"); + }); + } + assertFalse(threadPoolExecutor.isShutdown()); + threadPoolExecutor.shutdown(); + assertTrue(threadPoolExecutor.isShutdown()); + } + + + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java index b72e7b7d6418cb2bb4722713476d91a56e0ede4d..fc7f65930cf688d00b86973b35a6feefab68cf2b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 85467e12d485af0e38bed7997ff4e81ea357d9ae..af1cdb8b7897ae1cbb959c14ff5f29ea9859375c 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -162,12 +162,6 @@ spring-test test - - io.swagger - swagger-annotations - 1.5.20 - compile - org.yaml snakeyaml diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index c4df0679efb7999529db80bfcce1c41cc9180122..b6def18922adf8ee852118cff9eda80767ca5013 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -22,8 +22,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; @@ -33,7 +31,6 @@ import java.util.Date; */ @Data @TableName("t_ds_user") -@ApiModel(description = "UserModelDesc") public class User { /** @@ -45,13 +42,11 @@ public class User { /** * user name */ - @ApiModelProperty(name = "userName", notes = "USER_NAME",dataType = "String",required = true) private String userName; /** * user password */ - @ApiModelProperty(name = "userPassword", notes = "USER_PASSWORD",dataType = "String",required = true) private String userPassword; /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java index 79b25dd5fa524b9c2463532f71de04865db83647..c826236239d7d47440e6fb81edfc0a058c4b57a6 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DatasourceUser; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -27,88 +28,116 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; +import java.util.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +/** + * datasource mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class DataSourceMapperTest { + /** + * datasource mapper + */ @Autowired DataSourceMapper dataSourceMapper; + /** + * datasource user relation mapper + */ @Autowired DataSourceUserMapper dataSourceUserMapper; /** - * insert - * @return DataSource + * test insert */ - private DataSource insertOne(){ - //insertOne - DataSource dataSource = new DataSource(); - dataSource.setUserId(4); - dataSource.setName("data source test"); - dataSource.setType(DbType.MYSQL); - dataSource.setNote("mysql test"); - dataSource.setConnectionParams("hello mysql"); - dataSource.setUpdateTime(new Date()); - dataSource.setCreateTime(new Date()); - dataSourceMapper.insert(dataSource); - return dataSource; + @Test + public void testInsert(){ + DataSource dataSource = createDataSource(); + assertNotNull(dataSource.getId()); + assertThat(dataSource.getId(), greaterThan(0)); } /** - * test update + * test query */ @Test - public void testUpdate(){ - //insertOne - DataSource dataSource = insertOne(); - //update - dataSource.setUpdateTime(new Date()); - int update = dataSourceMapper.updateById(dataSource); - Assert.assertEquals(update, 1); - dataSourceMapper.deleteById(dataSource.getId()); + public void testSelectById() { + DataSource expectedDataSource = createDataSource(); + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + assertEquals(expectedDataSource, actualDataSource); } + /** - * test delete + * test query */ @Test - public void testDelete(){ + public void testUpdate() { + DataSource expectedDataSource = createDataSource(); + + expectedDataSource.setName("modify " + expectedDataSource.getName()); + expectedDataSource.setNote("modifiy " + expectedDataSource.getNote()); + expectedDataSource.setUserId(2); + expectedDataSource.setType(DbType.HIVE); + expectedDataSource.setConnectionParams("modify " + expectedDataSource.getConnectionParams()); + expectedDataSource.setUpdateTime(DateUtils.getCurrentDate()); - DataSource dataSource = insertOne(); - int delete = dataSourceMapper.deleteById(dataSource.getId()); - Assert.assertEquals(delete, 1); + dataSourceMapper.updateById(expectedDataSource); + + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + + assertEquals(expectedDataSource, actualDataSource); } + /** - * test query + * test delete */ @Test - public void testQuery() { - DataSource dataSource = insertOne(); - //query - List dataSources = dataSourceMapper.selectList(null); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + public void testDelete(){ + DataSource expectedDataSource = createDataSource(); + + dataSourceMapper.deleteById(expectedDataSource.getId()); + + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + + assertNull(actualDataSource); } + + /** * test query datasource by type */ @Test public void testQueryDataSourceByType() { - DataSource dataSource = insertOne(); - //query - List dataSources = dataSourceMapper.queryDataSourceByType( - 0, DbType.MYSQL.ordinal() - ); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + Integer userId = 1; + + Map datasourceMap = createDataSourceMap(userId, "test"); + + List actualDataSources = dataSourceMapper.queryDataSourceByType( + 0, DbType.MYSQL.ordinal()); + + assertThat(actualDataSources.size(), greaterThanOrEqualTo(2)); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = datasourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -116,12 +145,23 @@ public class DataSourceMapperTest { */ @Test public void testSelectPaging() { - DataSource dataSource = insertOne(); - Page page = new Page(1, 3); - IPage dataSourceIPage = dataSourceMapper.selectPaging(page, - 4, null); - Assert.assertNotEquals(dataSourceIPage.getTotal(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + Integer userId = 1; + + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + Page page = new Page(0, 4); + + IPage dataSourceIPage = dataSourceMapper.selectPaging(page, userId, name); + List actualDataSources = dataSourceIPage.getRecords(); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -129,10 +169,17 @@ public class DataSourceMapperTest { */ @Test public void testQueryDataSourceByName() { - DataSource dataSource = insertOne(); - List dataSources = dataSourceMapper.queryDataSourceByName("data source test"); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + DataSource expectedDataSource = createDataSource(name); + + List actualDataSources = dataSourceMapper.queryDataSourceByName(name); + + for (DataSource actualDataSource : actualDataSources){ + if (expectedDataSource.getId() == actualDataSource.getId()){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -140,17 +187,20 @@ public class DataSourceMapperTest { */ @Test public void testQueryAuthedDatasource() { + String name = "test"; + Integer userId = 1; - DataSource dataSource = insertOne(); - DatasourceUser datasourceUser = new DatasourceUser(); - datasourceUser.setUserId(3); - datasourceUser.setDatasourceId(dataSource.getId()); - dataSourceUserMapper.insert(datasourceUser); + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + List actualDataSources = dataSourceMapper.queryAuthedDatasource(userId); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } - List dataSources = dataSourceMapper.queryAuthedDatasource(3); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); - dataSourceUserMapper.deleteById(datasourceUser.getId()); } /** @@ -158,10 +208,19 @@ public class DataSourceMapperTest { */ @Test public void testQueryDatasourceExceptUserId() { - DataSource dataSource = insertOne(); - List dataSources = dataSourceMapper.queryDatasourceExceptUserId(3); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + Integer userId = 1; + + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + List actualDataSources = dataSourceMapper.queryDatasourceExceptUserId(userId); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } } /** @@ -169,13 +228,107 @@ public class DataSourceMapperTest { */ @Test public void testListAllDataSourceByType() { + Integer count = 10; + + Map expectedDataSourceMap = createDataSourceMap(count); + + List actualDataSources = dataSourceMapper.listAllDataSourceByType(DbType.MYSQL.ordinal()); + + assertThat(actualDataSources.size(), greaterThanOrEqualTo(count)); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } + + /** + * create datasource relation + * @param userId + */ + private Map createDataSourceMap(Integer userId,String name){ + + Map dataSourceMap = new HashMap<>(); + + DataSource dataSource = createDataSource(userId, name); + + dataSourceMap.put(dataSource.getId(),dataSource); + + DataSource otherDataSource = createDataSource(userId + 1,name); + + DatasourceUser datasourceUser = new DatasourceUser(); + + datasourceUser.setDatasourceId(otherDataSource.getId()); + datasourceUser.setUserId(userId); + datasourceUser.setPerm(7); + datasourceUser.setCreateTime(DateUtils.getCurrentDate()); + datasourceUser.setUpdateTime(DateUtils.getCurrentDate()); + + dataSourceUserMapper.insert(datasourceUser); + + dataSourceMap.put(otherDataSource.getId(), otherDataSource); + + return dataSourceMap; + } + + /** + * create datasource map + * @param count datasource count + * @return datasource map + */ + private Map createDataSourceMap(Integer count){ + Map dataSourceMap = new HashMap<>(); - DataSource dataSource = insertOne(); + for (int i = 0 ; i < count ;i++){ + DataSource dataSource = createDataSource("test"); + dataSourceMap.put(dataSource.getId(),dataSource); + } - List dataSources = dataSourceMapper.queryDataSourceByType(4, DbType.MYSQL.ordinal()); - Assert.assertNotEquals(dataSources.size(), 0); - List dataSources2 = dataSourceMapper.queryDataSourceByType(10091, DbType.MYSQL.ordinal()); - Assert.assertEquals(dataSources2.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + return dataSourceMap; } + + + /** + * create datasource + * @return datasource + */ + private DataSource createDataSource(){ + return createDataSource(1,"test"); + } + + + /** + * create datasource + * @param name name + * @return datasource + */ + private DataSource createDataSource(String name){ + return createDataSource(1,name); + } + + /** + * create datasource + * @param userId userId + * @param name name + * @return datasource + */ + private DataSource createDataSource(Integer userId,String name){ + Random random = new Random(); + DataSource dataSource = new DataSource(); + dataSource.setUserId(userId); + dataSource.setName(name); + dataSource.setType(DbType.MYSQL); + dataSource.setNote("mysql test"); + dataSource.setConnectionParams("hello mysql"); + dataSource.setUpdateTime(DateUtils.getCurrentDate()); + dataSource.setCreateTime(DateUtils.getCurrentDate()); + + dataSourceMapper.insert(dataSource); + + return dataSource; + } + + } \ No newline at end of file