From 039f02a624e0d3937f8253af3b369ac961ae67fd Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Tue, 17 Dec 2019 19:52:47 +0800 Subject: [PATCH] Add common utils CollectionUtils.java DateUtils.java unit test (#1496) * dateutil test * pom.xml --- .../common/utils/CollectionUtils.java | 42 ++++---- .../common/utils/DateUtils.java | 6 +- .../utils/dependent/DependentDateUtils.java | 2 +- .../common/utils/CollectionUtilsTest.java | 66 ++++++++++-- .../common/utils/DateUtilsTest.java | 102 +++++++++++++++++- .../dao/mapper/UserMapperTest.java | 4 +- .../server/worker/task/sql/SqlTask.java | 4 +- pom.xml | 4 +- 8 files changed, 187 insertions(+), 43 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index e69f9a9f7..9c02111c3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -86,21 +86,20 @@ public class CollectionUtils { * @return string to map */ public static Map stringToMap(String str, String separator, String keyPrefix) { - if (null == str || "".equals(str)) { - return null; + Map emptyMap = new HashMap<>(0); + if (StringUtils.isEmpty(str)) { + return emptyMap; } - if (null == separator || "".equals(separator)) { - return null; + if (StringUtils.isEmpty(separator)) { + return emptyMap; } String[] strings = str.split(separator); - int mapLength = strings.length; - if ((strings.length % 2) != 0) { - mapLength = mapLength + 1; - } - - Map map = new HashMap<>(mapLength); + Map map = new HashMap<>(strings.length); for (int i = 0; i < strings.length; i++) { String[] strArray = strings[i].split("="); + if (strArray.length != 2) { + return emptyMap; + } //strArray[0] KEY strArray[1] VALUE if (StringUtils.isEmpty(keyPrefix)) { map.put(strArray[0], strArray[1]); @@ -146,7 +145,7 @@ public class CollectionUtils { * @param obj the object * @return the maximum frequency of the object */ - public final int max(final Object obj) { + private int max(final Object obj) { return Math.max(freqA(obj), freqB(obj)); } @@ -156,7 +155,7 @@ public class CollectionUtils { * @param obj the object * @return the minimum frequency of the object */ - public final int min(final Object obj) { + private int min(final Object obj) { return Math.min(freqA(obj), freqB(obj)); } @@ -180,10 +179,10 @@ public class CollectionUtils { return getFreq(obj, cardinalityB); } - private final int getFreq(final Object obj, final Map freqMap) { + private int getFreq(final Object obj, final Map freqMap) { final Integer count = freqMap.get(obj); if (count != null) { - return count.intValue(); + return count; } return 0; } @@ -203,7 +202,7 @@ public class CollectionUtils { return true; } - if ((a == null && b != null) || a != null && b == null) { + if (a == null || b == null) { return false; } @@ -253,12 +252,7 @@ public class CollectionUtils { public static Map getCardinalityMap(final Iterable coll) { final Map count = new HashMap(); for (final O obj : coll) { - final Integer c = count.get(obj); - if (c == null) { - count.put(obj, Integer.valueOf(1)); - } else { - count.put(obj, Integer.valueOf(c.intValue() + 1)); - } + count.put(obj, count.getOrDefault(obj, 0) + 1); } return count; } @@ -273,6 +267,12 @@ public class CollectionUtils { */ public static List> getListByExclusion(List originList, Set exclusionSet) { List> instanceList = new ArrayList<>(); + if (exclusionSet == null) { + exclusionSet = new HashSet<>(); + } + if (originList == null) { + return instanceList; + } Map instanceMap; for (T instance : originList) { Map dataMap = new BeanMap(instance); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 0f30e7fbe..3455d5344 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -291,14 +291,14 @@ public class DateUtils { * get some hour of day * * @param date date - * @param hours hours + * @param offsetHour hours * @return some hour of day * */ - public static Date getSomeHourOfDay(Date date, int hours) { + public static Date getSomeHourOfDay(Date date, int offsetHour) { Calendar cal = Calendar.getInstance(); cal.setTime(date); - cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) - hours); + cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/dependent/DependentDateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/dependent/DependentDateUtils.java index 574343d0c..103e75fb6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/dependent/DependentDateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/dependent/DependentDateUtils.java @@ -34,7 +34,7 @@ public class DependentDateUtils { public static List getLastHoursInterval(Date businessDate, int hourNumber){ List dateIntervals = new ArrayList<>(); for(int index = hourNumber; index > 0; index--){ - Date lastHour = DateUtils.getSomeHourOfDay(businessDate, index); + Date lastHour = DateUtils.getSomeHourOfDay(businessDate, -index); Date beginTime = DateUtils.getStartOfHour(lastHour); Date endTime = DateUtils.getEndOfHour(lastHour); dateIntervals.add(new DateInterval(beginTime, endTime)); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java index 30c11522b..7321879ab 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.junit.Assert; import org.junit.Test; @@ -26,19 +27,26 @@ public class CollectionUtilsTest { @Test public void equalLists() { + Assert.assertTrue(CollectionUtils.equalLists(null,null)); + Assert.assertTrue(CollectionUtils.equalLists(new ArrayList(),new ArrayList())); List a = new ArrayList(); a.add(1); a.add(2); - a.add(3); List b = new ArrayList(); - b.add(3); + b.add(1); + b.add(2); + Assert.assertTrue(CollectionUtils.equalLists(a, b)); + a.add(1); + Assert.assertFalse(CollectionUtils.equalLists(a, b)); b.add(2); + Assert.assertFalse(CollectionUtils.equalLists(a, b)); + a.add(2); b.add(1); - Assert.assertTrue(CollectionUtils.equalLists(a,b)); - Assert.assertTrue(CollectionUtils.equalLists(null,null)); - List c = new ArrayList(); - Assert.assertFalse(CollectionUtils.equalLists(c,null)); - Assert.assertFalse(CollectionUtils.equalLists(c,a)); + a.add(4); + b.add(2); + Assert.assertFalse(CollectionUtils.equalLists(a, b)); + Assert.assertFalse(CollectionUtils.equalLists(null, new ArrayList())); + Assert.assertFalse(CollectionUtils.equalLists(new ArrayList(), null)); } @Test @@ -56,7 +64,49 @@ public class CollectionUtilsTest { @Test public void stringToMap() { - Map a = CollectionUtils.stringToMap("a=b;c=d", ";", ""); + Map a = CollectionUtils.stringToMap("a=b;c=d;", ";"); Assert.assertNotNull(a); + Assert.assertTrue(a.size() == 2); + a = CollectionUtils.stringToMap(null, ";"); + Assert.assertTrue(a.isEmpty()); + a = CollectionUtils.stringToMap("", ";"); + Assert.assertTrue(a.isEmpty()); + a = CollectionUtils.stringToMap("a=b;c=d", ""); + Assert.assertTrue(a.isEmpty()); + a = CollectionUtils.stringToMap("a=b;c=d", null); + Assert.assertTrue(a.isEmpty()); + a = CollectionUtils.stringToMap("a=b;c=d;e=f", ";"); + Assert.assertEquals(a.size(), 3); + a = CollectionUtils.stringToMap("a;b=f", ";"); + Assert.assertTrue(a.isEmpty()); + a = CollectionUtils.stringToMap("a=b;c=d;e=f;", ";", "test"); + Assert.assertEquals(a.size(), 3); + Assert.assertNotNull(a.get("testa")); } + + @Test + public void getListByExclusion() { + Assert.assertNotNull(CollectionUtils.getListByExclusion(null, null)); + List originList = new ArrayList<>(); + originList.add(1); + originList.add(2); + List> ret = CollectionUtils.getListByExclusion(originList, null); + Assert.assertEquals(ret.size(), 2); + ret = CollectionUtils.getListByExclusion(originList, new HashSet<>()); + Assert.assertEquals(ret.size(), 2); + Assert.assertFalse(ret.get(0).isEmpty()); + Set exclusion = new HashSet<>(); + exclusion.add(Constants.CLASS); + ret = CollectionUtils.getListByExclusion(originList, exclusion); + Assert.assertEquals(ret.size(), 2); + Assert.assertTrue(ret.get(0).isEmpty()); + } + + @Test + public void isNotEmpty() { + List list = new ArrayList<>(); + Assert.assertFalse(CollectionUtils.isNotEmpty(list)); + Assert.assertFalse(CollectionUtils.isNotEmpty(null)); + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java index bcaa39104..6800f6b54 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; - import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; public class DateUtilsTest { - @Test public void format2Readable() throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -54,4 +52,102 @@ public class DateUtilsTest { Assert.assertEquals(sunday, sunday1); } -} \ No newline at end of file + + @Test + public void diffHours(){ + Date d1 = DateUtils.stringToDate("2019-01-28 00:00:00"); + Date d2 = DateUtils.stringToDate("2019-01-28 20:00:00"); + Assert.assertEquals(DateUtils.diffHours(d1, d2), 20); + Date d3 = DateUtils.stringToDate("2019-01-28 20:00:00"); + Assert.assertEquals(DateUtils.diffHours(d3, d2), 0); + Assert.assertEquals(DateUtils.diffHours(d2, d1), 20); + Date d4 = null; + Assert.assertEquals(DateUtils.diffHours(d2, d4), 0); + } + + @Test + public void dateToString() { + Date d1 = DateUtils.stringToDate("2019-01-28"); + Assert.assertNull(d1); + d1 = DateUtils.stringToDate("2019-01-28 00:00:00"); + Assert.assertEquals(DateUtils.dateToString(d1), "2019-01-28 00:00:00"); + } + + @Test + public void getSomeDay() { + Date d1 = DateUtils.stringToDate("2019-01-31 00:00:00"); + Date curr = DateUtils.getSomeDay(d1, 1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-02-01 00:00:00"); + Assert.assertEquals(DateUtils.dateToString(DateUtils.getSomeDay(d1, -31)), "2018-12-31 00:00:00"); + } + + @Test + public void getFirstDayOfMonth() { + Date d1 = DateUtils.stringToDate("2019-01-31 00:00:00"); + Date curr = DateUtils.getFirstDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-01 00:00:00"); + + d1 = DateUtils.stringToDate("2019-01-31 01:59:00"); + curr = DateUtils.getFirstDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-01 01:59:00"); + } + + @Test + public void getSomeHourOfDay() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:59:59"); + Date curr = DateUtils.getSomeHourOfDay(d1, -1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 10:00:00"); + curr = DateUtils.getSomeHourOfDay(d1, 0); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:00:00"); + curr = DateUtils.getSomeHourOfDay(d1, 2); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 13:00:00"); + curr = DateUtils.getSomeHourOfDay(d1, 24); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-02-01 11:00:00"); + } + + @Test + public void getLastDayOfMonth() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:59:59"); + Date curr = DateUtils.getLastDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:59:59"); + d1 = DateUtils.stringToDate("2019-01-02 11:59:59"); + curr = DateUtils.getLastDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:59:59"); + + d1 = DateUtils.stringToDate("2019-02-02 11:59:59"); + curr = DateUtils.getLastDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-02-28 11:59:59"); + + d1 = DateUtils.stringToDate("2020-02-02 11:59:59"); + curr = DateUtils.getLastDayOfMonth(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2020-02-29 11:59:59"); + } + + @Test + public void getStartOfDay() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:59:59"); + Date curr = DateUtils.getStartOfDay(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 00:00:00"); + } + + @Test + public void getEndOfDay() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:00:59"); + Date curr = DateUtils.getEndOfDay(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 23:59:59"); + } + + @Test + public void getStartOfHour() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:00:59"); + Date curr = DateUtils.getStartOfHour(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:00:00"); + } + + @Test + public void getEndOfHour() { + Date d1 = DateUtils.stringToDate("2019-01-31 11:00:59"); + Date curr = DateUtils.getEndOfHour(d1); + Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:59:59"); + } +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java index a4b8618bf..da17e1404 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java @@ -154,7 +154,7 @@ public class UserMapperTest { accessToken.setToken("secrettoken"); accessToken.setCreateTime(new Date()); accessToken.setUpdateTime(new Date()); - accessToken.setExpireTime(DateUtils.getSomeHourOfDay(new Date(),-1)); + accessToken.setExpireTime(DateUtils.getSomeHourOfDay(new Date(),1)); accessTokenMapper.insert(accessToken); return accessToken; } @@ -356,4 +356,4 @@ public class UserMapperTest { accessTokenMapper.deleteById(accessToken.getId()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index f0478ac74..ccfee2efe 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -261,9 +261,7 @@ public class SqlTask extends AbstractTask { Map connParamMap = CollectionUtils.stringToMap(sqlParameters.getConnParams(), SEMICOLON, HIVE_CONF); - if(connParamMap != null){ - paramProp.putAll(connParamMap); - } + paramProp.putAll(connParamMap); connection = DriverManager.getConnection(baseDataSource.getJdbcUrl(), paramProp); diff --git a/pom.xml b/pom.xml index 61c229762..252446157 100644 --- a/pom.xml +++ b/pom.xml @@ -612,10 +612,10 @@ ${maven-surefire-plugin.version} + **/common/utils/*.java + **/common/graph/*.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java - **/common/graph/*.java - **/*CollectionUtilsTest.java -- GitLab