未验证 提交 4bec7921 编写于 作者: J JinYong Li 提交者: GitHub

[Fix][Common] rewrite code generate,fix bit shift (#6914)

* rewrite code generate,fix bit shift

* fix ut

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* fix ut
上级 595e4843
...@@ -26,6 +26,7 @@ header: ...@@ -26,6 +26,7 @@ header:
- LICENSE - LICENSE
- DISCLAIMER - DISCLAIMER
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java - dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java
- mvnw.cmd - mvnw.cmd
- dolphinscheduler-dao/src/main/resources/sql/soft_version - dolphinscheduler-dao/src/main/resources/sql/soft_version
- .mvn - .mvn
......
...@@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license. ...@@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license.
DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0 DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0
DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0 DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0
DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0 DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0
CodeGenerateUtils from https://github.com/twitter-archive/snowflake/tree/snowflake-2010 Apache 2.0
...@@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.api.service.EnvironmentService; ...@@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.api.service.EnvironmentService;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
...@@ -115,9 +115,9 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme ...@@ -115,9 +115,9 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
env.setUpdateTime(new Date()); env.setUpdateTime(new Date());
long code = 0L; long code = 0L;
try { try {
code = SnowFlakeUtils.getInstance().nextId(); code = CodeGenerateUtils.getInstance().genCode();
env.setCode(code); env.setCode(code);
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
logger.error("Environment code get error, ", e); logger.error("Environment code get error, ", e);
} }
if (code == 0L) { if (code == 0L) {
......
...@@ -40,10 +40,10 @@ import org.apache.dolphinscheduler.common.graph.DAG; ...@@ -40,10 +40,10 @@ import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
...@@ -222,8 +222,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro ...@@ -222,8 +222,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} }
long processDefinitionCode; long processDefinitionCode;
try { try {
processDefinitionCode = SnowFlakeUtils.getInstance().nextId(); processDefinitionCode = CodeGenerateUtils.getInstance().genCode();
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
return result; return result;
} }
...@@ -874,8 +874,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro ...@@ -874,8 +874,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
processDefinition.setProjectCode(projectCode); processDefinition.setProjectCode(projectCode);
processDefinition.setUserId(loginUser.getId()); processDefinition.setUserId(loginUser.getId());
try { try {
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId()); processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR); putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR);
return false; return false;
} }
...@@ -894,10 +894,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro ...@@ -894,10 +894,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
taskDefinitionLog.setOperator(loginUser.getId()); taskDefinitionLog.setOperator(loginUser.getId());
taskDefinitionLog.setOperateTime(now); taskDefinitionLog.setOperateTime(now);
try { try {
long code = SnowFlakeUtils.getInstance().nextId(); long code = CodeGenerateUtils.getInstance().genCode();
taskCodeMap.put(taskDefinitionLog.getCode(), code); taskCodeMap.put(taskDefinitionLog.getCode(), code);
taskDefinitionLog.setCode(code); taskDefinitionLog.setCode(code);
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
logger.error("Task code get error, ", e); logger.error("Task code get error, ", e);
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code"); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
return false; return false;
...@@ -1363,8 +1363,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro ...@@ -1363,8 +1363,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
processDefinition.setProjectCode(targetProjectCode); processDefinition.setProjectCode(targetProjectCode);
if (isCopy) { if (isCopy) {
try { try {
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId()); processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS); throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
} }
......
...@@ -25,8 +25,8 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; ...@@ -25,8 +25,8 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser;
...@@ -97,14 +97,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic ...@@ -97,14 +97,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
project = Project project = Project
.newBuilder() .newBuilder()
.name(name) .name(name)
.code(SnowFlakeUtils.getInstance().nextId()) .code(CodeGenerateUtils.getInstance().genCode())
.description(desc) .description(desc)
.userId(loginUser.getId()) .userId(loginUser.getId())
.userName(loginUser.getUserName()) .userName(loginUser.getUserName())
.createTime(now) .createTime(now)
.updateTime(now) .updateTime(now)
.build(); .build();
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
putMsg(result, Status.CREATE_PROJECT_ERROR); putMsg(result, Status.CREATE_PROJECT_ERROR);
return result; return result;
} }
......
...@@ -27,9 +27,9 @@ import org.apache.dolphinscheduler.api.utils.Result; ...@@ -27,9 +27,9 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
...@@ -431,9 +431,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe ...@@ -431,9 +431,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
List<Long> taskCodes = new ArrayList<>(); List<Long> taskCodes = new ArrayList<>();
try { try {
for (int i = 0; i < genNum; i++) { for (int i = 0; i < genNum; i++) {
taskCodes.add(SnowFlakeUtils.getInstance().nextId()); taskCodes.add(CodeGenerateUtils.getInstance().genCode());
} }
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
logger.error("Task code get error, ", e); logger.error("Task code get error, ", e);
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code"); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
} }
......
/** Copyright 2010-2012 Twitter, Inc.*/
package org.apache.dolphinscheduler.common.utils;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Objects;
/**
* Rewriting based on Twitter snowflake algorithm
*/
public class CodeGenerateUtils {
// start timestamp
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
// Each machine generates 32 in the same millisecond
private static final long LOW_DIGIT_BIT = 5L;
private static final long MIDDLE_BIT = 2L;
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
// The displacement to the left
private static final long MIDDLE_LEFT = LOW_DIGIT_BIT;
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT;
private final long machineHash;
private long lowDigit = 0L;
private long recordMillisecond = -1L;
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
private static final long SYSTEM_NANOTIME = System.nanoTime();
private CodeGenerateUtils() throws CodeGenerateException {
try {
this.machineHash = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1));
} catch (UnknownHostException e) {
throw new CodeGenerateException(e.getMessage());
}
}
private static CodeGenerateUtils instance = null;
public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException {
if (instance == null) {
instance = new CodeGenerateUtils();
}
return instance;
}
public synchronized long genCode() throws CodeGenerateException {
long nowtMillisecond = systemMillisecond();
if (nowtMillisecond < recordMillisecond) {
throw new CodeGenerateException("New code exception because time is set back.");
}
if (nowtMillisecond == recordMillisecond) {
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT;
if (lowDigit == 0L) {
while (nowtMillisecond <= recordMillisecond) {
nowtMillisecond = systemMillisecond();
}
}
} else {
lowDigit = 0L;
}
recordMillisecond = nowtMillisecond;
return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit;
}
private long systemMillisecond() {
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
}
public static class CodeGenerateException extends Exception {
public CodeGenerateException(String message) {
super(message);
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Objects;
public class SnowFlakeUtils {
// start timestamp
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
// Each machine generates 32 in the same millisecond
private static final long SEQUENCE_BIT = 5;
private static final long MACHINE_BIT = 2;
private static final long MAX_SEQUENCE = ~(-1L << SEQUENCE_BIT);
// The displacement to the left
private static final long MACHINE_LEFT = SEQUENCE_BIT + MACHINE_BIT;
private static final long TIMESTAMP_LEFT = SEQUENCE_BIT + MACHINE_BIT + MACHINE_LEFT;
private final int machineId;
private long sequence = 0L;
private long lastTimestamp = -1L;
private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
private static final long SYSTEM_NANOTIME = System.nanoTime();
private SnowFlakeUtils() throws SnowFlakeException {
try {
this.machineId = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % 4;
} catch (UnknownHostException e) {
throw new SnowFlakeException(e.getMessage());
}
}
private static SnowFlakeUtils instance = null;
public static synchronized SnowFlakeUtils getInstance() throws SnowFlakeException {
if (instance == null) {
instance = new SnowFlakeUtils();
}
return instance;
}
public synchronized long nextId() throws SnowFlakeException {
long currStmp = nowTimestamp();
if (currStmp < lastTimestamp) {
throw new SnowFlakeException("Clock moved backwards. Refusing to generate id");
}
if (currStmp == lastTimestamp) {
sequence = (sequence + 1) & MAX_SEQUENCE;
if (sequence == 0L) {
currStmp = getNextMill();
}
} else {
sequence = 0L;
}
lastTimestamp = currStmp;
return (currStmp - START_TIMESTAMP) << TIMESTAMP_LEFT
| machineId << MACHINE_LEFT
| sequence;
}
private long getNextMill() {
long mill = nowTimestamp();
while (mill <= lastTimestamp) {
mill = nowTimestamp();
}
return mill;
}
private long nowTimestamp() {
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
}
public static class SnowFlakeException extends Exception {
public SnowFlakeException(String message) {
super(message);
}
}
}
...@@ -22,14 +22,14 @@ import java.util.HashSet; ...@@ -22,14 +22,14 @@ import java.util.HashSet;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
public class SnowFlakeUtilsTest { public class CodeGenerateUtilsTest {
@Test @Test
public void testNoGenerateDuplicateId() throws SnowFlakeUtils.SnowFlakeException { public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException {
HashSet<Long> existsSnowFlakeId = new HashSet<>(); HashSet<Long> existsCode = new HashSet<>();
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
Long currentId = SnowFlakeUtils.getInstance().nextId(); Long currentCode = CodeGenerateUtils.getInstance().genCode();
Assert.assertFalse(existsSnowFlakeId.contains(currentId)); Assert.assertFalse(existsCode.contains(currentCode));
existsSnowFlakeId.add(currentId); existsCode.add(currentCode);
} }
} }
} }
...@@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.dao.upgrade; ...@@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import java.sql.Connection; import java.sql.Connection;
...@@ -110,7 +110,7 @@ public class ProcessDefinitionDao { ...@@ -110,7 +110,7 @@ public class ProcessDefinitionDao {
processDefinition.setId(rs.getInt(1)); processDefinition.setId(rs.getInt(1));
long code = rs.getLong(2); long code = rs.getLong(2);
if (code == 0L) { if (code == 0L) {
code = SnowFlakeUtils.getInstance().nextId(); code = CodeGenerateUtils.getInstance().genCode();
} }
processDefinition.setCode(code); processDefinition.setCode(code);
processDefinition.setVersion(Constants.VERSION_FIRST); processDefinition.setVersion(Constants.VERSION_FIRST);
......
...@@ -17,8 +17,8 @@ ...@@ -17,8 +17,8 @@
package org.apache.dolphinscheduler.dao.upgrade; package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
...@@ -51,7 +51,7 @@ public class ProjectDao { ...@@ -51,7 +51,7 @@ public class ProjectDao {
Integer id = rs.getInt(1); Integer id = rs.getInt(1);
long code = rs.getLong(2); long code = rs.getLong(2);
if (code == 0L) { if (code == 0L) {
code = SnowFlakeUtils.getInstance().nextId(); code = CodeGenerateUtils.getInstance().genCode();
} }
projectMap.put(id, code); projectMap.put(id, code);
} }
......
...@@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType; ...@@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SchemaUtils; import org.apache.dolphinscheduler.common.utils.SchemaUtils;
import org.apache.dolphinscheduler.common.utils.ScriptRunner; import org.apache.dolphinscheduler.common.utils.ScriptRunner;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
...@@ -673,7 +673,7 @@ public abstract class UpgradeDao { ...@@ -673,7 +673,7 @@ public abstract class UpgradeDao {
String name = task.get("name").asText(); String name = task.get("name").asText();
taskDefinitionLog.setName(name); taskDefinitionLog.setName(name);
taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText()); taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText());
long taskCode = SnowFlakeUtils.getInstance().nextId(); long taskCode = CodeGenerateUtils.getInstance().genCode();
taskDefinitionLog.setCode(taskCode); taskDefinitionLog.setCode(taskCode);
taskDefinitionLog.setVersion(Constants.VERSION_FIRST); taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode()); taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());
......
...@@ -402,6 +402,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. ...@@ -402,6 +402,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0
protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0 prometheus client_java(simpleclient) 0.12.0: https://github.com/prometheus/client_java, Apache 2.0
snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0
======================================================================== ========================================================================
BSD licenses BSD licenses
......
Copyright 2010-2012 Twitter, Inc.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
\ No newline at end of file
...@@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.common.enums.RunMode; ...@@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.common.enums.RunMode;
import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue;
...@@ -150,18 +150,18 @@ public class PythonGatewayServer extends SpringBootServletInitializer { ...@@ -150,18 +150,18 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
return taskDefinitionService.genTaskCodeList(genNum); return taskDefinitionService.genTaskCodeList(genNum);
} }
public Map<String, Long> getCodeAndVersion(String projectName, String taskName) throws SnowFlakeUtils.SnowFlakeException { public Map<String, Long> getCodeAndVersion(String projectName, String taskName) throws CodeGenerateUtils.CodeGenerateException {
Project project = projectMapper.queryByName(projectName); Project project = projectMapper.queryByName(projectName);
Map<String, Long> result = new HashMap<>(); Map<String, Long> result = new HashMap<>();
// project do not exists, mean task not exists too, so we should directly return init value // project do not exists, mean task not exists too, so we should directly return init value
if (project == null) { if (project == null) {
result.put("code", SnowFlakeUtils.getInstance().nextId()); result.put("code", CodeGenerateUtils.getInstance().genCode());
result.put("version", 0L); result.put("version", 0L);
return result; return result;
} }
TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName); TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), taskName);
if (taskDefinition == null) { if (taskDefinition == null) {
result.put("code", SnowFlakeUtils.getInstance().nextId()); result.put("code", CodeGenerateUtils.getInstance().genCode());
result.put("version", 0L); result.put("version", 0L);
} else { } else {
result.put("code", taskDefinition.getCode()); result.put("code", taskDefinition.getCode());
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.service.process; package org.apache.dolphinscheduler.service.process;
import static java.util.stream.Collectors.toSet;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
...@@ -27,8 +28,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS ...@@ -27,8 +28,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
import static java.util.stream.Collectors.toSet;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
...@@ -50,11 +49,11 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo; ...@@ -50,11 +49,11 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
...@@ -2290,8 +2289,8 @@ public class ProcessService { ...@@ -2290,8 +2289,8 @@ public class ProcessService {
taskDefinitionLog.setCreateTime(now); taskDefinitionLog.setCreateTime(now);
if (taskDefinitionLog.getCode() == 0) { if (taskDefinitionLog.getCode() == 0) {
try { try {
taskDefinitionLog.setCode(SnowFlakeUtils.getInstance().nextId()); taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
} catch (SnowFlakeException e) { } catch (CodeGenerateException e) {
logger.error("Task code get error, ", e); logger.error("Task code get error, ", e);
return Constants.DEFINITION_FAILURE; return Constants.DEFINITION_FAILURE;
} }
......
Copyright 2010-2012 Twitter, Inc.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册