Skip to content

DolphinScheduler
DolphinScheduler

apache / DolphinScheduler
上一次同步 1 年多

通知 706
Star 9572
Fork 3514
DolphinScheduler
DolphinScheduler

体验新版 GitCode，发现更多精彩内容 >>

提交 7e815b44 编写于 作者: L lgcareer 提交者： qiaozhanwei
浏览文件
操作

change escheduler to dolphinscheduler (#963) 

* rename from DatasourceUserMapper to DataSourceUserMapper

* add unit test in UserMapper and WorkerGroupMapper

* change cn.escheduler to org.apache.dolphinscheduler

* add unit test in UdfFuncMapperTest

* add unit test in UdfFuncMapperTest

* remove DatabaseConfiguration

* add ConnectionFactoryTest

* cal duration in processInstancesList

* change desc to description

* change table name in mysql ddl

* change table name in mysql ddl

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler

* change escheduler to dolphinscheduler
上级 ed04835c
隐藏空白更改
内联 并排
Showing with 36 addition and 38 deletion
dolphinscheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl
浏览文件 @ 7e815b44
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> easyscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; }</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>
\ No newline at end of file
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; }</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>
\ No newline at end of file
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/DruidStatViewServlet.java
浏览文件 @ 7e815b44
......@@ -25,7 +25,7 @@ import com.alibaba.druid.support.http.StatViewServlet;
// @WebInitParam(name="allow",value="127.0.0.1"),
// @WebInitParam(name="deny",value="192.168.16.111"),
@WebInitParam(name="loginUsername",value="admin"),
@WebInitParam(name="loginPassword",value="escheduler123"),
@WebInitParam(name="loginPassword",value="dolphinscheduler123"),
@WebInitParam(name="resetEnable",value="true")
}) */
public class DruidStatViewServlet extends StatViewServlet {
......
dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties
浏览文件 @ 7e815b44
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory，support s3，for example : s3a://escheduler
# to the conf directory，support s3，for example : s3a://dolphinscheduler
fs.defaultFS=hdfs://mycluster:8020
# s3 need，s3 endpoint
......
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
浏览文件 @ 7e815b44
......@@ -47,7 +47,7 @@ public class HadoopUtilsTest {
@Test
public void readFileTest(){
try {
byte[] bytes = HadoopUtils.getInstance().catFile("/escheduler/hdfs/resources/35435.sh");
byte[] bytes = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/35435.sh");
logger.info("------------------start");
logger.info(new String(bytes));
logger.info("---------------------end");
......@@ -84,7 +84,7 @@ public class HadoopUtilsTest {
@Test
public void catFileTest()throws Exception{
List<String> stringList = HadoopUtils.getInstance().catFile("/escheduler/hdfs/resources/WCSparkPython.py", 0, 1000);
List<String> stringList = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000);
logger.info(String.join(",",stringList));
}
}
\ No newline at end of file
dolphinscheduler-common/src/test/resources/dao/data_source.properties
浏览文件 @ 7e815b44
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java
浏览文件 @ 7e815b44
......@@ -31,7 +31,7 @@ public interface CommandMapper extends BaseMapper<Command> {
@Select("select * from t_escheduler_command ${ew.customSqlSegment}")
@Select("select * from t_ds_command ${ew.customSqlSegment}")
List<Command> getAll(@Param(Constants.WRAPPER) Wrapper wrapper);
Command getOneToRun();
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
浏览文件 @ 7e815b44
......@@ -52,7 +52,7 @@ public class DolphinSchedulerManager {
}
public void initDolphinScheduler() {
// Determines whether the escheduler table structure has been init
// Determines whether the dolphinscheduler table structure has been init
if(upgradeDao.isExistsTable("t_escheduler_version") || upgradeDao.isExistsTable("t_escheduler_queue")) {
logger.info("The database has been initialized. Skip the initialization step");
return;
......
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
浏览文件 @ 7e815b44
......@@ -95,10 +95,10 @@ public abstract class UpgradeDao extends AbstractBaseDao {
public void initSchema(String initSqlPath) {
// Execute the escheduler DDL, it cannot be rolled back
// Execute the dolphinscheduler DDL, it cannot be rolled back
runInitDDL(initSqlPath);
// Execute the escheduler DML, it can be rolled back
// Execute the dolphinscheduler DML, it can be rolled back
runInitDML(initSqlPath);
}
......@@ -108,14 +108,12 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (StringUtils.isEmpty(rootDir)) {
throw new RuntimeException("Environment variable user.dir not found");
}
//String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql";
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql";
try {
conn = ConnectionFactory.getDataSource().getConnection();
conn.setAutoCommit(false);
// 执行escheduler_dml.sql脚本，导入escheduler相关的数据
// Execute the ark_manager_dml.sql script to import the data related to escheduler
// Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
initScriptRunner.runScript(initSqlReader);
......@@ -153,7 +151,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql";
try {
conn = ConnectionFactory.getDataSource().getConnection();
// Execute the escheduler_ddl.sql script to create the table structure of escheduler
// Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
initScriptRunner.runScript(initSqlReader);
......@@ -228,16 +226,16 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (StringUtils.isEmpty(rootDir)) {
throw new RuntimeException("Environment variable user.dir not found");
}
String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
logger.info("mysqlSQLFilePath"+mysqlSQLFilePath);
String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_dml.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
logger.info("sqlSQLFilePath"+sqlFilePath);
Connection conn = null;
PreparedStatement pstmt = null;
try {
conn = ConnectionFactory.getDataSource().getConnection();
conn.setAutoCommit(false);
// Execute the upgraded escheduler dml
// Execute the upgraded dolphinscheduler dml
ScriptRunner scriptRunner = new ScriptRunner(conn, false, true);
Reader sqlReader = new FileReader(new File(mysqlSQLFilePath));
Reader sqlReader = new FileReader(new File(sqlFilePath));
scriptRunner.runScript(sqlReader);
if (isExistsTable(T_VERSION_NAME)) {
// Change version in the version table to the new version
......@@ -295,7 +293,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (StringUtils.isEmpty(rootDir)) {
throw new RuntimeException("Environment variable user.dir not found");
}
String mysqlSQLFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
String sqlFilePath = MessageFormat.format("{0}/sql/upgrade/{1}/{2}/dolphinscheduler_ddl.sql",rootDir,schemaDir,getDbType().name().toLowerCase());
Connection conn = null;
PreparedStatement pstmt = null;
try {
......@@ -303,9 +301,9 @@ public abstract class UpgradeDao extends AbstractBaseDao {
String dbName = conn.getCatalog();
logger.info(dbName);
conn.setAutoCommit(true);
// Execute the escheduler ddl.sql for the upgrade
// Execute the dolphinscheduler ddl.sql for the upgrade
ScriptRunner scriptRunner = new ScriptRunner(conn, true, true);
Reader sqlReader = new FileReader(new File(mysqlSQLFilePath));
Reader sqlReader = new FileReader(new File(sqlFilePath));
scriptRunner.runScript(sqlReader);
} catch (FileNotFoundException e) {
......
dolphinscheduler-dao/src/main/resources/application.yml
浏览文件 @ 7e815b44
......@@ -3,8 +3,8 @@ spring:
datasource:
driver-class-name: org.postgresql.Driver
# driver-class-name: com.mysql.jdbc.Driver
url: jdbc:postgresql://192.168.220.154:5432/escheduler
# url: jdbc:mysql://192.168.220.188:3306/escheduler_new?useUnicode=true&characterEncoding=UTF-8
url: jdbc:postgresql://192.168.220.154:5432/dolphinscheduler
# url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
username: root
password: root@123
# platform:
......
dolphinscheduler-dao/src/main/resources/dao/data_source.properties__
浏览文件 @ 7e815b44
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/escheduler_new?characterEncoding=UTF-8
spring.datasource.url=jdbc:mysql://192.168.220.188:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
......
dolphinscheduler-dao/src/test/resources/dao/data_source.properties
浏览文件 @ 7e815b44
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
......
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
浏览文件 @ 7e815b44
......@@ -59,7 +59,7 @@ public class MasterServer extends AbstractServer {
private ScheduledExecutorService heartbeatMasterService;
/**
* escheduler database interface
* dolphinscheduler database interface
*/
@Autowired
protected ProcessDao processDao;
......
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
浏览文件 @ 7e815b44
......@@ -43,7 +43,7 @@ public class MasterSchedulerThread implements Runnable {
private final ExecutorService masterExecService;
/**
* escheduler database interface
* dolphinscheduler database interface
*/
private final ProcessDao processDao;
......@@ -76,7 +76,7 @@ public class MasterSchedulerThread implements Runnable {
if(OSUtils.checkResource(conf, true)){
if (zkMasterClient.getZkClient().getState() == CuratorFrameworkState.STARTED) {
// create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master
// create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master
String znodeLock = zkMasterClient.getMasterLockPath();
mutex = new InterProcessMutex(zkMasterClient.getZkClient(), znodeLock);
......
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java
浏览文件 @ 7e815b44
......@@ -115,7 +115,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
* for example :
* your PYTHON_HOM is /opt/python3.7/
* you must set PYTHON_HOME is /opt/python3.7/python under nder common.properties
* escheduler.env.path file.
* dolphinscheduler.env.path file.
*
* @param envPath
* @return
......
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
浏览文件 @ 7e815b44
......@@ -103,7 +103,7 @@ public class ZKMasterClient extends AbstractZKClient {
InterProcessMutex mutex = null;
try {
// create distributed lock with the root node path of the lock space as /escheduler/lock/failover/master
// create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master
String znodeLock = getMasterStartUpLockPath();
mutex = new InterProcessMutex(zkClient, znodeLock);
mutex.acquire();
......
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
浏览文件 @ 7e815b44
......@@ -55,7 +55,7 @@ public class ShellCommandExecutorTest {
TaskProps taskProps = new TaskProps();
// processDefineId_processInstanceId_taskInstanceId
taskProps.setTaskDir("/opt/soft/program/tmp/escheduler/exec/flow/5/36/2864/7657");
taskProps.setTaskDir("/opt/soft/program/tmp/dolphinscheduler/exec/flow/5/36/2864/7657");
taskProps.setTaskAppId("36_2864_7657");
// set tenant -> task execute linux user
taskProps.setTenantCode("hdfs");
......
dolphinscheduler-server/src/test/resources/dao/data_source.properties
浏览文件 @ 7e815b44
# base spring data source configuration
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/escheduler?characterEncoding=UTF-8
spring.datasource.url=jdbc:mysql://192.168.10.32:3306/dolphinscheduler?characterEncoding=UTF-8
spring.datasource.username=root
spring.datasource.password=root@123
......
script/monitor-server.py
浏览文件 @ 7e815b44
......@@ -14,13 +14,13 @@ or
conda install -c conda-forge kazoo
run script and parameter description：
nohup python -u monitor_server.py /data1_1T/escheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /escheduler/masters /escheduler/workers> monitor_server.log 2>&1 &
nohup python -u monitor_server.py /data1_1T/dolphinscheduler 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 /dolphinscheduler/masters /dolphinscheduler/workers> monitor_server.log 2>&1 &
the parameters are as follows:
/data1_1T/escheduler : the value comes from the installPath in install.sh
/data1_1T/dolphinscheduler : the value comes from the installPath in install.sh
192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 : the value comes from zkQuorum in install.sh
the value comes from zkWorkers in install.sh
/escheduler/masters : the value comes from zkMasters in install.sh
/escheduler/workers : the value comes from zkWorkers in install.sh
/dolphinscheduler/masters : the value comes from zkMasters in install.sh
/dolphinscheduler/workers : the value comes from zkWorkers in install.sh
'''
import sys
import socket
......@@ -73,7 +73,7 @@ class ZkClient:
if (len(restart_master_list) != 0):
for master in restart_master_list:
print("master " + self.get_ip_by_hostname(master) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start master-server')
os.system('ssh ' + self.get_ip_by_hostname(master) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start master-server')
if (self.zk.exists(workers_zk_path)):
zk_worker_list = []
......@@ -84,7 +84,7 @@ class ZkClient:
if (len(restart_worker_list) != 0):
for worker in restart_worker_list:
print("worker " + self.get_ip_by_hostname(worker) + " server has down")
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/escheduler-daemon.sh start worker-server')
os.system('ssh ' + self.get_ip_by_hostname(worker) + ' sh ' + install_path + '/bin/dolphinscheduler-daemon.sh start worker-server')
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
schedule.enter(inc, 0, self.restart_server, (inc,))
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或