Merge remote-tracking branch 'upstream/dev-1.1.0' into dev-1.1.0

escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java

@@ -66,7 +66,7 @@ public class MonitorController extends BaseController{
         logger.info("login user: {}, query all master", loginUser.getUserName());
         try{
             logger.info("list master, user:{}", loginUser.getUserName());
-            Map<String, Object> result = serverService.queryMaster(loginUser);
+            Map<String, Object> result = monitorService.queryMaster(loginUser);
             return returnDataList(result);
         }catch (Exception e){
             logger.error(LIST_MASTERS_ERROR.getMsg(),e);
@@ -86,7 +86,7 @@ public class MonitorController extends BaseController{
     public Result listWorker(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
         logger.info("login user: {}, query all workers", loginUser.getUserName());
         try{
-            Map<String, Object> result = serverService.queryWorker(loginUser);
+            Map<String, Object> result = monitorService.queryWorker(loginUser);
             return returnDataList(result);
         }catch (Exception e){
             logger.error(LIST_WORKERS_ERROR.getMsg(),e);

escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java

@@ -163,6 +163,7 @@ public enum Status {
     BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error"),
     PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error"),
     PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error"),
+    SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end"),
 
 
     UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found"),

escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java

@@ -67,14 +67,10 @@ public class DataSourceService extends BaseService{
     public static final String PASSWORD = cn.escheduler.common.Constants.PASSWORD;
     public static final String OTHER = "other";
 
-    @Autowired
-    private ProjectMapper projectMapper;
 
     @Autowired
     private DataSourceMapper dataSourceMapper;
 
-    @Autowired
-    private ProjectService projectService;
 
     @Autowired
     private DatasourceUserMapper datasourceUserMapper;
@@ -522,7 +518,10 @@ public class DataSourceService extends BaseService{
         parameterMap.put(Constants.JDBC_URL, jdbcUrl);
         parameterMap.put(Constants.USER, userName);
         parameterMap.put(Constants.PASSWORD, password);
-        parameterMap.put(Constants.PRINCIPAL,principal);
+        if (CommonUtils.getKerberosStartupState() &&
+                (type == DbType.HIVE || type == DbType.SPARK)){
+            parameterMap.put(Constants.PRINCIPAL,principal);
+        }
         if (other != null && !"".equals(other)) {
             Map map = JSONObject.parseObject(other, new TypeReference<LinkedHashMap<String, String>>() {
             });

escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java

@@ -18,13 +18,16 @@ package cn.escheduler.api.service;
 
 import cn.escheduler.api.enums.Status;
 import cn.escheduler.api.utils.Constants;
-import cn.escheduler.api.utils.ZookeeperMonitorUtils;
+import cn.escheduler.api.utils.ZookeeperMonitor;
 import cn.escheduler.dao.MonitorDBDao;
+import cn.escheduler.dao.model.MasterServer;
 import cn.escheduler.dao.model.MonitorRecord;
 import cn.escheduler.dao.model.User;
 import cn.escheduler.dao.model.ZookeeperRecord;
+import org.apache.hadoop.mapred.Master;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -52,6 +55,22 @@ public class MonitorService extends BaseService{
 
   }
 
+  /**
+   * query master list
+   *
+   * @param loginUser
+   * @return
+   */
+  public Map<String,Object> queryMaster(User loginUser) {
+
+    Map<String, Object> result = new HashMap<>(5);
+
+    List<MasterServer> masterServers = new ZookeeperMonitor().getMasterServers();
+    result.put(Constants.DATA_LIST, masterServers);
+    putMsg(result,Status.SUCCESS);
+
+    return result;
+  }
 
   /**
    * query zookeeper state
@@ -61,7 +80,7 @@ public class MonitorService extends BaseService{
   public Map<String,Object> queryZookeeperState(User loginUser) {
     Map<String, Object> result = new HashMap<>(5);
 
-    List<ZookeeperRecord> zookeeperRecordList = ZookeeperMonitorUtils.zookeeperInfoList();
+    List<ZookeeperRecord> zookeeperRecordList = ZookeeperMonitor.zookeeperInfoList();
 
     result.put(Constants.DATA_LIST, zookeeperRecordList);
     putMsg(result, Status.SUCCESS);
@@ -69,4 +88,22 @@ public class MonitorService extends BaseService{
     return result;
 
   }
+
+
+  /**
+   * query master list
+   *
+   * @param loginUser
+   * @return
+   */
+  public Map<String,Object> queryWorker(User loginUser) {
+
+    Map<String, Object> result = new HashMap<>(5);
+
+    List<MasterServer> workerServers = new ZookeeperMonitor().getWorkerServers();
+    result.put(Constants.DATA_LIST, workerServers);
+    putMsg(result,Status.SUCCESS);
+
+    return result;
+  }
 }

escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java

@@ -119,6 +119,11 @@ public class SchedulerService extends BaseService {
         scheduleObj.setProcessDefinitionName(processDefinition.getName());
 
         ScheduleParam scheduleParam = JSONUtils.parseObject(schedule, ScheduleParam.class);
+        if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) {
+            logger.warn("The start time must not be the same as the end");
+            putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME);
+            return result;
+        }
         scheduleObj.setStartTime(scheduleParam.getStartTime());
         scheduleObj.setEndTime(scheduleParam.getEndTime());
         if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) {
@@ -205,6 +210,11 @@ public class SchedulerService extends BaseService {
         // updateProcessInstance param
         if (StringUtils.isNotEmpty(scheduleExpression)) {
             ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class);
+            if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) {
+                logger.warn("The start time must not be the same as the end");
+                putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME);
+                return result;
+            }
             schedule.setStartTime(scheduleParam.getStartTime());
             schedule.setEndTime(scheduleParam.getEndTime());
             if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) {

escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java

@@ -239,24 +239,25 @@ public class TenantService extends BaseService{
     if (PropertyUtils.getResUploadStartupState()){
       String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode();
 
-      String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode());
-      FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath);
-      if (fileStatus.length > 0) {
-        putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS);
-        return result;
-      }
-      fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode()));
-      if (fileStatus.length > 0) {
-        putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS);
-        return result;
-      }
+      if (HadoopUtils.getInstance().exists(tenantPath)){
+        String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode());
+        FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath);
+        if (fileStatus.length > 0) {
+          putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS);
+          return result;
+        }
+        fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode()));
+        if (fileStatus.length > 0) {
+          putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS);
+          return result;
+        }
 
-      HadoopUtils.getInstance().delete(tenantPath, true);
+        HadoopUtils.getInstance().delete(tenantPath, true);
+      }
     }
 
     tenantMapper.deleteById(id);
     putMsg(result, Status.SUCCESS);
-    
     return result;
   }
 
@@ -269,9 +270,6 @@ public class TenantService extends BaseService{
   public Map<String, Object> queryTenantList(User loginUser) {
 
     Map<String, Object> result = new HashMap<>(5);
-//    if (checkAdmin(loginUser, result)) {
-//      return result;
-//    }
 
     List<Tenant> resourceList = tenantMapper.queryAllTenant();
     result.put(Constants.DATA_LIST, resourceList);

escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java

@@ -245,35 +245,35 @@ public class UsersService extends BaseService {
             Tenant newTenant = tenantMapper.queryById(tenantId);
             if (newTenant != null) {
                 // if hdfs startup
-                if (PropertyUtils.getResUploadStartupState()){
+                if (PropertyUtils.getResUploadStartupState() && oldTenant != null){
                     String newTenantCode = newTenant.getTenantCode();
                     String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources";
                     String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode());
 
+                    if (HadoopUtils.getInstance().exists(oldResourcePath)){
+                        String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources";
+                        String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode);
 
-                    String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources";
-                    String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode);
-
-                    //file resources list
-                    List<Resource> fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0);
-                    if (CollectionUtils.isNotEmpty(fileResourcesList)) {
-                        for (Resource resource : fileResourcesList) {
-                            HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true);
+                        //file resources list
+                        List<Resource> fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0);
+                        if (CollectionUtils.isNotEmpty(fileResourcesList)) {
+                            for (Resource resource : fileResourcesList) {
+                                HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true);
+                            }
                         }
-                    }
 
-                    //udf resources
-                    List<Resource> udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1);
-                    if (CollectionUtils.isNotEmpty(udfResourceList)) {
-                        for (Resource resource : udfResourceList) {
-                            HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true);
+                        //udf resources
+                        List<Resource> udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1);
+                        if (CollectionUtils.isNotEmpty(udfResourceList)) {
+                            for (Resource resource : udfResourceList) {
+                                HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true);
+                            }
                         }
-                    }
-
-                    //Delete the user from the old tenant directory
-                    String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId;
-                    HadoopUtils.getInstance().delete(oldUserPath, true);
 
+                        //Delete the user from the old tenant directory
+                        String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId;
+                        HadoopUtils.getInstance().delete(oldUserPath, true);
+                    }
 
                     //create user in the new tenant directory
                     String newUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenant.getTenantCode() + "/home/" + user.getId();

escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java → escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java

 package cn.escheduler.api.utils;
 
 import cn.escheduler.common.zk.AbstractZKClient;
+import cn.escheduler.dao.model.MasterServer;
 import cn.escheduler.dao.model.ZookeeperRecord;
+import cn.escheduler.server.ResInfo;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -9,14 +11,15 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
 
 /**
  *	monitor zookeeper info
  */
-public class ZookeeperMonitorUtils {
+public class ZookeeperMonitor extends AbstractZKClient{
 
-	private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitorUtils.class);
+	private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitor.class);
 	private static final String zookeeperList = AbstractZKClient.getZookeeperQuorum();
 
 	/**
@@ -33,6 +36,38 @@ public class ZookeeperMonitorUtils {
 		return null;
 	}
 
+	/**
+	 * get server list.
+	 * @param isMaster
+	 * @return
+	 */
+	public List<MasterServer> getServers(boolean isMaster){
+		List<MasterServer> masterServers = new ArrayList<>();
+		Map<String, String> masterMap = getServerList(isMaster);
+		String parentPath = isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath();
+		for(String path : masterMap.keySet()){
+			MasterServer masterServer = ResInfo.parseHeartbeatForZKInfo(masterMap.get(path));
+			masterServer.setZkDirectory( parentPath + "/"+ path);
+			masterServers.add(masterServer);
+		}
+		return masterServers;
+	}
+
+	/**
+	 * get master servers
+	 * @return
+	 */
+	public List<MasterServer> getMasterServers(){
+	    return getServers(true);
+	}
+
+	/**
+	 * master construct is the same with worker, use the master instead
+	 * @return
+	 */
+	public List<MasterServer> getWorkerServers(){
+	    return getServers(false);
+	}
 
 	private static List<ZookeeperRecord> zookeeperInfoList(String zookeeperServers) {
 

escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java

+package cn.escheduler.api.utils;
+
+import cn.escheduler.dao.model.MasterServer;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+import java.util.List;
+
+public class ZookeeperMonitorUtilsTest {
+
+
+    @Test
+    public void testGetMasterLsit(){
+
+        ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor();
+
+
+        List<MasterServer> masterServerList = zookeeperMonitor.getMasterServers();
+
+        List<MasterServer> workerServerList = zookeeperMonitor.getWorkerServers();
+
+        Assert.assertEquals(masterServerList.size(), 1);
+        Assert.assertEquals(workerServerList.size(), 1);
+
+
+    }
+
+}
\ No newline at end of file

escheduler-common/src/main/java/cn/escheduler/common/Constants.java

@@ -337,7 +337,7 @@ public final class Constants {
     /**
      * email regex
      */
-    public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+$");
+    public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$");
 
     /**
      * read permission
@@ -490,7 +490,7 @@ public final class Constants {
 
     public static final String DEFAULT = "Default";
     public static final String PASSWORD = "password";
-    public static final String XXXXXX = "xxxxxx";
+    public static final String XXXXXX = "******";
 
     public static  String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd";
 

escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java

@@ -417,16 +417,6 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
         }
     }
 
-
-    /**
-     * get zookeeper client of CuratorFramework
-     * @return
-     */
-    public CuratorFramework getZkClient() {
-        return zkClient;
-    }
-
-
     /**
      * Get the task queue path
      * @param key  task queue name

escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java

@@ -61,10 +61,4 @@ public class IpUtils {
     return sb.toString();
   }
 
-
-
-  public static void main(String[] args){
-    long ipLong = ipToLong("11.3.4.5");
-    logger.info(longToIp(ipLong));
-  }
 }

escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java

@@ -30,13 +30,12 @@ import org.apache.curator.framework.imps.CuratorFrameworkState;
 import org.apache.curator.framework.state.ConnectionState;
 import org.apache.curator.framework.state.ConnectionStateListener;
 import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
 import org.apache.zookeeper.CreateMode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
+import java.util.*;
 
 import static cn.escheduler.common.Constants.*;
 
@@ -213,9 +212,9 @@ public abstract class AbstractZKClient {
 	protected void initSystemZNode(){
 		try {
 			// read master node parent path from conf
-			masterZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS);
+			masterZNodeParentPath = getMasterZNodeParentPath();
 			// read worker node parent path from conf
-			workerZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS);
+			workerZNodeParentPath = getWorkerZNodeParentPath();
 
 			// read server node parent path from conf
 			deadServerZNodeParentPath = conf.getString(ZOOKEEPER_ESCHEDULER_DEAD_SERVERS);
@@ -243,6 +242,7 @@ public abstract class AbstractZKClient {
 		}
 	}
 
+
 	public void removeDeadServerByHost(String host, String serverType) throws Exception {
         List<String> deadServers = zkClient.getChildren().forPath(deadServerZNodeParentPath);
         for(String serverPath : deadServers){
@@ -291,6 +291,8 @@ public abstract class AbstractZKClient {
 
 	}
 
+
+
 	/**
 	 * for stop server
 	 * @param serverStoppable
@@ -339,6 +341,81 @@ public abstract class AbstractZKClient {
 		return sb.toString();
 	}
 
+	/**
+	 * get master server list map.
+	 * result : {host : resource info}
+	 * @return
+	 */
+	public Map<String, String> getServerList(boolean isMaster ){
+
+		Map<String, String> masterMap = new HashMap<>();
+		try {
+			String path =  isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath();
+			List<String> serverList  = getZkClient().getChildren().forPath(path);
+			for(String server : serverList){
+				byte[] bytes  = getZkClient().getData().forPath(path + "/" + server);
+				masterMap.putIfAbsent(server, new String(bytes));
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+
+		return masterMap;
+	}
+
+	/**
+	 *  get zkclient
+	 * @return
+	 */
+	public  CuratorFramework getZkClient() {
+		return zkClient;
+	}
+
+	/**
+	 * get worker node parent path
+	 * @return
+	 */
+	protected String getWorkerZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS);};
+
+	/**
+	 * get master node parent path
+	 * @return
+	 */
+	protected String getMasterZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS);}
+
+	/**
+	 *  get master lock path
+	 * @return
+	 */
+	public String getMasterLockPath(){
+		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS);
+	}
+
+	/**
+	 *  get master start up lock path
+	 * @return
+	 */
+	public String getMasterStartUpLockPath(){
+		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS);
+	}
+
+	/**
+	 *  get master failover lock path
+	 * @return
+	 */
+	public String getMasterFailoverLockPath(){
+		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS);
+	}
+
+	/**
+	 * get worker failover lock path
+	 * @return
+	 */
+	public String getWorkerFailoverLockPath(){
+		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS);
+	}
+
+
 	@Override
 	public String toString() {
 		return "AbstractZKClient{" +

escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java

@@ -20,13 +20,11 @@ import cn.escheduler.common.Constants;
 import cn.escheduler.common.utils.IpUtils;
 import cn.escheduler.common.utils.OSUtils;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
 
@@ -84,8 +82,6 @@ public class TaskQueueImplTest {
             return;
         }
 
-        String node2 = tasks.get(0);
-
     }
 
 

escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java

+package cn.escheduler.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class IpUtilsTest {
+
+    @Test
+    public void ipToLong() {
+
+        String ip = "192.168.110.1";
+        String ip2 = "0.0.0.0";
+        long longNumber = IpUtils.ipToLong(ip);
+        long longNumber2 = IpUtils.ipToLong(ip2);
+        System.out.println(longNumber);
+        Assert.assertEquals(longNumber, 3232263681L);
+        Assert.assertEquals(longNumber2, 0L);
+
+        String ip3 = "255.255.255.255";
+        long longNumber3 = IpUtils.ipToLong(ip3);
+        System.out.println(longNumber3);
+        Assert.assertEquals(longNumber3, 4294967295L);
+
+    }
+
+    @Test
+    public void longToIp() {
+
+        String ip = "192.168.110.1";
+        String ip2 = "0.0.0.0";
+        long longNum = 3232263681L;
+        String i1 = IpUtils.longToIp(longNum);
+
+        String i2 = IpUtils.longToIp(0);
+
+        Assert.assertEquals(ip, i1);
+        Assert.assertEquals(ip2, i2);
+    }
+}
\ No newline at end of file

escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java

@@ -30,13 +30,15 @@ public class CreateEscheduler {
 
 	public static void main(String[] args) {
 		EschedulerManager eschedulerManager = new EschedulerManager();
-		eschedulerManager.initEscheduler();
-		logger.info("init escheduler finished");
+
 		try {
+			eschedulerManager.initEscheduler();
+			logger.info("init escheduler finished");
 			eschedulerManager.upgradeEscheduler();
 			logger.info("upgrade escheduler finished");
+			logger.info("create escheduler success");
 		} catch (Exception e) {
-			logger.error("upgrade escheduler failed",e);
+			logger.error("create escheduler failed",e);
 		}
 
 	}

escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java

@@ -30,7 +30,7 @@ public class UpgradeEscheduler {
 		EschedulerManager eschedulerManager = new EschedulerManager();
 		try {
 			eschedulerManager.upgradeEscheduler();
-			logger.info("upgrade escheduler finished");
+			logger.info("upgrade escheduler success");
 		} catch (Exception e) {
 			logger.error(e.getMessage(),e);
 			logger.info("Upgrade escheduler failed");

escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java

@@ -17,8 +17,12 @@
 package cn.escheduler.server;
 
 import cn.escheduler.common.Constants;
+import cn.escheduler.common.utils.DateUtils;
 import cn.escheduler.common.utils.JSONUtils;
 import cn.escheduler.common.utils.OSUtils;
+import cn.escheduler.dao.model.MasterServer;
+
+import java.util.Date;
 
 /**
  *  heartbeat for ZK reigster res info
@@ -98,6 +102,16 @@ public class ResInfo {
     }
 
 
+    public static String getHeartBeatInfo(Date now){
+        return buildHeartbeatForZKInfo(OSUtils.getHost(),
+                OSUtils.getProcessID(),
+                OSUtils.cpuUsage(),
+                OSUtils.memoryUsage(),
+                DateUtils.dateToString(now),
+                DateUtils.dateToString(now));
+
+    }
+
     /**
      * build heartbeat info for zk
      * @param host
@@ -119,4 +133,25 @@ public class ResInfo {
                 + lastHeartbeatTime;
     }
 
+    /**
+     * parse heartbeat info for zk
+     * @param heartBeatInfo
+     * @return
+     */
+    public static MasterServer parseHeartbeatForZKInfo(String heartBeatInfo){
+        MasterServer masterServer =  null;
+        String[] masterArray = heartBeatInfo.split(Constants.COMMA);
+        if(masterArray.length != 6){
+            return masterServer;
+
+        }
+        masterServer = new MasterServer();
+        masterServer.setHost(masterArray[0]);
+        masterServer.setPort(Integer.parseInt(masterArray[1]));
+        masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[2]), Double.parseDouble(masterArray[3])));
+        masterServer.setCreateTime(DateUtils.stringToDate(masterArray[4]));
+        masterServer.setLastHeartbeatTime(DateUtils.stringToDate(masterArray[5]));
+        return masterServer;
+    }
+
 }

escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java

@@ -204,7 +204,7 @@ public class ZKMasterClient extends AbstractZKClient {
 			}
 
 			// specify the format of stored data in ZK nodes
-			String heartbeatZKInfo = getOsInfo(now);
+			String heartbeatZKInfo = ResInfo.getHeartBeatInfo(now);
 			// create temporary sequence nodes for master znode
 			masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(
 					masterZNodeParentPath + "/" + OSUtils.getHost() + "_", heartbeatZKInfo.getBytes());
@@ -259,10 +259,10 @@ public class ZKMasterClient extends AbstractZKClient {
 			return false;
 		}
 
-		List<String> masterZNodeList = null;
-        masterZNodeList = zkClient.getChildren().forPath(path);
-		if (CollectionUtils.isNotEmpty(masterZNodeList)){
-            for (String masterZNode : masterZNodeList){
+		List<String> serverList = null;
+        serverList = zkClient.getChildren().forPath(path);
+		if (CollectionUtils.isNotEmpty(serverList)){
+            for (String masterZNode : serverList){
                 if (masterZNode.startsWith(host)){
                     return true;
                 }
@@ -423,22 +423,6 @@ public class ZKMasterClient extends AbstractZKClient {
 
 	}
 
-
-	/**
-	 * get os info
-	 * @param now
-	 * @return
-	 */
-	private String getOsInfo(Date now) {
-		return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(),
-				OSUtils.getProcessID(),
-				OSUtils.cpuUsage(),
-				OSUtils.memoryUsage(),
-				DateUtils.dateToString(now),
-				DateUtils.dateToString(now));
-	}
-
-
 	/**
 	 *  get master znode
 	 * @return
@@ -448,45 +432,6 @@ public class ZKMasterClient extends AbstractZKClient {
 	}
 
 
-	/**
-	 *  get master lock path
-	 * @return
-	 */
-	public String getMasterLockPath(){
-		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS);
-	}
-
-	/**
-	 *  get master start up lock path
-	 * @return
-	 */
-	public String getMasterStartUpLockPath(){
-		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS);
-	}
-
-	/**
-	 *  get master failover lock path
-	 * @return
-	 */
-	public String getMasterFailoverLockPath(){
-		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS);
-	}
-
-	/**
-	 * get worker failover lock path
-	 * @return
-	 */
-	public String getWorkerFailoverLockPath(){
-		return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS);
-	}
-
-	/**
-	 *  get zkclient
-	 * @return
-	 */
-	public  CuratorFramework getZkClient() {
-		return zkClient;
-	}
 
 
 	/**
@@ -580,7 +525,7 @@ public class ZKMasterClient extends AbstractZKClient {
 	}
 
 	/**
-	 *  get host ip
+	 *  get host ip, string format: masterParentPath/ip_000001/value
 	 * @param path
 	 * @return
 	 */

escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java

@@ -116,11 +116,10 @@ public class ZKWorkerClient extends AbstractZKClient {
 
 	public String initWorkZNode() throws Exception {
 
-		Date now = new Date();
-		String heartbeatZKInfo = getOsInfo(now);
-
+		String heartbeatZKInfo = ResInfo.getHeartBeatInfo(new Date());
 
 		workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_";
+
 		workerZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(workerZNode,
 				heartbeatZKInfo.getBytes());
 		logger.info("register worker node {} success", workerZNode);
@@ -141,7 +140,6 @@ public class ZKWorkerClient extends AbstractZKClient {
 			workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_";
 			List<String> workerZNodeList = zkClient.getChildren().forPath(workerZNodeParentPath);
 
-
 			if (CollectionUtils.isNotEmpty(workerZNodeList)){
 				boolean flag = false;
 				for (String workerZNode : workerZNodeList){
@@ -241,21 +239,6 @@ public class ZKWorkerClient extends AbstractZKClient {
 
 	}
 
-	/**
-	 * get os info
-	 * @param now
-	 * @return
-	 */
-	private String getOsInfo(Date now) {
-		return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(),
-				OSUtils.getProcessID(),
-				OSUtils.cpuUsage(),
-				OSUtils.memoryUsage(),
-				DateUtils.dateToString(now),
-				DateUtils.dateToString(now));
-	}
-
-
 	/**
 	 * get worker znode
 	 * @return
@@ -264,16 +247,6 @@ public class ZKWorkerClient extends AbstractZKClient {
 		return workerZNode;
 	}
 
-
-	/**
-	 *  get zkclient
-	 * @return
-	 */
-	public  CuratorFramework getZkClient() {
-		return zkClient;
-	}
-
-
 	/**
 	 *  get worker lock path
 	 * @return

escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java

 package cn.escheduler.server.zk;
 
 import cn.escheduler.common.Constants;
+import cn.escheduler.common.zk.AbstractZKClient;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -17,8 +18,8 @@ public class ZKWorkerClientTest {
     public void getZKWorkerClient() throws Exception {
 
 
-        ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient();
-        zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX);
+//        ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient();
+//        zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX);
 
 
     }

install.sh

@@ -134,7 +134,7 @@ s3Endpoint="http://192.168.199.91:9010"
 s3AccessKey="A3DXS30FO22544RE"
 s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK"
 
-# resourcemanager HA配置，如果是单resourcemanager,这里为空即可
+# resourcemanager HA配置，如果是单resourcemanager,这里为yarnHaIps=""
 yarnHaIps="192.168.xx.xx,192.168.xx.xx"
 
 # 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好
@@ -144,7 +144,7 @@ singleYarnIp="ark1"
 hdfsPath="/escheduler"
 
 # 拥有在hdfs根路径/下创建目录权限的用户
-# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以
+# 注意：如果开启了kerberos，则直接hdfsRootUser=""，就可以
 hdfsRootUser="hdfs"
 
 # common 配置

sql/quartz.sql

- #
- # In your Quartz properties file, you'll need to set 
- # org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate
- #
- #
- # By: Ron Cordell - roncordell
- #  I didn't see this anywhere, so I thought I'd post it here. This is the script from Quartz to create the tables in a MySQL database, modified to use INNODB instead of MYISAM.
- 
- DROP TABLE IF EXISTS QRTZ_FIRED_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_PAUSED_TRIGGER_GRPS;
- DROP TABLE IF EXISTS QRTZ_SCHEDULER_STATE;
- DROP TABLE IF EXISTS QRTZ_LOCKS;
- DROP TABLE IF EXISTS QRTZ_SIMPLE_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_SIMPROP_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_CRON_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_BLOB_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_TRIGGERS;
- DROP TABLE IF EXISTS QRTZ_JOB_DETAILS;
- DROP TABLE IF EXISTS QRTZ_CALENDARS;
- 
- CREATE TABLE QRTZ_JOB_DETAILS(
- SCHED_NAME VARCHAR(120) NOT NULL,
- JOB_NAME VARCHAR(200) NOT NULL,
- JOB_GROUP VARCHAR(200) NOT NULL,
- DESCRIPTION VARCHAR(250) NULL,
- JOB_CLASS_NAME VARCHAR(250) NOT NULL,
- IS_DURABLE VARCHAR(1) NOT NULL,
- IS_NONCONCURRENT VARCHAR(1) NOT NULL,
- IS_UPDATE_DATA VARCHAR(1) NOT NULL,
- REQUESTS_RECOVERY VARCHAR(1) NOT NULL,
- JOB_DATA BLOB NULL,
- PRIMARY KEY (SCHED_NAME,JOB_NAME,JOB_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_TRIGGERS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- TRIGGER_NAME VARCHAR(200) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- JOB_NAME VARCHAR(200) NOT NULL,
- JOB_GROUP VARCHAR(200) NOT NULL,
- DESCRIPTION VARCHAR(250) NULL,
- NEXT_FIRE_TIME BIGINT(13) NULL,
- PREV_FIRE_TIME BIGINT(13) NULL,
- PRIORITY INTEGER NULL,
- TRIGGER_STATE VARCHAR(16) NOT NULL,
- TRIGGER_TYPE VARCHAR(8) NOT NULL,
- START_TIME BIGINT(13) NOT NULL,
- END_TIME BIGINT(13) NULL,
- CALENDAR_NAME VARCHAR(200) NULL,
- MISFIRE_INSTR SMALLINT(2) NULL,
- JOB_DATA BLOB NULL,
- PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP),
- FOREIGN KEY (SCHED_NAME,JOB_NAME,JOB_GROUP)
- REFERENCES QRTZ_JOB_DETAILS(SCHED_NAME,JOB_NAME,JOB_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_SIMPLE_TRIGGERS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- TRIGGER_NAME VARCHAR(200) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- REPEAT_COUNT BIGINT(7) NOT NULL,
- REPEAT_INTERVAL BIGINT(12) NOT NULL,
- TIMES_TRIGGERED BIGINT(10) NOT NULL,
- PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP),
- FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)
- REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_CRON_TRIGGERS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- TRIGGER_NAME VARCHAR(200) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- CRON_EXPRESSION VARCHAR(120) NOT NULL,
- TIME_ZONE_ID VARCHAR(80),
- PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP),
- FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)
- REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_SIMPROP_TRIGGERS
-   (          
-     SCHED_NAME VARCHAR(120) NOT NULL,
-     TRIGGER_NAME VARCHAR(200) NOT NULL,
-     TRIGGER_GROUP VARCHAR(200) NOT NULL,
-     STR_PROP_1 VARCHAR(512) NULL,
-     STR_PROP_2 VARCHAR(512) NULL,
-     STR_PROP_3 VARCHAR(512) NULL,
-     INT_PROP_1 INT NULL,
-     INT_PROP_2 INT NULL,
-     LONG_PROP_1 BIGINT NULL,
-     LONG_PROP_2 BIGINT NULL,
-     DEC_PROP_1 NUMERIC(13,4) NULL,
-     DEC_PROP_2 NUMERIC(13,4) NULL,
-     BOOL_PROP_1 VARCHAR(1) NULL,
-     BOOL_PROP_2 VARCHAR(1) NULL,
-     PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP),
-     FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) 
-     REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_BLOB_TRIGGERS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- TRIGGER_NAME VARCHAR(200) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- BLOB_DATA BLOB NULL,
- PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP),
- INDEX (SCHED_NAME,TRIGGER_NAME, TRIGGER_GROUP),
- FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)
- REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_CALENDARS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- CALENDAR_NAME VARCHAR(200) NOT NULL,
- CALENDAR BLOB NOT NULL,
- PRIMARY KEY (SCHED_NAME,CALENDAR_NAME))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- PRIMARY KEY (SCHED_NAME,TRIGGER_GROUP))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_FIRED_TRIGGERS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- ENTRY_ID VARCHAR(95) NOT NULL,
- TRIGGER_NAME VARCHAR(200) NOT NULL,
- TRIGGER_GROUP VARCHAR(200) NOT NULL,
- INSTANCE_NAME VARCHAR(200) NOT NULL,
- FIRED_TIME BIGINT(13) NOT NULL,
- SCHED_TIME BIGINT(13) NOT NULL,
- PRIORITY INTEGER NOT NULL,
- STATE VARCHAR(16) NOT NULL,
- JOB_NAME VARCHAR(200) NULL,
- JOB_GROUP VARCHAR(200) NULL,
- IS_NONCONCURRENT VARCHAR(1) NULL,
- REQUESTS_RECOVERY VARCHAR(1) NULL,
- PRIMARY KEY (SCHED_NAME,ENTRY_ID))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_SCHEDULER_STATE (
- SCHED_NAME VARCHAR(120) NOT NULL,
- INSTANCE_NAME VARCHAR(200) NOT NULL,
- LAST_CHECKIN_TIME BIGINT(13) NOT NULL,
- CHECKIN_INTERVAL BIGINT(13) NOT NULL,
- PRIMARY KEY (SCHED_NAME,INSTANCE_NAME))
- ENGINE=InnoDB;
- 
- CREATE TABLE QRTZ_LOCKS (
- SCHED_NAME VARCHAR(120) NOT NULL,
- LOCK_NAME VARCHAR(40) NOT NULL,
- PRIMARY KEY (SCHED_NAME,LOCK_NAME))
- ENGINE=InnoDB;
- 
- CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY);
- CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP);
- 
- CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP);
- CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP);
- CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME);
- CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP);
- CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE);
- CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE);
- CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE);
- CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME);
- CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME);
- CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME);
- CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE);
- CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE);
- 
- CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME);
- CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY);
- CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP);
- CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP);
- CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP);
- CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP);
- 
- commit; 
\ No newline at end of file