diff --git a/apollo-core/src/main/java/com/ctrip/apollo/core/enums/Env.java b/apollo-core/src/main/java/com/ctrip/apollo/core/enums/Env.java index 7f7abfd33823c4ea320891f8b571830111767b4e..42277dc074760610809027859417020aa6b9a544 100644 --- a/apollo-core/src/main/java/com/ctrip/apollo/core/enums/Env.java +++ b/apollo-core/src/main/java/com/ctrip/apollo/core/enums/Env.java @@ -3,6 +3,6 @@ package com.ctrip.apollo.core.enums; /** * @author Jason Song(song_s@ctrip.com) */ -public enum Env { +public enum Env{ LOCAL, DEV, FWS, FAT, UAT, LPT, PRO, TOOLS } diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/PortalSettings.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/PortalSettings.java index 4bbcc87b92715761b28f4305a5e5e2fd742b7660..caa1d51198f1981e47b794ae7421750444b87444 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/PortalSettings.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/PortalSettings.java @@ -1,35 +1,165 @@ package com.ctrip.apollo.portal; import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import javax.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.health.Health; +import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import com.ctrip.apollo.core.enums.Env; +import com.ctrip.apollo.portal.api.AdminServiceAPI; @Component public class PortalSettings { + private Logger logger = LoggerFactory.getLogger(PortalSettings.class); + + private static final int HEALTH_CHECK_INTERVAL = 5000; + @Value("#{'${apollo.portal.env}'.split(',')}") - private List env; + private List allStrEnvs; + + @Autowired + ApplicationContext applicationContext; + + private List allEnvs = new ArrayList(); + + private volatile boolean updatedFromLastHealthCheck = true; - private List envs = new ArrayList(); + //for cache + private List activeEnvs = new LinkedList<>(); + + //mark env up or down + private Map envStatusMark = new ConcurrentHashMap<>(); + + private ScheduledExecutorService healthCheckService; + + private Lock lock = new ReentrantLock(); @PostConstruct private void postConstruct() { - for (String e : env) { - envs.add(Env.valueOf(e.toUpperCase())); + //init origin config envs + for (String e : allStrEnvs) { + allEnvs.add(Env.valueOf(e.toUpperCase())); + } + + for (Env env : allEnvs) { + envStatusMark.put(env, true); } + + healthCheckService = Executors.newScheduledThreadPool(1); + + healthCheckService + .scheduleWithFixedDelay(new HealthCheckTask(applicationContext), 1000, HEALTH_CHECK_INTERVAL, + TimeUnit.MILLISECONDS); + } - public List getEnvs() { + public List getActiveEnvs() { + if (updatedFromLastHealthCheck) { + lock.lock(); + //maybe refresh many times but not create a bad impression. + activeEnvs = refreshActiveEnvs(); + lock.unlock(); + } + return activeEnvs; + } + + private List refreshActiveEnvs() { + List envs = new LinkedList<>(); + for (Env env : allEnvs) { + if (envStatusMark.get(env)) { + envs.add(env); + } + } + logger.info("refresh active envs"); return envs; } - public Env getFirstEnv(){ - return envs.get(0); + public Env getFirstAliveEnv() { + return activeEnvs.get(0); + } + + + class HealthCheckTask implements Runnable { + + private static final int ENV_DIED_THREADHOLD = 2; + + private Map healthCheckFailCnt = new HashMap<>(); + + private AdminServiceAPI.HealthAPI healthAPI; + + public HealthCheckTask(ApplicationContext context) { + healthAPI = context.getBean(AdminServiceAPI.HealthAPI.class); + for (Env env : allEnvs) { + healthCheckFailCnt.put(env, 0l); + } + } + + public void run() { + logger.info("admin server health check start..."); + boolean hasUpdateStatus = false; + + for (Env env : allEnvs) { + try { + if (isUp(env)) { + //revive + if (!envStatusMark.get(env)) { + envStatusMark.put(env, true); + healthCheckFailCnt.put(env, 0l); + hasUpdateStatus = true; + logger.info("env up again [env:{}]", env); + } + } else { + //maybe meta server up but admin server down + handleEnvDown(env); + hasUpdateStatus = true; + } + + } catch (Exception e) { + //maybe meta server down + logger.warn("health check fail. [env:{}]", env, e.getMessage()); + handleEnvDown(env); + hasUpdateStatus = true; + } + } + + if (!hasUpdateStatus) { + logger.info("admin server health check OK"); + } + updatedFromLastHealthCheck = hasUpdateStatus; + } + + private boolean isUp(Env env) { + Health health = healthAPI.health(env); + return "UP".equals(health.getStatus().getCode()); + } + + private void handleEnvDown(Env env) { + long failCnt = healthCheckFailCnt.get(env); + healthCheckFailCnt.put(env, ++failCnt); + + if (failCnt >= ENV_DIED_THREADHOLD) { + envStatusMark.put(env, false); + logger.error("env down [env:{}]", env); + } + } + } } diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/api/AdminServiceAPI.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/api/AdminServiceAPI.java index 96fdfb32f63f524aef296f7244c2bea93b3bb6dc..752a9d0ad580d3dfc67002b67ba2fe4f5eabff65 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/api/AdminServiceAPI.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/api/AdminServiceAPI.java @@ -10,6 +10,7 @@ import com.ctrip.apollo.core.dto.ItemDTO; import com.ctrip.apollo.core.dto.NamespaceDTO; import com.ctrip.apollo.core.dto.ReleaseDTO; +import org.springframework.boot.actuate.health.Health; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; @@ -25,6 +26,14 @@ import java.util.List; @Service public class AdminServiceAPI { + @Service + public static class HealthAPI extends API{ + + public Health health(Env env){ + return restTemplate.getForObject(getAdminServiceHost(env) + "/health", Health.class); + } + } + @Service public static class AppAPI extends API { diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalAppController.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalAppController.java index c4fd6af2b19f6861d7cbab23ae5eaccb03937683..6650cfaaffcd9eec6e991c1215686fa6496c224a 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalAppController.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalAppController.java @@ -47,7 +47,7 @@ public class PortalAppController { throw new BadRequestException("app id can not be empty."); } MultiResponseEntity response = MultiResponseEntity.ok(); - List envs = portalSettings.getEnvs(); + List envs = portalSettings.getActiveEnvs(); for (Env env : envs) { try { response.addResponseEntity(RichResponseEntity.ok(appService.createEnvNavNode(env, appId))); @@ -84,7 +84,7 @@ public class PortalAppController { @RequestMapping(value = "/{appId}/miss_envs") public MultiResponseEntity findMissEnvs(@PathVariable String appId) { MultiResponseEntity response = MultiResponseEntity.ok(); - for (Env env : portalSettings.getEnvs()) { + for (Env env : portalSettings.getActiveEnvs()) { try { appService.load(env, appId); } catch (Exception e) { diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalEnvController.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalEnvController.java index 75a28f02827816ca4e28c3d8a8e0771c996db6ec..725c1176eff9b5d740ed42424769ab69e3d324f6 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalEnvController.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/controller/PortalEnvController.java @@ -19,7 +19,7 @@ public class PortalEnvController { @RequestMapping(value = "", method = RequestMethod.GET) public List envs(){ - return portalSettings.getEnvs(); + return portalSettings.getActiveEnvs(); } } diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalAppService.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalAppService.java index 88eb9a1b1af249933b6c5cff764a2860516b2df8..3f2f0946fb213af524128ed40d061a4168deb9e2 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalAppService.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalAppService.java @@ -41,7 +41,7 @@ public class PortalAppService { //轮询环境直到能找到此app的信息 AppDTO app = null; boolean isCallAdminServiceError = false; - for (Env env : portalSettings.getEnvs()) { + for (Env env : portalSettings.getActiveEnvs()) { try { app = appAPI.loadApp(env, appId); break; @@ -71,7 +71,7 @@ public class PortalAppService { } public void createAppInAllEnvs(AppDTO app) { - List envs = portalSettings.getEnvs(); + List envs = portalSettings.getActiveEnvs(); for (Env env : envs) { try { appAPI.createApp(env, app); diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalConfigService.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalConfigService.java index c4f365577961a4facdc7839d02bf876d80a8572b..00d1ed84fc17ee92d2137a1fb7f8b0e613c589c0 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalConfigService.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalConfigService.java @@ -1,8 +1,5 @@ package com.ctrip.apollo.portal.service; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,21 +9,25 @@ import org.springframework.util.CollectionUtils; import org.springframework.web.client.HttpClientErrorException; import com.ctrip.apollo.common.utils.BeanUtils; +import com.ctrip.apollo.core.enums.Env; import com.ctrip.apollo.core.dto.ItemChangeSets; import com.ctrip.apollo.core.dto.ItemDTO; import com.ctrip.apollo.core.dto.NamespaceDTO; import com.ctrip.apollo.core.dto.ReleaseDTO; -import com.ctrip.apollo.core.enums.Env; import com.ctrip.apollo.core.exception.BadRequestException; import com.ctrip.apollo.core.exception.NotFoundException; import com.ctrip.apollo.core.exception.ServiceException; import com.ctrip.apollo.portal.api.AdminServiceAPI; import com.ctrip.apollo.portal.entity.ItemDiffs; import com.ctrip.apollo.portal.entity.NamespaceIdentifer; -import com.ctrip.apollo.portal.entity.form.NamespaceReleaseModel; import com.ctrip.apollo.portal.entity.form.NamespaceTextModel; +import com.ctrip.apollo.portal.entity.form.NamespaceReleaseModel; import com.ctrip.apollo.portal.service.txtresolver.ConfigTextResolver; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + @Service public class PortalConfigService { diff --git a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalNamespaceService.java b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalNamespaceService.java index 3e22dfe28d847c6b7c5e477e1f2cac05edc78892..9ac6dfacfb645dd6d9e66582168a0afaf6324f80 100644 --- a/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalNamespaceService.java +++ b/apollo-portal/src/main/java/com/ctrip/apollo/portal/service/PortalNamespaceService.java @@ -48,7 +48,7 @@ public class PortalNamespaceService { public List findPublicAppNamespaces(){ - return namespaceAPI.findPublicAppNamespaces(portalSettings.getFirstEnv()); + return namespaceAPI.findPublicAppNamespaces(portalSettings.getFirstAliveEnv()); } public NamespaceDTO createNamespace(Env env, NamespaceDTO namespace){ @@ -56,7 +56,7 @@ public class PortalNamespaceService { } public void createAppNamespace(AppNamespaceDTO appNamespace) { - for (Env env : portalSettings.getEnvs()) { + for (Env env : portalSettings.getActiveEnvs()) { try { namespaceAPI.createAppNamespace(env, appNamespace); } catch (HttpStatusCodeException e) { diff --git a/apollo-portal/src/main/resources/portal.properties b/apollo-portal/src/main/resources/portal.properties index 9df7dd5baeff505d66649fa73e819e8bcaf785e1..e1fc7e398ecf8f98a0c35b1be73a56a3351c6448 100644 --- a/apollo-portal/src/main/resources/portal.properties +++ b/apollo-portal/src/main/resources/portal.properties @@ -2,4 +2,4 @@ spring.application.name= apollo-portal apollo.portal.env= dev,fat,uat ctrip.appid= 100003173 server.port= 8080 -logging.file= /opt/logs/100003173/apollo-portal.log \ No newline at end of file +logging.file= /opt/logs/100003173/apollo-portal.log