diff --git a/docs/install_guide/config_description.md b/docs/install_guide/config_description.md index 54ec4d85f5405b55cdc67c4dc12172e526d7e274..8f8fc0be7216194a027d701080d2b9c003c35e04 100644 --- a/docs/install_guide/config_description.md +++ b/docs/install_guide/config_description.md @@ -54,6 +54,11 @@ custom: topic-throttled-metrics: false # 滴滴埋入的指标, 社区AK不存在该指标，因此默认关闭 save-days: 7 #指标在DB中保持的天数，-1表示永久保存，7表示保存近7天的数据 +# 任务相关的开关 +task: + op: + sync-topic-enabled: false # 未落盘的Topic定期同步到DB中 + account: # ldap相关的配置, 社区版本暂时支持不够完善，可以先忽略，欢迎贡献代码对这块做优化 ldap: @@ -71,7 +76,7 @@ kcm: # 集群升级部署相关的功能，需要配合夜莺及S3进行使用 monitor: # 监控告警相关的功能，需要配合夜莺进行使用 enabled: false # 默认关闭，true就是开启 n9e: - nid: 2 + nid: 2 user-token: 1234567890 mon: # 夜莺 mon监控服务 地址 diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java index 0c4c877aa57ab7edec8485e4e69a24809f0f6980..73569d5677be07b6d76aed48a4a824ce9eb31cf0 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/IDCEnum.java @@ -6,8 +6,6 @@ package com.xiaojukeji.kafka.manager.common.bizenum; */ public enum IDCEnum { CN("cn", "国内"), - US("us", "美东"), - RU("ru", "俄罗斯"), ; private String idc; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ModuleEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ModuleEnum.java index ac2a894710cb28ab2467db044d0932e5a716cdd1..280506cbf19249ff0c050ad3ee8b5d28fdeca5cc 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ModuleEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/ModuleEnum.java @@ -21,6 +21,8 @@ public enum ModuleEnum { PARTITION(5, "分区"), + GATEWAY_CONFIG(6, "Gateway配置"), + UNKNOWN(-1, "未知") ; ModuleEnum(int code, String message) { diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java index 6659f904e1f1e70051a8300035720c7d031efa44..e196e8c213aca742690c49f1d7896ecdf79e73d4 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/RebalanceDimensionEnum.java @@ -10,6 +10,7 @@ public enum RebalanceDimensionEnum { REGION(1, "Region维度"), BROKER(2, "Broker维度"), TOPIC(3, "Topic维度"), + PARTITION(4, "Partition维度"), ; private Integer code; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java index 667be791bbcef51dfaf633b5ec68cae4907df601..226cc5c617ef8db3897d0af9982cb161f98a9c5f 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/bizenum/gateway/GatewayConfigKeyEnum.java @@ -45,4 +45,13 @@ public enum GatewayConfigKeyEnum { ", configName='" + configName + '\'' + '}'; } + + public static GatewayConfigKeyEnum getByConfigType(String configType) { + for (GatewayConfigKeyEnum configKeyEnum: GatewayConfigKeyEnum.values()) { + if (configKeyEnum.getConfigType().equals(configType)) { + return configKeyEnum; + } + } + return null; + } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java index b8c5ef1977dd87a6761e4eb8865e607f631b91e3..9242530381ebf54c02f5abe7c38b0e213873fd71 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/KafkaConstant.java @@ -7,6 +7,8 @@ package com.xiaojukeji.kafka.manager.common.constant; public class KafkaConstant { public static final String COORDINATOR_TOPIC_NAME = "__consumer_offsets"; + public static final String TRANSACTION_TOPIC_NAME = "__transaction_state"; + public static final String BROKER_HOST_NAME_SUFFIX = ".diditaxi.com"; public static final String CLIENT_VERSION_CODE_UNKNOWN = "-1"; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java index f2f1922cd78cf5202d6bed40215467a5f287c070..b8c361b3f640a1a98c251ab477de9453c9a12279 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/constant/TopicCreationConstant.java @@ -12,11 +12,6 @@ public class TopicCreationConstant { */ public static final String LOG_X_CREATE_TOPIC_CONFIG_KEY_NAME = "LOG_X_CREATE_TOPIC_CONFIG"; - /** - * 治理平台创建Topic配置KEY - */ - public static final String CHORUS_CREATE_TOPIC_CONFIG_KEY_NAME = "CHORUS_CREATE_TOPIC_CONFIG"; - /** * 内部创建Topic配置KEY */ @@ -30,6 +25,8 @@ public class TopicCreationConstant { public static final String TOPIC_RETENTION_TIME_KEY_NAME = "retention.ms"; + public static final Long DEFAULT_QUOTA = 3 * 1024 * 1024L; + public static Properties createNewProperties(Long retentionTime) { Properties properties = new Properties(); properties.put(TOPIC_RETENTION_TIME_KEY_NAME, String.valueOf(retentionTime)); diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ConsumerMetadata.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ConsumerMetadata.java index eb384c473fa7cb7089ff7c04ef9d8bea0a3ef98f..ae943c7c1d3992b856274dd44c67d159e5d1b1e5 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ConsumerMetadata.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ConsumerMetadata.java @@ -3,7 +3,6 @@ package com.xiaojukeji.kafka.manager.common.entity; import kafka.admin.AdminClient; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; /** * @author zengqiao @@ -16,17 +15,12 @@ public class ConsumerMetadata { private Map consumerGroupSummaryMap = new HashMap<>(); - private Map> consumerGroupAppMap = new ConcurrentHashMap<>(); - - public ConsumerMetadata(Set consumerGroupSet, Map> topicNameConsumerGroupMap, - Map consumerGroupSummaryMap, - Map> consumerGroupAppMap) { + Map consumerGroupSummaryMap) { this.consumerGroupSet = consumerGroupSet; this.topicNameConsumerGroupMap = topicNameConsumerGroupMap; this.consumerGroupSummaryMap = consumerGroupSummaryMap; - this.consumerGroupAppMap = consumerGroupAppMap; } public Set getConsumerGroupSet() { @@ -40,8 +34,4 @@ public class ConsumerMetadata { public Map getConsumerGroupSummaryMap() { return consumerGroupSummaryMap; } - - public Map> getConsumerGroupAppMap() { - return consumerGroupAppMap; - } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java index a3e8b655c30a09594eed1f6f1759f25ca3ab657c..0fb383023c6f78293402164f00bb6145b79228f2 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/Result.java @@ -1,6 +1,7 @@ package com.xiaojukeji.kafka.manager.common.entity; import com.alibaba.fastjson.JSON; +import com.xiaojukeji.kafka.manager.common.constant.Constant; import java.io.Serializable; @@ -118,4 +119,9 @@ public class Result implements Serializable { result.setData(data); return result; } + + public boolean failed() { + return !Constant.SUCCESS.equals(code); + } + } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/cluster/ControllerPreferredCandidate.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/cluster/ControllerPreferredCandidate.java new file mode 100644 index 0000000000000000000000000000000000000000..2de2fe575f700dd732ff6d0f3779ea696ed3f647 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/cluster/ControllerPreferredCandidate.java @@ -0,0 +1,53 @@ +package com.xiaojukeji.kafka.manager.common.entity.ao.cluster; + +public class ControllerPreferredCandidate { + private Integer brokerId; + + private String host; + + private Long startTime; + + private Integer status; + + public Integer getBrokerId() { + return brokerId; + } + + public void setBrokerId(Integer brokerId) { + this.brokerId = brokerId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + @Override + public String toString() { + return "ControllerPreferredBroker{" + + "brokerId=" + brokerId + + ", host='" + host + '\'' + + ", startTime=" + startTime + + ", status=" + status + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroup.java similarity index 71% rename from kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupDTO.java rename to kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroup.java index e09f93e8520a94b9ffacc2bfeaf44afbcd4ad74e..9f18808634cc793438e279335afcaa1c46aefd6c 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroup.java @@ -2,30 +2,18 @@ package com.xiaojukeji.kafka.manager.common.entity.ao.consumer; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetLocationEnum; -import java.util.List; import java.util.Objects; -/** - * 消费组信息 - * @author zengqiao - * @date 19/4/18 - */ -public class ConsumerGroupDTO { +public class ConsumerGroup { private Long clusterId; private String consumerGroup; - private List appIdList; - private OffsetLocationEnum offsetStoreLocation; - public ConsumerGroupDTO(Long clusterId, - String consumerGroup, - List appIdList, - OffsetLocationEnum offsetStoreLocation) { + public ConsumerGroup(Long clusterId, String consumerGroup, OffsetLocationEnum offsetStoreLocation) { this.clusterId = clusterId; this.consumerGroup = consumerGroup; - this.appIdList = appIdList; this.offsetStoreLocation = offsetStoreLocation; } @@ -45,14 +33,6 @@ public class ConsumerGroupDTO { this.consumerGroup = consumerGroup; } - public List getAppIdList() { - return appIdList; - } - - public void setAppIdList(List appIdList) { - this.appIdList = appIdList; - } - public OffsetLocationEnum getOffsetStoreLocation() { return offsetStoreLocation; } @@ -63,10 +43,9 @@ public class ConsumerGroupDTO { @Override public String toString() { - return "ConsumerGroupDTO{" + + return "ConsumerGroup{" + "clusterId=" + clusterId + ", consumerGroup='" + consumerGroup + '\'' + - ", appIdList=" + appIdList + ", offsetStoreLocation=" + offsetStoreLocation + '}'; } @@ -79,7 +58,7 @@ public class ConsumerGroupDTO { if (o == null || getClass() != o.getClass()) { return false; } - ConsumerGroupDTO that = (ConsumerGroupDTO) o; + ConsumerGroup that = (ConsumerGroup) o; return clusterId.equals(that.clusterId) && consumerGroup.equals(that.consumerGroup) && offsetStoreLocation == that.offsetStoreLocation; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupSummary.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupSummary.java new file mode 100644 index 0000000000000000000000000000000000000000..ca89836e59cb5628a867311fedda89580ca66727 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/ao/consumer/ConsumerGroupSummary.java @@ -0,0 +1,68 @@ +package com.xiaojukeji.kafka.manager.common.entity.ao.consumer; + +import com.xiaojukeji.kafka.manager.common.bizenum.OffsetLocationEnum; + +import java.util.List; + +public class ConsumerGroupSummary { + private Long clusterId; + + private String consumerGroup; + + private OffsetLocationEnum offsetStoreLocation; + + private List appIdList; + + private String state; + + public Long getClusterId() { + return clusterId; + } + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public OffsetLocationEnum getOffsetStoreLocation() { + return offsetStoreLocation; + } + + public void setOffsetStoreLocation(OffsetLocationEnum offsetStoreLocation) { + this.offsetStoreLocation = offsetStoreLocation; + } + + public List getAppIdList() { + return appIdList; + } + + public void setAppIdList(List appIdList) { + this.appIdList = appIdList; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + @Override + public String toString() { + return "ConsumerGroupSummary{" + + "clusterId=" + clusterId + + ", consumerGroup='" + consumerGroup + '\'' + + ", offsetStoreLocation=" + offsetStoreLocation + + ", appIdList=" + appIdList + + ", state='" + state + '\'' + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/RebalanceDTO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/RebalanceDTO.java index 29db6bdc10ffea57dfbc02d87a83482ff7744f49..a27ca9a0741179b83cfef1d98a94cc0bacbe1686 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/RebalanceDTO.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/dto/op/RebalanceDTO.java @@ -25,7 +25,10 @@ public class RebalanceDTO { @ApiModelProperty(value = "TopicName") private String topicName; - @ApiModelProperty(value = "维度[0: Cluster维度, 1: Region维度, 2:Broker维度, 3:Topic维度]") + @ApiModelProperty(value = "分区ID") + private Integer partitionId; + + @ApiModelProperty(value = "维度[0: Cluster维度, 1: Region维度, 2:Broker维度, 3:Topic维度, 4:Partition纬度]") private Integer dimension; public Long getClusterId() { @@ -60,6 +63,14 @@ public class RebalanceDTO { this.topicName = topicName; } + public Integer getPartitionId() { + return partitionId; + } + + public void setPartitionId(Integer partitionId) { + this.partitionId = partitionId; + } + public Integer getDimension() { return dimension; } @@ -68,22 +79,12 @@ public class RebalanceDTO { this.dimension = dimension; } - @Override - public String toString() { - return "RebalanceDTO{" + - "clusterId=" + clusterId + - ", regionId=" + regionId + - ", brokerId=" + brokerId + - ", topicName='" + topicName + '\'' + - ", dimension=" + dimension + - '}'; - } - public boolean paramLegal() { if (ValidateUtils.isNull(clusterId) - || RebalanceDimensionEnum.REGION.getCode().equals(dimension) && ValidateUtils.isNull(regionId) - || RebalanceDimensionEnum.BROKER.getCode().equals(dimension) && ValidateUtils.isNull(brokerId) - || RebalanceDimensionEnum.TOPIC.getCode().equals(dimension) && ValidateUtils.isNull(topicName) ) { + || (RebalanceDimensionEnum.REGION.getCode().equals(dimension) && ValidateUtils.isNull(regionId)) + || (RebalanceDimensionEnum.BROKER.getCode().equals(dimension) && ValidateUtils.isNull(brokerId)) + || (RebalanceDimensionEnum.TOPIC.getCode().equals(dimension) && ValidateUtils.isNull(topicName)) + || (RebalanceDimensionEnum.PARTITION.getCode().equals(dimension) && (ValidateUtils.isNull(topicName) || ValidateUtils.isNull(partitionId))) ) { return false; } return true; diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/consumer/ConsumerGroupSummaryVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/consumer/ConsumerGroupSummaryVO.java new file mode 100644 index 0000000000000000000000000000000000000000..0049468d8e4a9c07d7536e6fb7a648aaeb483f8e --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/normal/consumer/ConsumerGroupSummaryVO.java @@ -0,0 +1,67 @@ +package com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +/** + * @author zengqiao + * @date 21/01/14 + */ +@ApiModel(value = "Topic消费组概要信息") +public class ConsumerGroupSummaryVO { + @ApiModelProperty(value = "消费组名称") + private String consumerGroup; + + @ApiModelProperty(value = "使用的AppID") + private String appIds; + + @ApiModelProperty(value = "offset存储位置") + private String location; + + @ApiModelProperty(value = "消费组状态") + private String state; + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public String getAppIds() { + return appIds; + } + + public void setAppIds(String appIds) { + this.appIds = appIds; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + @Override + public String toString() { + return "ConsumerGroupSummaryVO{" + + "consumerGroup='" + consumerGroup + '\'' + + ", appIds=" + appIds + + ", location='" + location + '\'' + + ", state='" + state + '\'' + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java new file mode 100644 index 0000000000000000000000000000000000000000..a0b402afca0b36e68b9cb8af59fba88e84d1e4ec --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/GatewayConfigVO.java @@ -0,0 +1,103 @@ +package com.xiaojukeji.kafka.manager.common.entity.vo.rd; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.Date; + +/** + * @author zengqiao + * @date 20/3/19 + */ +@ApiModel(value = "GatewayConfigVO", description = "Gateway配置信息") +public class GatewayConfigVO { + @ApiModelProperty(value="ID") + private Long id; + + @ApiModelProperty(value="配置类型") + private String type; + + @ApiModelProperty(value="配置名称") + private String name; + + @ApiModelProperty(value="配置值") + private String value; + + @ApiModelProperty(value="版本") + private Long version; + + @ApiModelProperty(value="创建时间") + private Date createTime; + + @ApiModelProperty(value="修改时间") + private Date modifyTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + @Override + public String toString() { + return "GatewayConfigVO{" + + "id=" + id + + ", type='" + type + '\'' + + ", name='" + name + '\'' + + ", value='" + value + '\'' + + ", version=" + version + + ", createTime=" + createTime + + ", modifyTime=" + modifyTime + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/cluster/ControllerPreferredCandidateVO.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/cluster/ControllerPreferredCandidateVO.java new file mode 100644 index 0000000000000000000000000000000000000000..399b35fb451d3091232bf449c0516dc9db192194 --- /dev/null +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/entity/vo/rd/cluster/ControllerPreferredCandidateVO.java @@ -0,0 +1,61 @@ +package com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel(description = "Broker基本信息") +public class ControllerPreferredCandidateVO { + @ApiModelProperty(value = "brokerId") + private Integer brokerId; + + @ApiModelProperty(value = "主机名") + private String host; + + @ApiModelProperty(value = "启动时间") + private Long startTime; + + @ApiModelProperty(value = "broker状态[0:在线, -1:不在线]") + private Integer status; + + public Integer getBrokerId() { + return brokerId; + } + + public void setBrokerId(Integer brokerId) { + this.brokerId = brokerId; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + @Override + public String toString() { + return "ControllerPreferredBrokerVO{" + + "brokerId=" + brokerId + + ", host='" + host + '\'' + + ", startTime=" + startTime + + ", status=" + status + + '}'; + } +} diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java index 1d4bce26f6b2eaeda4e0554ddea918350e9b98c0..d9724065c7e5df827db63130de65bf512e812af3 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/JsonUtils.java @@ -9,6 +9,7 @@ import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.TopicConnectionDO import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Date; import java.util.List; /** @@ -52,7 +53,7 @@ public class JsonUtils { return JSON.toJSONString(obj); } - public static List parseTopicConnections(Long clusterId, JSONObject jsonObject) { + public static List parseTopicConnections(Long clusterId, JSONObject jsonObject, long postTime) { List connectionDOList = new ArrayList<>(); for (String clientType: jsonObject.keySet()) { JSONObject topicObject = jsonObject.getJSONObject(clientType); @@ -73,6 +74,7 @@ public class JsonUtils { connectionDO.setClusterId(clusterId); connectionDO.setTopicName(topicName); connectionDO.setType(clientType); + connectionDO.setCreateTime(new Date(postTime)); connectionDOList.add(connectionDO); } } diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java index 9b202ec6bf0f5f8065af1185d51ef536e5ad0d8b..1ece8f9ff706b79bca1ce088b043c20203513dc4 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/utils/ValidateUtils.java @@ -1,7 +1,5 @@ package com.xiaojukeji.kafka.manager.common.utils; -import com.xiaojukeji.kafka.manager.common.bizenum.IDCEnum; -import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import org.apache.commons.lang.StringUtils; import java.util.List; @@ -83,23 +81,4 @@ public class ValidateUtils { public static boolean isNullOrLessThanZero(Double value) { return value == null || value < 0; } - - public static boolean topicNameLegal(String idc, String topicName) { - if (ValidateUtils.isNull(idc) || ValidateUtils.isNull(topicName)) { - return false; - } - - // 校验Topic的长度 - if (topicName.length() >= TopicCreationConstant.TOPIC_NAME_MAX_LENGTH) { - return false; - } - - // 校验前缀 - if (IDCEnum.CN.getIdc().equals(idc) || - (IDCEnum.US.getIdc().equals(idc) && topicName.startsWith(TopicCreationConstant.TOPIC_NAME_PREFIX_US)) || - (IDCEnum.RU.getIdc().equals(idc) && topicName.startsWith(TopicCreationConstant.TOPIC_NAME_PREFIX_RU))) { - return true; - } - return false; - } } \ No newline at end of file diff --git a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java index 464dba7bf39369a77d6b92586bdffb12400c7599..e0a5632afda1a45f4b7195a58c8d51fdee22a8da 100644 --- a/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java +++ b/kafka-manager-common/src/main/java/com/xiaojukeji/kafka/manager/common/zookeeper/ZkPathUtil.java @@ -18,6 +18,8 @@ public class ZkPathUtil { public static final String CONSUMER_ROOT_NODE = ZOOKEEPER_SEPARATOR + "consumers"; + public static final String REASSIGN_PARTITIONS_ROOT_NODE = "/admin/reassign_partitions"; + /** * config */ @@ -27,11 +29,11 @@ public class ZkPathUtil { public static final String CONFIG_CLIENTS_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "clients"; - public static final String CONFIG_ENTITY_CHANGES_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "changes/config_change_"; + public static final String CONFIG_ENTITY_CHANGES_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "changes/config_change_"; - public static final String REASSIGN_PARTITIONS_ROOT_NODE = "/admin/reassign_partitions"; + private static final String D_METRICS_CONFIG_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "KafkaExMetrics"; - private static final String D_METRICS_CONFIG_ROOT_NODE = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "KafkaExMetrics"; + public static final String D_CONTROLLER_CANDIDATES = CONFIG_ROOT_NODE + ZOOKEEPER_SEPARATOR + "extension/candidates"; public static String getBrokerIdNodePath(Integer brokerId) { return BROKER_IDS_ROOT + ZOOKEEPER_SEPARATOR + String.valueOf(brokerId); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/ConsumerMetadataCache.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/ConsumerMetadataCache.java index 577540634c382ecee9fd11e1882a3278df5757be..41fd0092d7987230270ffb10debafcd190e2dfb3 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/ConsumerMetadataCache.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/ConsumerMetadataCache.java @@ -92,20 +92,4 @@ public class ConsumerMetadataCache { } return consumerMetadata.getTopicNameConsumerGroupMap().getOrDefault(topicName, new HashSet<>()); } - - public static Map> getConsumerGroupAppIdListInZk(Long clusterId) { - ConsumerMetadata consumerMetadata = CG_METADATA_IN_ZK_MAP.get(clusterId); - if(consumerMetadata == null){ - return new HashMap<>(0); - } - return consumerMetadata.getConsumerGroupAppMap(); - } - - public static Map> getConsumerGroupAppIdListInBK(Long clusterId) { - ConsumerMetadata consumerMetadata = CG_METADATA_IN_BK_MAP.get(clusterId); - if(consumerMetadata == null){ - return new HashMap<>(0); - } - return consumerMetadata.getConsumerGroupAppMap(); - } } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/LogicalClusterMetadataManager.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/LogicalClusterMetadataManager.java index 07c8187853ee8e1b0fac13bab0521fb74b8fb20d..72bdcb76ce68862de70156ba880e8d903d93c4c5 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/LogicalClusterMetadataManager.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/LogicalClusterMetadataManager.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.service.cache; +import com.google.common.collect.Sets; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.LogicalClusterDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.RegionDO; @@ -15,6 +16,7 @@ import org.springframework.stereotype.Service; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; /** * 逻辑集群元信息 @@ -144,9 +146,16 @@ public class LogicalClusterMetadataManager { @Scheduled(cron="0/30 * * * * ?") public void flush() { List logicalClusterDOList = logicalClusterService.listAll(); - if (ValidateUtils.isEmptyList(logicalClusterDOList)) { - return; + if (ValidateUtils.isNull(logicalClusterDOList)) { + logicalClusterDOList = Collections.EMPTY_LIST; } + Set inDbLogicalClusterIds = logicalClusterDOList.stream() + .map(LogicalClusterDO::getId) + .collect(Collectors.toSet()); + + // inCache 和 inDb 取差集，差集结果为已删除的、新增的. + Sets.SetView diffLogicalClusterIds = Sets.difference(LOGICAL_CLUSTER_MAP.keySet(), inDbLogicalClusterIds); + diffLogicalClusterIds.forEach(logicalClusterId -> delLogicalClusterInCache(logicalClusterId)); Map regionMap = new HashMap<>(); List regionDOList = regionService.listAll(); @@ -197,4 +206,11 @@ public class LogicalClusterMetadataManager { } TOPIC_LOGICAL_MAP.put(logicalClusterDO.getClusterId(), subMap); } + + private void delLogicalClusterInCache(Long logicalClusterId) { + LOGICAL_CLUSTER_ID_TOPIC_NAME_MAP.remove(logicalClusterId); + LOGICAL_CLUSTER_ID_BROKER_ID_MAP.remove(logicalClusterId); + LOGICAL_CLUSTER_MAP.remove(logicalClusterId); + TOPIC_LOGICAL_MAP.remove(logicalClusterId); + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java index 3c68b30fd3e1a8d16fba9465b6aee65f1baa1f2d..345f7b9c1d9720f2cc316178cec20884805f7daf 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/cache/PhysicalClusterMetadataManager.java @@ -13,6 +13,8 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; import com.xiaojukeji.kafka.manager.dao.ControllerDao; import com.xiaojukeji.kafka.manager.common.utils.jmx.JmxConnectorWrap; +import com.xiaojukeji.kafka.manager.dao.TopicDao; +import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.service.JmxService; import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils; import com.xiaojukeji.kafka.manager.service.zookeeper.*; @@ -48,6 +50,12 @@ public class PhysicalClusterMetadataManager { @Autowired private ConfigUtils configUtils; + @Autowired + private TopicDao topicDao; + + @Autowired + private AuthorityDao authorityDao; + private final static Map CLUSTER_MAP = new ConcurrentHashMap<>(); private final static Map CONTROLLER_DATA_MAP = new ConcurrentHashMap<>(); @@ -116,7 +124,7 @@ public class PhysicalClusterMetadataManager { zkConfig.watchChildren(ZkPathUtil.BROKER_IDS_ROOT, brokerListener); //增加Topic监控 - TopicStateListener topicListener = new TopicStateListener(clusterDO.getId(), zkConfig); + TopicStateListener topicListener = new TopicStateListener(clusterDO.getId(), zkConfig, topicDao, authorityDao); topicListener.init(); zkConfig.watchChildren(ZkPathUtil.BROKER_TOPICS_ROOT, topicListener); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/AdminService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/AdminService.java index b17d1b1475ccdef874f04ffe146231bdfc80cfbb..78af8316977a3e15a5431304d58e5332e7ae6da7 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/AdminService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/AdminService.java @@ -9,6 +9,19 @@ import java.util.List; import java.util.Properties; public interface AdminService { + /** + * 创建Topic + * @param clusterDO 集群DO + * @param topicDO TopicDO + * @param partitionNum 分区数 + * @param replicaNum 副本数 + * @param regionId RegionID + * @param brokerIdList BrokerId + * @param properties Topic属性 + * @param applicant 申请人 + * @param operator 操作人 + * @return 操作状态 + */ ResultStatus createTopic(ClusterDO clusterDO, TopicDO topicDO, Integer partitionNum, @@ -19,19 +32,86 @@ public interface AdminService { String applicant, String operator); + /** + * 删除Topic + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @param operator 操作人 + * @return 操作状态 + */ ResultStatus deleteTopic(ClusterDO clusterDO, String topicName, String operator); + /** + * 优先副本选举状态 + * @param clusterDO 集群DO + * @return 任务状态 + */ TaskStatusEnum preferredReplicaElectionStatus(ClusterDO clusterDO); + /** + * 集群纬度优先副本选举 + * @param clusterDO 集群DO + * @return 任务状态 + */ ResultStatus preferredReplicaElection(ClusterDO clusterDO, String operator); + /** + * Broker纬度优先副本选举 + * @param clusterDO 集群DO + * @param brokerId BrokerID + * @param operator 操作人 + * @return 任务状态 + */ ResultStatus preferredReplicaElection(ClusterDO clusterDO, Integer brokerId, String operator); + /** + * Topic纬度优先副本选举 + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @param operator 操作人 + * @return 任务状态 + */ + ResultStatus preferredReplicaElection(ClusterDO clusterDO, String topicName, String operator); + + /** + * 分区纬度优先副本选举 + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @param partitionId 分区ID + * @param operator 操作人 + * @return 任务状态 + */ + ResultStatus preferredReplicaElection(ClusterDO clusterDO, String topicName, Integer partitionId, String operator); + + /** + * Topic扩分区 + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @param partitionNum 新增？ 分区数 + * @param regionId RegionID + * @param brokerIdList 集群ID + * @param operator 操作人 + * @return 任务状态 + */ ResultStatus expandPartitions(ClusterDO clusterDO, String topicName, Integer partitionNum, Long regionId, List brokerIdList, String operator); + /** + * 获取Topic配置 + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @return 任务状态 + */ Properties getTopicConfig(ClusterDO clusterDO, String topicName); + /** + * 修改Topic配置 + * @param clusterDO 集群DO + * @param topicName Topic名称 + * @param properties 新的属性 + * @param operator 操作人 + * @return 任务状态 + */ ResultStatus modifyTopicConfig(ClusterDO clusterDO, String topicName, Properties properties, String operator); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java index 1ee2f05228d297b9a8683da7b63994936a12d333..004a3f51ed254e853e397013000d26be53c54e02 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ClusterService.java @@ -1,7 +1,9 @@ package com.xiaojukeji.kafka.manager.service.service; +import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.ClusterDetailDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ControllerPreferredCandidate; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.ClusterNameDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterMetricsDO; @@ -43,5 +45,10 @@ public interface ClusterService { ResultStatus deleteById(Long clusterId); - ClusterDO selectSuitableCluster(Long clusterId, String dataCenter); + /** + * 获取优先被选举为controller的broker + * @param clusterId 集群ID + * @return void + */ + Result> getControllerPreferredCandidates(Long clusterId); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ConsumerService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ConsumerService.java index 24836d299821034aac66fc75ad66400b5b8018a8..3eab40b8880b9e0aa6eff9392c0f89efad0ed097 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ConsumerService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ConsumerService.java @@ -2,14 +2,14 @@ package com.xiaojukeji.kafka.manager.service.service; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetLocationEnum; import com.xiaojukeji.kafka.manager.common.entity.Result; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupSummary; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import java.util.List; import java.util.Map; -import java.util.Set; /** * consumer相关的服务接口 @@ -20,33 +20,36 @@ public interface ConsumerService { /** * 获取消费组列表 */ - List getConsumerGroupList(Long clusterId); + List getConsumerGroupList(Long clusterId); /** * 查询消费Topic的消费组 */ - List getConsumerGroupList(Long clusterId, String topicName); + List getConsumerGroupList(Long clusterId, String topicName); + + /** + * 获取消费Topic的消费组概要信息 + */ + List getConsumerGroupSummaries(Long clusterId, String topicName); /** * 查询消费详情 */ - List getConsumeDetail(ClusterDO clusterDO, String topicName, ConsumerGroupDTO consumerGroupDTO); + List getConsumeDetail(ClusterDO clusterDO, String topicName, ConsumerGroup consumerGroup); /** * 获取消费组消费的Topic列表 */ List getConsumerGroupConsumedTopicList(Long clusterId, String consumerGroup, String location); - Map getConsumerOffset(ClusterDO clusterDO, - String topicName, - ConsumerGroupDTO consumerGroupDTO); + Map getConsumerOffset(ClusterDO clusterDO, String topicName, ConsumerGroup consumerGroup); /** * 重置offset */ List resetConsumerOffset(ClusterDO clusterDO, String topicName, - ConsumerGroupDTO consumerGroupDTO, + ConsumerGroup consumerGroup, List partitionOffsetDTOList); Map getConsumerGroupNumMap(List clusterDOList); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java index 382770afce37313e65ad9698534ebc0498176351..5a1fc11b6b875407cfde31f97ed3d7c68d8dbb95 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/TopicManagerService.java @@ -66,6 +66,19 @@ public interface TopicManagerService { */ ResultStatus modifyTopic(Long clusterId, String topicName, String description, String operator); + /** + * 修改Topic + * @param clusterId 集群ID + * @param topicName Topic名称 + * @param appId 所属应用 + * @param description 备注 + * @param operator 操作人 + * @author zengqiao + * @date 20/5/12 + * @return ResultStatus + */ + ResultStatus modifyTopicByOp(Long clusterId, String topicName, String appId, String description, String operator); + /** * 通过topictopic名称删除 * @param clusterId 集群id diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java index 00f1ea38e2a9c449e06e2dc882a31add0807686c..d24b2d24324dc82b5a2027fa2051ecc9965cedf3 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/ZookeeperService.java @@ -3,11 +3,27 @@ package com.xiaojukeji.kafka.manager.service.service; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.didi.TopicJmxSwitch; +import java.util.List; + /** * ZK相关的接口 * @author tukun * @date 2015/11/11. */ public interface ZookeeperService { + /** + * 开启JMX + * @param clusterId 集群ID + * @param topicName Topic名称 + * @param jmxSwitch JMX开关 + * @return 操作结果 + */ Result openTopicJmx(Long clusterId, String topicName, TopicJmxSwitch jmxSwitch); + + /** + * 获取优先被选举为controller的broker + * @param clusterId 集群ID + * @return 操作结果 + */ + Result> getControllerPreferredCandidates(Long clusterId); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AuthorityService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AuthorityService.java index 26bf9ba0c264b8dfc1f134da738eefa1346c85c5..6a19d84e8ce64da072384bae71e97ab03fe0c65f 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AuthorityService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/AuthorityService.java @@ -60,4 +60,6 @@ public interface AuthorityService { int addAuthorityAndQuota(AuthorityDO authorityDO, TopicQuota quota); Map>> getAllAuthority(); + + int deleteAuthorityByTopic(Long clusterId, String topicName); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/GatewayConfigService.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/GatewayConfigService.java index c67cf2401be984512e64025b951521ae48519c2f..af3a8304c6029631d7316358be953df566a767dc 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/GatewayConfigService.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/GatewayConfigService.java @@ -1,18 +1,86 @@ package com.xiaojukeji.kafka.manager.service.service.gateway; +import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.*; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; +import java.util.List; + public interface GatewayConfigService { + /** + * 获取集群服务地址 + * @param requestVersion 请求的版本 + * @return + */ KafkaBootstrapServerConfig getKafkaBootstrapServersConfig(Long requestVersion); + /** + * 获取服务发现的请求队列的配置 + * @param requestVersion 请求的版本 + * @return + */ RequestQueueConfig getRequestQueueConfig(Long requestVersion); + /** + * 获取服务发现的App请求速度的配置 + * @param requestVersion 请求的版本 + * @return + */ AppRateConfig getAppRateConfig(Long requestVersion); + /** + * 获取服务发现的IP请求速度的配置 + * @param requestVersion 请求的版本 + * @return + */ IpRateConfig getIpRateConfig(Long requestVersion); + /** + * 获取服务发现的具体IP或者应用纬度的限速配置 + * @param requestVersion 请求的版本 + * @return + */ SpRateConfig getSpRateConfig(Long requestVersion); + /** + * 获取配置 + * @param configType 配置类型 + * @param configName 配置名称 + * @return + */ GatewayConfigDO getByTypeAndName(String configType, String configName); + + /** + * 获取配置 + * @return + */ + List list(); + + /** + * 新建配置 + * @param gatewayConfigDO 配置信息 + * @return + */ + Result insert(GatewayConfigDO gatewayConfigDO); + + /** + * 删除配置 + * @param id 配置ID + * @return + */ + Result deleteById(Long id); + + /** + * 更新配置 + * @param gatewayConfigDO 配置信息 + * @return + */ + Result updateById(GatewayConfigDO gatewayConfigDO); + + /** + * 获取配置 + * @param id 配置ID + * @return + */ + GatewayConfigDO getById(Long id); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java index eada7df56150d0eeda3b247d62b70df238643fe8..09b4a0710785483f01f7b92af29d351f3e248962 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AppServiceImpl.java @@ -196,8 +196,7 @@ public class AppServiceImpl implements AppService { } @Override - public List getAppTopicDTOList(String appId, - Boolean mine) { + public List getAppTopicDTOList(String appId, Boolean mine) { // 查询AppID AppDO appDO = appDao.getByAppId(appId); if (ValidateUtils.isNull(appDO)) { @@ -223,13 +222,17 @@ public class AppServiceImpl implements AppService { TopicDO topicDO = topicMap .getOrDefault(authorityDO.getClusterId(), new HashMap<>()) .get(authorityDO.getTopicName()); + + if (ValidateUtils.isNull(topicDO)) { + continue; + } + if (Boolean.TRUE.equals(mine) - && (ValidateUtils.isNull(topicDO) || !topicDO.getAppId().equals(appId))) { + && !topicDO.getAppId().equals(appId)) { continue; } if (Boolean.FALSE.equals(mine) - && !ValidateUtils.isNull(topicDO) && topicDO.getAppId().equals(appId)) { continue; } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AuthorityServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AuthorityServiceImpl.java index 58be1bcbd312908bb25af3dac942b993b0102a93..4f804107fa2137a328cc2e233499afa4841d1599 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AuthorityServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/AuthorityServiceImpl.java @@ -192,4 +192,10 @@ public class AuthorityServiceImpl implements AuthorityService { public Map>> getAllAuthority() { return authorityDao.getAllAuthority(); } + + @Override + public int deleteAuthorityByTopic(Long clusterId, String topicName) { + return authorityDao.deleteAuthorityByTopic(clusterId, topicName); + } + } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java index 3db556a592b383581f68a2ea04afe8bbf2d06c74..fce7b605f9bfb45110826c53ddee39ef1b2ea5e8 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/gateway/impl/GatewayConfigServiceImpl.java @@ -2,6 +2,8 @@ package com.xiaojukeji.kafka.manager.service.service.gateway.impl; import com.alibaba.fastjson.JSON; import com.xiaojukeji.kafka.manager.common.bizenum.gateway.GatewayConfigKeyEnum; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.gateway.*; import com.xiaojukeji.kafka.manager.common.utils.ListUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; @@ -13,6 +15,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -21,7 +24,7 @@ import java.util.Map; * @author zengqiao * @date 20/7/28 */ -@Service("gatewayConfigService") +@Service public class GatewayConfigServiceImpl implements GatewayConfigService { private final Logger LOGGER = LoggerFactory.getLogger(GatewayConfigServiceImpl.class); @@ -52,7 +55,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { ? new KafkaBootstrapServerConfig(maxVersion, clusterIdBootstrapServersMap) : new KafkaBootstrapServerConfig(requestVersion, new HashMap<>(0)); } catch (Exception e) { - LOGGER.error("get kafka bootstrap servers config failed, data:{}.", JSON.toJSONString(doList), e); + LOGGER.error("class=GatewayConfigServiceImpl||method=getKafkaBootstrapServersConfig||data={}||errMsg={}||msg=get kafka bootstrap servers config failed", + JSON.toJSONString(doList), e.getMessage()); } return null; } @@ -71,7 +75,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { return new RequestQueueConfig(configDO.getVersion(), Long.valueOf(configDO.getValue())); } catch (Exception e) { - LOGGER.error("get request queue config failed, data:{}.", JSON.toJSONString(configDO), e); + LOGGER.error("class=GatewayConfigServiceImpl||method=getRequestQueueConfig||data={}||errMsg={}||msg=get request queue config failed", + JSON.toJSONString(configDO), e.getMessage()); } return null; } @@ -90,7 +95,8 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { return new AppRateConfig(configDO.getVersion(), Long.valueOf(configDO.getValue())); } catch (Exception e) { - LOGGER.error("get app rate config failed, data:{}.", JSON.toJSONString(configDO), e); + LOGGER.error("class=GatewayConfigServiceImpl||method=getAppRateConfig||data={}||errMsg={}||msg=get app rate config failed", + JSON.toJSONString(configDO), e.getMessage()); } return null; } @@ -153,4 +159,94 @@ public class GatewayConfigServiceImpl implements GatewayConfigService { } return null; } + + @Override + public List list() { + try { + return gatewayConfigDao.list(); + } catch (Exception e) { + LOGGER.debug("class=GatewayConfigServiceImpl||method=list||errMsg={}||msg=list failed", e.getMessage()); + } + return new ArrayList<>(); + } + + @Override + public Result insert(GatewayConfigDO gatewayConfigDO) { + try { + GatewayConfigKeyEnum configKeyEnum = GatewayConfigKeyEnum.getByConfigType(gatewayConfigDO.getType()); + if (ValidateUtils.isNull(configKeyEnum) + && ValidateUtils.isBlank(gatewayConfigDO.getName()) + && ValidateUtils.isBlank(gatewayConfigDO.getValue())) { + // 参数错误 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + // 获取当前同类配置, 插入之后需要增大这个version + List gatewayConfigDOList = gatewayConfigDao.getByConfigType(gatewayConfigDO.getType()); + Long version = 1L; + for (GatewayConfigDO elem: gatewayConfigDOList) { + if (elem.getVersion() > version) { + version = elem.getVersion() + 1L; + } + } + + gatewayConfigDO.setVersion(version); + if (gatewayConfigDao.insert(gatewayConfigDO) > 0) { + return Result.buildSuc(); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } catch (Exception e) { + LOGGER.debug("class=GatewayConfigServiceImpl||method=insert||data={}||errMsg={}||msg=insert failed", gatewayConfigDO, e.getMessage()); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } + + @Override + public Result deleteById(Long id) { + try { + if (gatewayConfigDao.deleteById(id) > 0) { + return Result.buildSuc(); + } + return Result.buildFrom(ResultStatus.RESOURCE_NOT_EXIST); + } catch (Exception e) { + LOGGER.debug("class=GatewayConfigServiceImpl||method=deleteById||id={}||errMsg={}||msg=delete failed", id, e.getMessage()); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } + + @Override + public Result updateById(GatewayConfigDO newGatewayConfigDO) { + try { + GatewayConfigDO oldGatewayConfigDO = this.getById(newGatewayConfigDO.getId()); + if (ValidateUtils.isNull(oldGatewayConfigDO)) { + return Result.buildFrom(ResultStatus.RESOURCE_NOT_EXIST); + } + if (!oldGatewayConfigDO.getName().equals(newGatewayConfigDO.getName()) + || !oldGatewayConfigDO.getType().equals(newGatewayConfigDO.getType()) + || ValidateUtils.isBlank(newGatewayConfigDO.getValue())) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + newGatewayConfigDO.setVersion(oldGatewayConfigDO.getVersion() + 1); + if (gatewayConfigDao.updateById(oldGatewayConfigDO) > 0) { + return Result.buildSuc(); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } catch (Exception e) { + LOGGER.debug("class=GatewayConfigServiceImpl||method=updateById||data={}||errMsg={}||msg=update failed", newGatewayConfigDO, e.getMessage()); + } + return Result.buildFrom(ResultStatus.MYSQL_ERROR); + } + + @Override + public GatewayConfigDO getById(Long id) { + if (ValidateUtils.isNull(id)) { + return null; + } + try { + return gatewayConfigDao.getById(id); + } catch (Exception e) { + LOGGER.debug("class=GatewayConfigServiceImpl||method=getById||id={}||errMsg={}||msg=get failed", id, e.getMessage()); + } + return null; + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java index 0da0cad9e7544a59d7c79c74127a8146348c637c..b49e41a38b988970a0d4a1f5b354960aa3003dd0 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/AdminServiceImpl.java @@ -13,6 +13,7 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; +import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; @@ -139,6 +140,9 @@ public class AdminServiceImpl implements AdminService { // 3. 数据库中删除topic topicManagerService.deleteByTopicName(clusterDO.getId(), topicName); + + // 4. 数据库中删除authority + authorityService.deleteAuthorityByTopic(clusterDO.getId(), topicName); return rs; } @@ -191,15 +195,55 @@ public class AdminServiceImpl implements AdminService { @Override public ResultStatus preferredReplicaElection(ClusterDO clusterDO, Integer brokerId, String operator) { BrokerMetadata brokerMetadata = PhysicalClusterMetadataManager.getBrokerMetadata(clusterDO.getId(), brokerId); - if (null == brokerMetadata) { + if (ValidateUtils.isNull(brokerMetadata)) { return ResultStatus.PARAM_ILLEGAL; } + + Map> partitionMap = topicService.getTopicPartitionIdMap(clusterDO.getId(), brokerId); + if (ValidateUtils.isEmptyMap(partitionMap)) { + return ResultStatus.SUCCESS; + } + + return preferredReplicaElection(clusterDO, partitionMap, operator); + } + + @Override + public ResultStatus preferredReplicaElection(ClusterDO clusterDO, String topicName, String operator) { + TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterDO.getId(), topicName); + if (ValidateUtils.isNull(topicMetadata)) { + return ResultStatus.TOPIC_NOT_EXIST; + } + + Map> partitionMap = new HashMap<>(); + partitionMap.put(topicName, new ArrayList<>(topicMetadata.getPartitionMap().getPartitions().keySet())); + + return preferredReplicaElection(clusterDO, partitionMap, operator); + } + + @Override + public ResultStatus preferredReplicaElection(ClusterDO clusterDO, String topicName, Integer partitionId, String operator) { + TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterDO.getId(), topicName); + if (ValidateUtils.isNull(topicMetadata)) { + return ResultStatus.TOPIC_NOT_EXIST; + } + + if (!topicMetadata.getPartitionMap().getPartitions().containsKey(partitionId)) { + return ResultStatus.PARTITION_NOT_EXIST; + } + + Map> partitionMap = new HashMap<>(); + partitionMap.put(topicName, Arrays.asList(partitionId)); + + return preferredReplicaElection(clusterDO, partitionMap, operator); + } + + private ResultStatus preferredReplicaElection(ClusterDO clusterDO, Map> partitionMap, String operator) { + if (ValidateUtils.isEmptyMap(partitionMap)) { + return ResultStatus.SUCCESS; + } + ZkUtils zkUtils = null; try { - Map> partitionMap = topicService.getTopicPartitionIdMap(clusterDO.getId(), brokerId); - if (partitionMap == null || partitionMap.isEmpty()) { - return ResultStatus.SUCCESS; - } String preferredReplicaElectString = convert2preferredReplicaElectString(partitionMap); zkUtils = ZkUtils.apply(clusterDO.getZookeeper(), diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java index 49c16c5dfc8fddc21148e5256f6feda27a1e557a..9f9727e108ecf5b006c50f8e3b0593add89455b6 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ClusterServiceImpl.java @@ -1,11 +1,16 @@ package com.xiaojukeji.kafka.manager.service.service.impl; +import com.xiaojukeji.kafka.manager.common.bizenum.DBStatusEnum; +import com.xiaojukeji.kafka.manager.common.constant.Constant; +import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.ClusterDetailDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ControllerPreferredCandidate; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.ClusterNameDTO; import com.xiaojukeji.kafka.manager.common.utils.ListUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.BrokerMetadata; import com.xiaojukeji.kafka.manager.dao.ClusterDao; import com.xiaojukeji.kafka.manager.dao.ClusterMetricsDao; import com.xiaojukeji.kafka.manager.dao.ControllerDao; @@ -14,6 +19,7 @@ import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager import com.xiaojukeji.kafka.manager.service.service.ClusterService; import com.xiaojukeji.kafka.manager.service.service.ConsumerService; import com.xiaojukeji.kafka.manager.service.service.RegionService; +import com.xiaojukeji.kafka.manager.service.service.ZookeeperService; import com.xiaojukeji.kafka.manager.service.utils.ConfigUtils; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; @@ -57,6 +63,9 @@ public class ClusterServiceImpl implements ClusterService { @Autowired private ConfigUtils configUtils; + @Autowired + private ZookeeperService zookeeperService; + @Override public ResultStatus addNew(ClusterDO clusterDO, String operator) { if (ValidateUtils.isNull(clusterDO) || ValidateUtils.isNull(operator)) { @@ -262,21 +271,6 @@ public class ClusterServiceImpl implements ClusterService { return ResultStatus.SUCCESS; } - @Override - public ClusterDO selectSuitableCluster(Long clusterId, String dataCenter) { - if (!ValidateUtils.isNullOrLessThanZero(clusterId)) { - return getById(clusterId); - } - if (ValidateUtils.isBlank(dataCenter)) { - return null; - } - List clusterDOList = this.listAll(); - if (ValidateUtils.isEmptyList(clusterDOList)) { - return null; - } - return clusterDOList.get(0); - } - private ClusterDetailDTO getClusterDetailDTO(ClusterDO clusterDO, Boolean needDetail) { if (ValidateUtils.isNull(clusterDO)) { return null; @@ -300,4 +294,31 @@ public class ClusterServiceImpl implements ClusterService { dto.setControllerId(PhysicalClusterMetadataManager.getControllerId(clusterDO.getId())); return dto; } + + @Override + public Result> getControllerPreferredCandidates(Long clusterId) { + Result> candidateResult = zookeeperService.getControllerPreferredCandidates(clusterId); + if (candidateResult.failed()) { + return new Result<>(candidateResult.getCode(), candidateResult.getMessage()); + } + if (ValidateUtils.isEmptyList(candidateResult.getData())) { + return Result.buildSuc(new ArrayList<>()); + } + + List controllerPreferredCandidateList = new ArrayList<>(); + for (Integer brokerId: candidateResult.getData()) { + ControllerPreferredCandidate controllerPreferredCandidate = new ControllerPreferredCandidate(); + controllerPreferredCandidate.setBrokerId(brokerId); + BrokerMetadata brokerMetadata = PhysicalClusterMetadataManager.getBrokerMetadata(clusterId, brokerId); + if (ValidateUtils.isNull(brokerMetadata)) { + controllerPreferredCandidate.setStatus(DBStatusEnum.DEAD.getStatus()); + } else { + controllerPreferredCandidate.setHost(brokerMetadata.getHost()); + controllerPreferredCandidate.setStartTime(brokerMetadata.getTimestamp()); + controllerPreferredCandidate.setStatus(DBStatusEnum.ALIVE.getStatus()); + } + controllerPreferredCandidateList.add(controllerPreferredCandidate); + } + return Result.buildSuc(controllerPreferredCandidateList); + } } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java index fcfc72ece69eba752313b049860c646327257789..e228d36c28409a71d70c5282a840752e6a63474d 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ConsumerServiceImpl.java @@ -2,13 +2,14 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetLocationEnum; -import com.xiaojukeji.kafka.manager.common.bizenum.SinkMonitorSystemEnum; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupSummary; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; +import com.xiaojukeji.kafka.manager.common.utils.ListUtils; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.exception.ConfigException; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; @@ -23,6 +24,7 @@ import kafka.admin.AdminClient; import org.apache.commons.lang.StringUtils; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.types.SchemaException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -44,70 +46,116 @@ public class ConsumerServiceImpl implements ConsumerService { private TopicService topicService; @Override - public List getConsumerGroupList(Long clusterId) { - List consumerGroupDTOList = new ArrayList<>(); + public List getConsumerGroupList(Long clusterId) { + List consumerGroupList = new ArrayList<>(); for (OffsetLocationEnum location: OffsetLocationEnum.values()) { - Map> consumerGroupAppIdMap = null; Set consumerGroupSet = null; if (OffsetLocationEnum.ZOOKEEPER.equals(location)) { // 获取ZK中的消费组 - consumerGroupAppIdMap = ConsumerMetadataCache.getConsumerGroupAppIdListInZk(clusterId); consumerGroupSet = ConsumerMetadataCache.getGroupInZkMap(clusterId); } else if (OffsetLocationEnum.BROKER.equals(location)) { // 获取Broker中的消费组 - consumerGroupAppIdMap = ConsumerMetadataCache.getConsumerGroupAppIdListInBK(clusterId); consumerGroupSet = ConsumerMetadataCache.getGroupInBrokerMap(clusterId); } - if (consumerGroupSet == null || consumerGroupAppIdMap == null) { + if (ValidateUtils.isEmptySet(consumerGroupSet)) { continue; } for (String consumerGroup : consumerGroupSet) { - consumerGroupDTOList.add(new ConsumerGroupDTO( - clusterId, - consumerGroup, - consumerGroupAppIdMap.getOrDefault(consumerGroup, new ArrayList<>()), - location) - ); } + consumerGroupList.add(new ConsumerGroup(clusterId, consumerGroup, location)); + } } - return consumerGroupDTOList; + return consumerGroupList; } @Override - public List getConsumerGroupList(Long clusterId, String topicName) { - List consumerGroupDTOList = new ArrayList<>(); + public List getConsumerGroupList(Long clusterId, String topicName) { + List consumerGroupList = new ArrayList<>(); for (OffsetLocationEnum location: OffsetLocationEnum.values()) { - Map> consumerGroupAppIdMap = null; Set consumerGroupSet = null; if (OffsetLocationEnum.ZOOKEEPER.equals(location)) { // 获取ZK中的消费组 - consumerGroupAppIdMap = ConsumerMetadataCache.getConsumerGroupAppIdListInZk(clusterId); consumerGroupSet = ConsumerMetadataCache.getTopicConsumerGroupInZk(clusterId, topicName); } else if (OffsetLocationEnum.BROKER.equals(location)) { // 获取Broker中的消费组 - consumerGroupAppIdMap = ConsumerMetadataCache.getConsumerGroupAppIdListInBK(clusterId); consumerGroupSet = ConsumerMetadataCache.getTopicConsumerGroupInBroker(clusterId, topicName); } - if (consumerGroupSet == null || consumerGroupAppIdMap == null) { + if (ValidateUtils.isEmptySet(consumerGroupSet)) { continue; } for (String consumerGroup : consumerGroupSet) { - consumerGroupDTOList.add(new ConsumerGroupDTO( - clusterId, - consumerGroup, - consumerGroupAppIdMap.getOrDefault(consumerGroup, new ArrayList<>()), - location - ) - ); + consumerGroupList.add(new ConsumerGroup(clusterId, consumerGroup, location)); + } + } + return consumerGroupList; + } + + @Override + public List getConsumerGroupSummaries(Long clusterId, String topicName) { + List consumerGroupList = this.getConsumerGroupList(clusterId, topicName); + if (ValidateUtils.isEmptyList(consumerGroupList)) { + return Collections.emptyList(); + } + + List summaryList = new ArrayList<>(); + for (ConsumerGroup consumerGroup: consumerGroupList) { + ConsumerGroupSummary consumerGroupSummary = null; + if (OffsetLocationEnum.ZOOKEEPER.equals(consumerGroup.getOffsetStoreLocation())) { + consumerGroupSummary = new ConsumerGroupSummary(); + consumerGroupSummary.setClusterId(consumerGroup.getClusterId()); + consumerGroupSummary.setConsumerGroup(consumerGroup.getConsumerGroup()); + consumerGroupSummary.setOffsetStoreLocation(consumerGroup.getOffsetStoreLocation()); + } else { + consumerGroupSummary = getConsumerGroupSummary(clusterId, topicName, consumerGroup.getConsumerGroup()); + } + summaryList.add(consumerGroupSummary); + } + return summaryList; + } + + private ConsumerGroupSummary getConsumerGroupSummary(Long clusterId, String topicName, String consumerGroup) { + ConsumerGroupSummary summary = new ConsumerGroupSummary(); + summary.setClusterId(clusterId); + summary.setConsumerGroup(consumerGroup); + summary.setOffsetStoreLocation(OffsetLocationEnum.BROKER); + summary.setAppIdList(new ArrayList<>()); + summary.setState(""); + try { + AdminClient adminClient = KafkaClientPool.getAdminClient(clusterId); + + AdminClient.ConsumerGroupSummary consumerGroupSummary = adminClient.describeConsumerGroup(consumerGroup); + if (ValidateUtils.isNull(consumerGroupSummary)) { + return summary; + } + summary.setState(consumerGroupSummary.state()); + + java.util.Iterator> it = JavaConversions.asJavaIterator(consumerGroupSummary.consumers().iterator()); + while (it.hasNext()) { + List consumerSummaryList = JavaConversions.asJavaList(it.next()); + for (AdminClient.ConsumerSummary consumerSummary: consumerSummaryList) { + List topicPartitionList = JavaConversions.asJavaList(consumerSummary.assignment()); + if (ValidateUtils.isEmptyList(topicPartitionList)) { + continue; + } + if (topicPartitionList.stream().anyMatch(elem -> elem.topic().equals(topicName)) && consumerSummary.clientId().contains(".")) { + String [] splitArray = consumerSummary.clientId().split("\\."); + summary.getAppIdList().add(splitArray[0]); + } + } } + } catch (SchemaException e) { + logger.error("class=ConsumerServiceImpl||method=getConsumerGroupSummary||clusterId={}||topicName={}||consumerGroup={}||errMsg={}||schema exception", + clusterId, topicName, consumerGroup, e.getMessage()); + } catch (Exception e) { + logger.error("class=ConsumerServiceImpl||method=getConsumerGroupSummary||clusterId={}||topicName={}||consumerGroup={}||errMsg={}||throws exception", + clusterId, topicName, consumerGroup, e.getMessage()); } - return consumerGroupDTOList; + summary.setAppIdList(new ArrayList<>(new HashSet<>(summary.getAppIdList()))); + return summary; } @Override - public List getConsumeDetail(ClusterDO clusterDO, - String topicName, - ConsumerGroupDTO consumeGroupDTO) { + public List getConsumeDetail(ClusterDO clusterDO, String topicName, ConsumerGroup consumerGroup) { TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterDO.getId(), topicName); if (topicMetadata == null) { logger.warn("class=ConsumerServiceImpl||method=getConsumeDetail||clusterId={}||topicName={}||msg=topicMetadata is null!", @@ -116,10 +164,10 @@ public class ConsumerServiceImpl implements ConsumerService { } List consumerGroupDetailDTOList = null; - if (OffsetLocationEnum.ZOOKEEPER.equals(consumeGroupDTO.getOffsetStoreLocation())) { - consumerGroupDetailDTOList = getConsumerPartitionStateInZK(clusterDO, topicMetadata, consumeGroupDTO); - } else if (OffsetLocationEnum.BROKER.equals(consumeGroupDTO.getOffsetStoreLocation())){ - consumerGroupDetailDTOList = getConsumerPartitionStateInBroker(clusterDO, topicMetadata, consumeGroupDTO); + if (OffsetLocationEnum.ZOOKEEPER.equals(consumerGroup.getOffsetStoreLocation())) { + consumerGroupDetailDTOList = getConsumerPartitionStateInZK(clusterDO, topicMetadata, consumerGroup); + } else if (OffsetLocationEnum.BROKER.equals(consumerGroup.getOffsetStoreLocation())){ + consumerGroupDetailDTOList = getConsumerPartitionStateInBroker(clusterDO, topicMetadata, consumerGroup); } if (consumerGroupDetailDTOList == null) { logger.info("class=ConsumerServiceImpl||method=getConsumeDetail||msg=consumerGroupDetailDTOList is null!"); @@ -147,7 +195,7 @@ public class ConsumerServiceImpl implements ConsumerService { } @Override - public List resetConsumerOffset(ClusterDO clusterDO, String topicName, ConsumerGroupDTO consumerGroupDTO, List partitionOffsetDTOList) { + public List resetConsumerOffset(ClusterDO clusterDO, String topicName, ConsumerGroup consumerGroup, List partitionOffsetDTOList) { Map offsetMap = partitionOffsetDTOList.stream().collect(Collectors.toMap(elem -> {return new TopicPartition(topicName, elem.getPartitionId());}, PartitionOffsetDTO::getOffset)); List resultList = new ArrayList<>(); @@ -155,12 +203,12 @@ public class ConsumerServiceImpl implements ConsumerService { KafkaConsumer kafkaConsumer = null; try { Properties properties = KafkaClientPool.createProperties(clusterDO, false); - properties.setProperty("group.id", consumerGroupDTO.getConsumerGroup()); + properties.setProperty("group.id", consumerGroup.getConsumerGroup()); kafkaConsumer = new KafkaConsumer<>(properties); checkAndCorrectPartitionOffset(kafkaConsumer, offsetMap); - return resetConsumerOffset(clusterDO, kafkaConsumer, consumerGroupDTO, offsetMap); + return resetConsumerOffset(clusterDO, kafkaConsumer, consumerGroup, offsetMap); } catch (Exception e) { - logger.error("create kafka consumer failed, clusterId:{} topicName:{} consumerGroup:{} partition:{}.", clusterDO.getId(), topicName, consumerGroupDTO, partitionOffsetDTOList, e); + logger.error("create kafka consumer failed, clusterId:{} topicName:{} consumerGroup:{} partition:{}.", clusterDO.getId(), topicName, consumerGroup, partitionOffsetDTOList, e); resultList.add(new Result( ResultStatus.OPERATION_FAILED.getCode(), "reset failed, create KafkaConsumer or check offset failed" @@ -173,20 +221,20 @@ public class ConsumerServiceImpl implements ConsumerService { return resultList; } - private List resetConsumerOffset(ClusterDO cluster, KafkaConsumer kafkaConsumer, ConsumerGroupDTO consumerGroupDTO, Map offsetMap) { + private List resetConsumerOffset(ClusterDO cluster, KafkaConsumer kafkaConsumer, ConsumerGroup consumerGroup, Map offsetMap) { List resultList = new ArrayList<>(); for(Map.Entry entry: offsetMap.entrySet()){ TopicPartition tp = entry.getKey(); Long offset = entry.getValue(); try { - if (consumerGroupDTO.getOffsetStoreLocation().equals(OffsetLocationEnum.ZOOKEEPER)) { - resetConsumerOffsetInZK(cluster, consumerGroupDTO.getConsumerGroup(), tp, offset); - } else if (consumerGroupDTO.getOffsetStoreLocation().equals(OffsetLocationEnum.BROKER)) { + if (consumerGroup.getOffsetStoreLocation().equals(OffsetLocationEnum.ZOOKEEPER)) { + resetConsumerOffsetInZK(cluster, consumerGroup.getConsumerGroup(), tp, offset); + } else if (consumerGroup.getOffsetStoreLocation().equals(OffsetLocationEnum.BROKER)) { resetConsumerOffsetInBroker(kafkaConsumer, tp, offset); } } catch (Exception e) { - logger.error("reset failed, clusterId:{} consumerGroup:{} topic-partition:{}.", cluster.getId(), consumerGroupDTO, tp, e); + logger.error("reset failed, clusterId:{} consumerGroup:{} topic-partition:{}.", cluster.getId(), consumerGroup, tp, e); resultList.add(new Result( ResultStatus.OPERATION_FAILED.getCode(), "reset failed...")); @@ -232,14 +280,14 @@ public class ConsumerServiceImpl implements ConsumerService { @Override public Map getConsumerOffset(ClusterDO clusterDO, String topicName, - ConsumerGroupDTO consumerGroupDTO) { - if (ValidateUtils.isNull(clusterDO) || ValidateUtils.isBlank(topicName) || ValidateUtils.isNull(consumerGroupDTO)) { + ConsumerGroup consumerGroup) { + if (ValidateUtils.isNull(clusterDO) || ValidateUtils.isBlank(topicName) || ValidateUtils.isNull(consumerGroup)) { return null; } - if (OffsetLocationEnum.BROKER.equals(consumerGroupDTO.getOffsetStoreLocation())) { - return getConsumerOffsetFromBK(clusterDO, topicName, consumerGroupDTO.getConsumerGroup()); - } else if (OffsetLocationEnum.ZOOKEEPER.equals(consumerGroupDTO.getOffsetStoreLocation())) { - return getConsumerOffsetFromZK(clusterDO.getId(), topicName, consumerGroupDTO.getConsumerGroup()); + if (OffsetLocationEnum.BROKER.equals(consumerGroup.getOffsetStoreLocation())) { + return getConsumerOffsetFromBK(clusterDO, topicName, consumerGroup.getConsumerGroup()); + } else if (OffsetLocationEnum.ZOOKEEPER.equals(consumerGroup.getOffsetStoreLocation())) { + return getConsumerOffsetFromZK(clusterDO.getId(), topicName, consumerGroup.getConsumerGroup()); } return null; } @@ -306,9 +354,9 @@ public class ConsumerServiceImpl implements ConsumerService { return consumerIdMap; } - private List getConsumerPartitionStateInBroker(ClusterDO clusterDO, TopicMetadata topicMetadata, ConsumerGroupDTO consumerGroupDTO) { - Map consumerIdMap = getConsumeIdMap(clusterDO.getId(), topicMetadata.getTopic(), consumerGroupDTO.getConsumerGroup()); - Map consumeOffsetMap = getOffsetByGroupAndTopicFromBroker(clusterDO, consumerGroupDTO.getConsumerGroup(), topicMetadata.getTopic()); + private List getConsumerPartitionStateInBroker(ClusterDO clusterDO, TopicMetadata topicMetadata, ConsumerGroup consumerGroup) { + Map consumerIdMap = getConsumeIdMap(clusterDO.getId(), topicMetadata.getTopic(), consumerGroup.getConsumerGroup()); + Map consumeOffsetMap = getOffsetByGroupAndTopicFromBroker(clusterDO, consumerGroup.getConsumerGroup(), topicMetadata.getTopic()); List consumeDetailDTOList = new ArrayList<>(); for (int partitionId : topicMetadata.getPartitionMap().getPartitions().keySet()) { @@ -318,7 +366,7 @@ public class ConsumerServiceImpl implements ConsumerService { try { consumeDetailDTO.setConsumeOffset(StringUtils.isEmpty(consumeOffsetStr)? null: Long.valueOf(consumeOffsetStr)); } catch (Exception e) { - logger.error("illegal consumer offset, clusterId:{} topicName:{} consumerGroup:{} offset:{}.", clusterDO.getId(), topicMetadata.getTopic(), consumerGroupDTO.getConsumerGroup(), consumeOffsetStr, e); + logger.error("illegal consumer offset, clusterId:{} topicName:{} consumerGroup:{} offset:{}.", clusterDO.getId(), topicMetadata.getTopic(), consumerGroup.getConsumerGroup(), consumeOffsetStr, e); } consumeDetailDTO.setConsumerId(consumerIdMap.get(partitionId)); consumeDetailDTOList.add(consumeDetailDTO); @@ -326,21 +374,19 @@ public class ConsumerServiceImpl implements ConsumerService { return consumeDetailDTOList; } - private List getConsumerPartitionStateInZK(ClusterDO clusterDO, - TopicMetadata topicMetadata, - ConsumerGroupDTO consumerGroupDTO) { + private List getConsumerPartitionStateInZK(ClusterDO clusterDO, TopicMetadata topicMetadata, ConsumerGroup consumerGroup) { ZkConfigImpl zkConfig = PhysicalClusterMetadataManager.getZKConfig(clusterDO.getId()); List consumeDetailDTOList = new ArrayList<>(); for (Integer partitionId : topicMetadata.getPartitionMap().getPartitions().keySet()) { - String consumeGroupPath = ZkPathUtil.getConsumerGroupOffsetTopicPartitionNode(consumerGroupDTO.getConsumerGroup(), topicMetadata.getTopic(), partitionId); + String consumeGroupPath = ZkPathUtil.getConsumerGroupOffsetTopicPartitionNode(consumerGroup.getConsumerGroup(), topicMetadata.getTopic(), partitionId); String consumeOffset = null; try { consumeOffset = zkConfig.get(consumeGroupPath); } catch (ConfigException e) { logger.error("get consumeOffset error for zk path:{}", consumeGroupPath, e); } - String consumeIdZkPath = ZkPathUtil.getConsumerGroupOwnersTopicPartitionNode(consumerGroupDTO.getConsumerGroup(), topicMetadata.getTopic(), partitionId); + String consumeIdZkPath = ZkPathUtil.getConsumerGroupOwnersTopicPartitionNode(consumerGroup.getConsumerGroup(), topicMetadata.getTopic(), partitionId); String consumerId = null; try { consumerId = zkConfig.get(consumeIdZkPath); @@ -394,7 +440,7 @@ public class ConsumerServiceImpl implements ConsumerService { @Override public boolean checkConsumerGroupExist(OffsetLocationEnum offsetLocation, Long clusterId, String topicName, String consumerGroup) { - List consumerGroupList = getConsumerGroupList(clusterId, topicName).stream() + List consumerGroupList = getConsumerGroupList(clusterId, topicName).stream() .filter(group -> offsetLocation.location.equals(group.getOffsetStoreLocation().location) && consumerGroup.equals(group.getConsumerGroup())) .collect(Collectors.toList()); return !ValidateUtils.isEmptyList(consumerGroupList); diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java index a953a50b9ab064d2c6cb21a7353ea94dad127520..0b42d068d85ab78f271f8dfa3e1b3c664c1429f3 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicManagerServiceImpl.java @@ -3,6 +3,7 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.bizenum.KafkaClientEnum; import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum; import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; +import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.RdTopicBasic; @@ -14,6 +15,7 @@ import com.xiaojukeji.kafka.manager.common.entity.metrics.TopicMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; import com.xiaojukeji.kafka.manager.common.utils.DateUtils; +import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; import com.xiaojukeji.kafka.manager.common.utils.NumberUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; @@ -33,6 +35,7 @@ import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -345,6 +348,47 @@ public class TopicManagerServiceImpl implements TopicManagerService { return ResultStatus.MYSQL_ERROR; } + @Override + public ResultStatus modifyTopicByOp(Long clusterId, String topicName, String appId, String description, String operator) { + try { + if (!PhysicalClusterMetadataManager.isTopicExist(clusterId, topicName)) { + return ResultStatus.TOPIC_NOT_EXIST; + } + AppDO appDO = appService.getByAppId(appId); + if (ValidateUtils.isNull(appDO)) { + return ResultStatus.APP_NOT_EXIST; + } + + TopicDO topicDO = topicDao.getByTopicName(clusterId, topicName); + if (ValidateUtils.isNull(topicDO)) { + // 不存在, 则需要插入 + topicDO = new TopicDO(); + topicDO.setAppId(appId); + topicDO.setClusterId(clusterId); + topicDO.setTopicName(topicName); + topicDO.setPeakBytesIn(TopicCreationConstant.DEFAULT_QUOTA); + topicDO.setDescription(description); + this.addTopic(topicDO); + } else { + // 存在, 则直接更新 + topicDO.setAppId(appId); + topicDO.setDescription(description); + topicDao.updateByName(topicDO); + } + + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setAppId(appId); + authorityDO.setClusterId(clusterId); + authorityDO.setTopicName(topicName); + authorityDO.setAccess(TopicAuthorityEnum.READ_WRITE.getCode()); + authorityService.addAuthority(authorityDO); + } catch (Exception e) { + LOGGER.error("modify topic failed, clusterId:{} topicName:{} description:{} operator:{} ", + clusterId, topicName, description, operator, e); + } + return ResultStatus.MYSQL_ERROR; + } + @Override public int deleteByTopicName(Long clusterId, String topicName) { try { @@ -359,6 +403,9 @@ public class TopicManagerServiceImpl implements TopicManagerService { public int addTopic(TopicDO topicDO) { try { return topicDao.insert(topicDO); + } catch (DuplicateKeyException duplicateKeyException) { + // 主建重复了, 非重要问题 + LOGGER.debug("class=TopicManagerServiceImpl||method=addTopic||data={}||msg=exist duplicate topic", JsonUtils.toJSONString(topicDO)); } catch (Exception e) { LOGGER.error("insert topic failed, TopicDO:{}", topicDO.toString(), e); } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java index 05a8c4e120554545b37f719dd0d36d723a29540c..5dea05619ccb39f21a5f033d7291e4514c4c12ce 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/TopicServiceImpl.java @@ -29,6 +29,7 @@ import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; +import com.xiaojukeji.kafka.manager.service.strategy.AbstractHealthScoreStrategy; import com.xiaojukeji.kafka.manager.service.utils.KafkaZookeeperUtils; import com.xiaojukeji.kafka.manager.service.utils.MetricsConvertUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -83,6 +84,9 @@ public class TopicServiceImpl implements TopicService { @Autowired private RegionService regionService; + @Autowired + private AbstractHealthScoreStrategy healthScoreStrategy; + @Override public List getTopicMetricsFromDB(Long clusterId, String topicName, Date startTime, Date endTime) { try { @@ -235,7 +239,7 @@ public class TopicServiceImpl implements TopicService { basicDTO.setRegionNameList(regionDOList.stream().map(RegionDO::getName).collect(Collectors.toList())); basicDTO.setTopicCodeC(jmxService.getTopicCodeCValue(clusterId, topicName)); - basicDTO.setScore(100); + basicDTO.setScore(healthScoreStrategy.calTopicHealthScore(clusterId, topicName)); return basicDTO; } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java index 9775f9e468194118a73559957934031d72b003b8..796410da35135aa81ccc8baf422a8765e23df1d1 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/service/impl/ZookeeperServiceImpl.java @@ -2,8 +2,10 @@ package com.xiaojukeji.kafka.manager.service.service.impl; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.utils.ListUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; +import com.xiaojukeji.kafka.manager.common.zookeeper.ZkPathUtil; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; import com.xiaojukeji.kafka.manager.common.zookeeper.znode.didi.TopicJmxSwitch; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; @@ -13,6 +15,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; + /** * @author zengqiao * @date 20/8/27 @@ -40,4 +45,29 @@ public class ZookeeperServiceImpl implements ZookeeperService { } return new Result(); } + + @Override + public Result> getControllerPreferredCandidates(Long clusterId) { + if (ValidateUtils.isNull(clusterId)) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + ZkConfigImpl zkConfig = PhysicalClusterMetadataManager.getZKConfig(clusterId); + if (ValidateUtils.isNull(zkConfig)) { + return Result.buildFrom(ResultStatus.CONNECT_ZOOKEEPER_FAILED); + } + + try { + if (!zkConfig.checkPathExists(ZkPathUtil.D_CONTROLLER_CANDIDATES)) { + return Result.buildSuc(new ArrayList<>()); + } + List brokerIdList = zkConfig.getChildren(ZkPathUtil.D_CONTROLLER_CANDIDATES); + if (ValidateUtils.isEmptyList(brokerIdList)) { + return Result.buildSuc(new ArrayList<>()); + } + return Result.buildSuc(ListUtils.string2IntList(ListUtils.strList2String(brokerIdList))); + } catch (Exception e) { + LOGGER.error("class=ZookeeperServiceImpl||method=getControllerPreferredCandidates||clusterId={}||errMsg={}", clusterId, e.getMessage()); + } + return Result.buildFrom(ResultStatus.READ_ZOOKEEPER_FAILED); + } } \ No newline at end of file diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/strategy/healthscore/DidiHealthScoreStrategy.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/strategy/healthscore/DidiHealthScoreStrategy.java index fbef70fb9d843aa19bd621c7e98c306fdae6276e..d75dec5a091ae8a49631fb1dcac23a7f61ff252e 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/strategy/healthscore/DidiHealthScoreStrategy.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/strategy/healthscore/DidiHealthScoreStrategy.java @@ -72,8 +72,8 @@ public class DidiHealthScoreStrategy extends AbstractHealthScoreStrategy { // 数据获取失败 return Constant.INVALID_CODE; } - if (((Double) failedFetchRequestsPerSecOneMinuteRate) > 0 - || ((Double) failedProduceRequestsPerSecOneMinuteRate) > 0) { + if (((Double) failedFetchRequestsPerSecOneMinuteRate) > 0.01 + || ((Double) failedProduceRequestsPerSecOneMinuteRate) > 0.01) { return HEALTH_SCORE_VERY_BAD; } diff --git a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java index 26f964cf8b4d1a305b05444409881cc24cfb11ae..f808b97694f5f59df827b398816f8bdbc1b6fe92 100644 --- a/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java +++ b/kafka-manager-core/src/main/java/com/xiaojukeji/kafka/manager/service/zookeeper/TopicStateListener.java @@ -5,6 +5,8 @@ import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata import com.xiaojukeji.kafka.manager.common.zookeeper.StateChangeListener; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkConfigImpl; import com.xiaojukeji.kafka.manager.common.zookeeper.ZkPathUtil; +import com.xiaojukeji.kafka.manager.dao.TopicDao; +import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.cache.ThreadPool; import org.apache.zookeeper.data.Stat; @@ -28,11 +30,22 @@ public class TopicStateListener implements StateChangeListener { private ZkConfigImpl zkConfig; + private TopicDao topicDao; + + private AuthorityDao authorityDao; + public TopicStateListener(Long clusterId, ZkConfigImpl zkConfig) { this.clusterId = clusterId; this.zkConfig = zkConfig; } + public TopicStateListener(Long clusterId, ZkConfigImpl zkConfig, TopicDao topicDao, AuthorityDao authorityDao) { + this.clusterId = clusterId; + this.zkConfig = zkConfig; + this.topicDao = topicDao; + this.authorityDao = authorityDao; + } + @Override public void init() { try { @@ -79,6 +92,8 @@ public class TopicStateListener implements StateChangeListener { private void processTopicDelete(String topicName) { LOGGER.warn("delete topic, clusterId:{} topicName:{}.", clusterId, topicName); PhysicalClusterMetadataManager.removeTopicMetadata(clusterId, topicName); + topicDao.removeTopicInCache(clusterId, topicName); + authorityDao.removeAuthorityInCache(clusterId, topicName); } private void processTopicAdded(String topicName) { diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java index 64e089a62d1841c36fea0325f25a02b6638c3544..3d3f5410c599509207ddbda570be68fb6c5ebf2d 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/TopicDao.java @@ -22,4 +22,6 @@ public interface TopicDao { List listAll(); TopicDO getTopic(Long clusterId, String topicName, String appId); + + TopicDO removeTopicInCache(Long clusterId, String topicName); } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java index e04102f58753c052ebee7b3ef4bc277cf5c1fe4b..a7a8affe7551ed75dacd51ffa94f6f4626258378 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/AuthorityDao.java @@ -37,4 +37,8 @@ public interface AuthorityDao { List listAll(); Map>> getAllAuthority(); + + void removeAuthorityInCache(Long clusterId, String topicName); + + int deleteAuthorityByTopic(Long clusterId, String topicName); } diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/GatewayConfigDao.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/GatewayConfigDao.java index dc7fc7d51b629206b98bd0c3cb3ec5f95a18ea33..584223fe2a6e30ad677490489244a2ff4ca02f65 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/GatewayConfigDao.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/GatewayConfigDao.java @@ -1,5 +1,6 @@ package com.xiaojukeji.kafka.manager.dao.gateway; +import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; import java.util.List; @@ -12,4 +13,14 @@ public interface GatewayConfigDao { List getByConfigType(String configType); GatewayConfigDO getByConfigTypeAndName(String configType, String configName); + + List list(); + + int insert(GatewayConfigDO gatewayConfigDO); + + int deleteById(Long id); + + int updateById(GatewayConfigDO gatewayConfigDO); + + GatewayConfigDO getById(Long id); } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java index 39b4902ccef56379d3d4bccc69e87797c3b44021..74a7cab04373cc73106cbdaf14d9da4f12499920 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/AuthorityDaoImpl.java @@ -1,6 +1,7 @@ package com.xiaojukeji.kafka.manager.dao.gateway.impl; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.dao.gateway.AuthorityDao; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.beans.factory.annotation.Autowired; @@ -86,6 +87,32 @@ public class AuthorityDaoImpl implements AuthorityDao { return AUTHORITY_MAP; } + @Override + public void removeAuthorityInCache(Long clusterId, String topicName) { + AUTHORITY_MAP.forEach((appId, map) -> { + map.forEach((id, subMap) -> { + if (id.equals(clusterId)) { + subMap.remove(topicName); + if (subMap.isEmpty()) { + map.remove(id); + } + } + }); + if (map.isEmpty()) { + AUTHORITY_MAP.remove(appId); + } + }); + } + + @Override + public int deleteAuthorityByTopic(Long clusterId, String topicName) { + Map params = new HashMap<>(2); + params.put("clusterId", clusterId); + params.put("topicName", topicName); + return sqlSession.delete("AuthorityDao.deleteByTopic", params); + } + + private void updateAuthorityCache() { Long timestamp = System.currentTimeMillis(); diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/GatewayConfigDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/GatewayConfigDaoImpl.java index 3c39124df5f182f0afe07813386385b938700bda..fab4830c35f772b90128fabd8af15d2a81ad4da3 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/GatewayConfigDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/gateway/impl/GatewayConfigDaoImpl.java @@ -35,4 +35,29 @@ public class GatewayConfigDaoImpl implements GatewayConfigDao { params.put("configName", configName); return sqlSession.selectOne("GatewayConfigDao.getByConfigTypeAndName", params); } + + @Override + public List list() { + return sqlSession.selectList("GatewayConfigDao.list"); + } + + @Override + public int insert(GatewayConfigDO gatewayConfigDO) { + return sqlSession.insert("GatewayConfigDao.insert", gatewayConfigDO); + } + + @Override + public int deleteById(Long id) { + return sqlSession.delete("GatewayConfigDao.deleteById", id); + } + + @Override + public int updateById(GatewayConfigDO gatewayConfigDO) { + return sqlSession.update("GatewayConfigDao.updateById", gatewayConfigDO); + } + + @Override + public GatewayConfigDO getById(Long id) { + return sqlSession.selectOne("GatewayConfigDao.getById", id); + } } \ No newline at end of file diff --git a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java index 4afb12aec792400ee9abd7960f72b9857fcec7e1..ba4468df38c7f1b5b9c5ceeecfbe6439e240776c 100644 --- a/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java +++ b/kafka-manager-dao/src/main/java/com/xiaojukeji/kafka/manager/dao/impl/TopicDaoImpl.java @@ -89,6 +89,11 @@ public class TopicDaoImpl implements TopicDao { return sqlSession.selectOne("TopicDao.getTopic", params); } + @Override + public TopicDO removeTopicInCache(Long clusterId, String topicName) { + return TOPIC_MAP.getOrDefault(clusterId, new HashMap<>(0)).remove(topicName); + } + private void updateTopicCache() { Long timestamp = System.currentTimeMillis(); diff --git a/kafka-manager-dao/src/main/resources/mapper/AuthorityDao.xml b/kafka-manager-dao/src/main/resources/mapper/AuthorityDao.xml index 0f3bb4ab810d62248a780bc182c746973a0716cd..8783803cad32ce4ee4e9cc85d5b8cbcba2e81df1 100644 --- a/kafka-manager-dao/src/main/resources/mapper/AuthorityDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/AuthorityDao.xml @@ -45,4 +45,9 @@ + + + DELETE FROM authority WHERE cluster_id = #{clusterId} AND topic_name = #{topicName} + + \ No newline at end of file diff --git a/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml b/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml index 1d8b85a463937bdfd3fc6f80aa356811e655f67e..8aa91925035c58f980d9fcb0629f41aee1aef826 100644 --- a/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml +++ b/kafka-manager-dao/src/main/resources/mapper/GatewayConfigDao.xml @@ -19,4 +19,38 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaAclDao.xml b/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaAclDao.xml deleted file mode 100644 index d3f88fdfabdf00447f825831b68f7ccca25d4fe8..0000000000000000000000000000000000000000 --- a/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaAclDao.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - INSERT INTO kafka_acl - (cluster_id, topic_name, user_name, access, operation, gm_create, gm_modify) - VALUES - (#{clusterId}, #{topicName}, #{userName}, #{access}, #{operation}, #{gmCreate}, #{gmModify}) - - - - \ No newline at end of file diff --git a/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaUserDao.xml b/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaUserDao.xml deleted file mode 100644 index 44665f58f607f1d638029b8114d9faef9f1b0f4e..0000000000000000000000000000000000000000 --- a/kafka-manager-dao/src/main/resources/mapper/gateway/DeprecatedKafkaUserDao.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - INSERT INTO kafka_user - (`name`, password, user_type, operation, gm_create, gm_modify) - VALUES - (#{name}, #{password}, #{userType}, #{operation}, #{gmtCreate}, #{gmtModify}) - - - - - diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java index 3ea258761790b8768d4e87b7c7038e9606bdca04..bb84593229b5b124d83722e82a1e7a1144f3835a 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/AccountService.java @@ -7,30 +7,80 @@ import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account; import com.xiaojukeji.kafka.manager.common.entity.pojo.AccountDO; import java.util.List; -import java.util.Map; /** * @author huangyiminghappy@163.com * @date 2019-04-26 */ public interface AccountService { + /** + * 增加账号 + * @param accountDO 账号信息 + * @return + */ ResultStatus createAccount(AccountDO accountDO); + /** + * 查询账号信息 + * @param username 用户名 + * @return + */ AccountDO getAccountDO(String username); + /** + * 删除用户 + * @param username 用户名 + * @return + */ ResultStatus deleteByName(String username); + /** + * 更新账号 + * @param accountDO 账号信息 + * @return + */ ResultStatus updateAccount(AccountDO accountDO); + /** + * 获取用户列表 + * @return + */ List list(); + /** + * 依据前缀获取查询用户信息 + * @param prefix + * @return + */ List searchAccountByPrefix(String prefix); + /** + * 从cache中获取用户角色 + * @param username + * @return + */ AccountRoleEnum getAccountRoleFromCache(String username); + /** + * 从cache中获取用户信息 + * @param userName + * @return + */ Account getAccountFromCache(String userName); + /** + * 判断当前用户是否是管理员工单的审批人 + * @param username + * @return + */ boolean isAdminOrderHandler(String username); + /** + * 是否是运维或者研发角色 + * @param username + * @return + */ + boolean isOpOrRd(String username); + List getAdminOrderHandlerFromCache(); } diff --git a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java index 591e09ac03d6ed71915fc99db59afcc6546a20b6..b03cd1952b732c2a4141637f8e161a6a888c4a10 100644 --- a/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java +++ b/kafka-manager-extends/kafka-manager-account/src/main/java/com/xiaojukeji/kafka/manager/account/impl/AccountServiceImpl.java @@ -226,6 +226,18 @@ public class AccountServiceImpl implements AccountService { return false; } + @Override + public boolean isOpOrRd(String username) { + if (ValidateUtils.isNull(ACCOUNT_ROLE_CACHE)) { + flush(); + } + AccountRoleEnum accountRoleEnum = ACCOUNT_ROLE_CACHE.getOrDefault(username, AccountRoleEnum.NORMAL); + if (accountRoleEnum.equals(AccountRoleEnum.OP) || accountRoleEnum.equals(AccountRoleEnum.RD)) { + return true; + } + return false; + } + @Override public List getAdminOrderHandlerFromCache() { if (ValidateUtils.isEmptyList(ADMIN_ORDER_HANDLER_CACHE)) { diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/OrderTypeEnum.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/OrderTypeEnum.java index 6bd309973c00340ade5d5df6a5d3289026b757ff..d651c57e14548bd21699fc4747462df1d423f2c1 100644 --- a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/OrderTypeEnum.java +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/OrderTypeEnum.java @@ -25,6 +25,10 @@ public enum OrderTypeEnum { APPLY_EXPAND_CLUSTER (05, "集群扩容", "modifyClusterOrder"), APPLY_REDUCE_CLUSTER (15, "集群缩容", "modifyClusterOrder"), + ADD_GATEWAY_CONFIG (06, "增加GateWay配置", "addGatewayConfigOrder"), + DELETE_GATEWAY_CONFIG (16, "删除GateWay配置", "deleteGatewayConfigOrder"), + MODIFY_GATEWAY_CONFIG (26, "修改GateWay配置", "modifyGatewayConfigOrder"), + ; private Integer code; diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..0045bfe255f78011a71e63ba57ea41b4b41f5993 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionAddGatewayConfigDTO.java @@ -0,0 +1,62 @@ +package com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway; + +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import io.swagger.annotations.ApiModelProperty; + +/** + * 增加gateway配置 + * @author zengqiao + * @date 2021/01/12 + */ +public class OrderExtensionAddGatewayConfigDTO { + @ApiModelProperty(value = "类型") + private String type; + + @ApiModelProperty(value = "名称") + private String name; + + @ApiModelProperty(value = "值") + private String value; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + return "OrderExtensionAddGatewayConfigDTO{" + + "type='" + type + '\'' + + ", name='" + name + '\'' + + ", value='" + value + '\'' + + '}'; + } + + public boolean legal() { + if (ValidateUtils.isBlank(type) + || ValidateUtils.isBlank(name) + || ValidateUtils.isBlank(value)) { + return false; + } + return true; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionDeleteGatewayConfigDTO.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionDeleteGatewayConfigDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..d830c2d2cad3efbd0293b765c3bd75288c84b477 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionDeleteGatewayConfigDTO.java @@ -0,0 +1,36 @@ +package com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway; + +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import io.swagger.annotations.ApiModelProperty; + +/** + * 删除gateway配置 + * @author zengqiao + * @date 2021/01/12 + */ +public class OrderExtensionDeleteGatewayConfigDTO { + @ApiModelProperty(value = "配置ID") + private Long id; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public String toString() { + return "OrderExtensionDeleteGatewayConfigDTO{" + + "id=" + id + + '}'; + } + + public boolean legal() { + if (ValidateUtils.isNull(id)) { + return false; + } + return true; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java new file mode 100644 index 0000000000000000000000000000000000000000..f5212f8c4929b04900f990ef1c0f6573b1f997c2 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/apply/gateway/OrderExtensionModifyGatewayConfigDTO.java @@ -0,0 +1,77 @@ +package com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import io.swagger.annotations.ApiModelProperty; + +/** + * 修改gateway配置 + * @author zengqiao + * @date 2021/01/12 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class OrderExtensionModifyGatewayConfigDTO { + @ApiModelProperty(value = "配置ID") + private Long id; + + @ApiModelProperty(value = "类型") + private String type; + + @ApiModelProperty(value = "名称") + private String name; + + @ApiModelProperty(value = "值") + private String value; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + return "OrderExtensionModifyGatewayConfigDTO{" + + "id=" + id + + ", type='" + type + '\'' + + ", name='" + name + '\'' + + ", value='" + value + '\'' + + '}'; + } + + public boolean legal() { + if (ValidateUtils.isNull(id) + || ValidateUtils.isBlank(name) + || ValidateUtils.isBlank(type) + || ValidateUtils.isBlank(value)) { + return false; + } + return true; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigData.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigData.java new file mode 100644 index 0000000000000000000000000000000000000000..8a26d696521820a31340a822d39e49c81a21effe --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigData.java @@ -0,0 +1,65 @@ +package com.xiaojukeji.kafka.manager.bpm.common.entry.detail; + + +public class OrderDetailGatewayConfigData extends AbstractOrderDetailData { + private Long id; + + private String type; + + private String name; + + private String value; + + private Long version; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + @Override + public String toString() { + return "OrderDetailGatewayConfigData{" + + "id=" + id + + ", type='" + type + '\'' + + ", name='" + name + '\'' + + ", value='" + value + '\'' + + ", version=" + version + + '}'; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigModifyData.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigModifyData.java new file mode 100644 index 0000000000000000000000000000000000000000..87dfe50e8228d97a438cbf2bb792166fc1531432 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/common/entry/detail/OrderDetailGatewayConfigModifyData.java @@ -0,0 +1,42 @@ +package com.xiaojukeji.kafka.manager.bpm.common.entry.detail; + +/** + * gateway config修改 + * @author zengqiao + * @date 2021/01/13 + */ +public class OrderDetailGatewayConfigModifyData extends AbstractOrderDetailData { + /** + * 旧的Gateway Config + */ + private OrderDetailGatewayConfigData oldGatewayConfig; + + /** + * 新的Gateway Config + */ + private OrderDetailGatewayConfigData newGatewayConfig; + + public OrderDetailGatewayConfigData getOldGatewayConfig() { + return oldGatewayConfig; + } + + public void setOldGatewayConfig(OrderDetailGatewayConfigData oldGatewayConfig) { + this.oldGatewayConfig = oldGatewayConfig; + } + + public OrderDetailGatewayConfigData getNewGatewayConfig() { + return newGatewayConfig; + } + + public void setNewGatewayConfig(OrderDetailGatewayConfigData newGatewayConfig) { + this.newGatewayConfig = newGatewayConfig; + } + + @Override + public String toString() { + return "OrderDetailGatewayConfigModifyData{" + + "oldGatewayConfig=" + oldGatewayConfig + + ", newGatewayConfig=" + newGatewayConfig + + '}'; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/AbstractGatewayConfigOrder.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/AbstractGatewayConfigOrder.java new file mode 100644 index 0000000000000000000000000000000000000000..10e9cac60476896cdbe5edaac7053708a5aad7cd --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/AbstractGatewayConfigOrder.java @@ -0,0 +1,27 @@ +package com.xiaojukeji.kafka.manager.bpm.order; + +import com.xiaojukeji.kafka.manager.account.AccountService; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.ao.account.Account; +import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +public abstract class AbstractGatewayConfigOrder extends AbstractOrder { + @Autowired + private AccountService accountService; + + @Override + public ResultStatus checkAuthority(OrderDO orderDO, String username) { + if (!accountService.isAdminOrderHandler(username)) { + return ResultStatus.USER_WITHOUT_AUTHORITY; + } + return ResultStatus.SUCCESS; + } + + @Override + public List getApproverList(String extensions) { + return accountService.getAdminOrderHandlerFromCache(); + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/AddGatewayConfigOrder.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/AddGatewayConfigOrder.java new file mode 100644 index 0000000000000000000000000000000000000000..6b017772381de8986c9d539234294a0e19c01ec2 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/AddGatewayConfigOrder.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.kafka.manager.bpm.order.impl.gateway; + +import com.xiaojukeji.kafka.manager.bpm.common.entry.detail.AbstractOrderDetailData; +import com.xiaojukeji.kafka.manager.bpm.common.handle.OrderHandleBaseDTO; +import com.xiaojukeji.kafka.manager.bpm.order.AbstractGatewayConfigOrder; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; +import org.springframework.stereotype.Component; + +/** + * @author zengqiao + * @date 2021/01/12 + */ +@Component("addGatewayConfigOrder") +public class AddGatewayConfigOrder extends AbstractGatewayConfigOrder { + + @Override + public Result checkExtensionFieldsAndGenerateTitle(String extensions) { + return Result.buildSuc(); + } + + @Override + public AbstractOrderDetailData getOrderExtensionDetailData(String extensions) { + return null; + } + + @Override + public ResultStatus handleOrderDetail(OrderDO orderDO, OrderHandleBaseDTO baseDTO, String userName) { + return ResultStatus.SUCCESS; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/DeleteGatewayConfigOrder.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/DeleteGatewayConfigOrder.java new file mode 100644 index 0000000000000000000000000000000000000000..395cc42bf358b476aaff022845682339c7a7ff88 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/DeleteGatewayConfigOrder.java @@ -0,0 +1,31 @@ +package com.xiaojukeji.kafka.manager.bpm.order.impl.gateway; + +import com.xiaojukeji.kafka.manager.bpm.common.entry.detail.AbstractOrderDetailData; +import com.xiaojukeji.kafka.manager.bpm.common.handle.OrderHandleBaseDTO; +import com.xiaojukeji.kafka.manager.bpm.order.AbstractGatewayConfigOrder; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; +import org.springframework.stereotype.Component; + +/** + * @author zengqiao + * @date 2021/01/12 + */ +@Component("deleteGatewayConfigOrder") +public class DeleteGatewayConfigOrder extends AbstractGatewayConfigOrder { + @Override + public Result checkExtensionFieldsAndGenerateTitle(String extensions) { + return Result.buildSuc(); + } + + @Override + public AbstractOrderDetailData getOrderExtensionDetailData(String extensions) { + return null; + } + + @Override + public ResultStatus handleOrderDetail(OrderDO orderDO, OrderHandleBaseDTO baseDTO, String userName) { + return ResultStatus.SUCCESS; + } +} diff --git a/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/ModifyGatewayConfigOrder.java b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/ModifyGatewayConfigOrder.java new file mode 100644 index 0000000000000000000000000000000000000000..699fe93591274adf045337b408bfa802b813de55 --- /dev/null +++ b/kafka-manager-extends/kafka-manager-bpm/src/main/java/com/xiaojukeji/kafka/manager/bpm/order/impl/gateway/ModifyGatewayConfigOrder.java @@ -0,0 +1,109 @@ +package com.xiaojukeji.kafka.manager.bpm.order.impl.gateway; + +import com.alibaba.fastjson.JSONObject; +import com.xiaojukeji.kafka.manager.bpm.common.OrderTypeEnum; +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionModifyGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.bpm.common.entry.detail.AbstractOrderDetailData; +import com.xiaojukeji.kafka.manager.bpm.common.entry.detail.OrderDetailGatewayConfigData; +import com.xiaojukeji.kafka.manager.bpm.common.entry.detail.OrderDetailGatewayConfigModifyData; +import com.xiaojukeji.kafka.manager.bpm.common.handle.OrderHandleBaseDTO; +import com.xiaojukeji.kafka.manager.bpm.order.AbstractGatewayConfigOrder; +import com.xiaojukeji.kafka.manager.common.bizenum.gateway.GatewayConfigKeyEnum; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.pojo.OrderDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.service.service.OperateRecordService; +import com.xiaojukeji.kafka.manager.service.service.gateway.GatewayConfigService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * @author zengqiao + * @date 2021/01/12 + */ +@Component("modifyGatewayConfigOrder") +public class ModifyGatewayConfigOrder extends AbstractGatewayConfigOrder { + private static final Logger LOGGER = LoggerFactory.getLogger(ModifyGatewayConfigOrder.class); + + @Autowired + private GatewayConfigService gatewayConfigService; + + @Autowired + private OperateRecordService operateRecordService; + + @Override + public Result checkExtensionFieldsAndGenerateTitle(String extensions) { + OrderExtensionModifyGatewayConfigDTO orderExtensionDTO = null; + try { + orderExtensionDTO = JSONObject.parseObject(extensions, OrderExtensionModifyGatewayConfigDTO.class); + } catch (Exception e) { + LOGGER.error("class=ModifyGatewayConfigOrder||method=checkExtensionFieldsAndGenerateTitle||params={}||errMsg={}", extensions, e.getMessage()); + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + if (!orderExtensionDTO.legal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + GatewayConfigDO gatewayConfigDO = gatewayConfigService.getById(orderExtensionDTO.getId()); + if (ValidateUtils.isNull(gatewayConfigDO)) { + // 配置不存在 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + GatewayConfigKeyEnum configKeyEnum = GatewayConfigKeyEnum.getByConfigType(orderExtensionDTO.getType()); + if (ValidateUtils.isNull(configKeyEnum)) { + // 配置类型不对 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + + return new Result<>(OrderTypeEnum.MODIFY_GATEWAY_CONFIG.getMessage()); + } + + @Override + public AbstractOrderDetailData getOrderExtensionDetailData(String extensions) { + OrderExtensionModifyGatewayConfigDTO orderExtensionDTO = null; + try { + orderExtensionDTO = JSONObject.parseObject(extensions, OrderExtensionModifyGatewayConfigDTO.class); + } catch (Exception e) { + LOGGER.error("class=ModifyGatewayConfigOrder||method=getOrderExtensionDetailData||params={}||errMsg={}", extensions, e.getMessage()); + return null; + } + + // 返回的数据 + OrderDetailGatewayConfigModifyData orderDetailDTO = new OrderDetailGatewayConfigModifyData(); + + // 新的配置 + OrderDetailGatewayConfigData newGatewayConfig = new OrderDetailGatewayConfigData(); + newGatewayConfig.setId(orderExtensionDTO.getId()); + newGatewayConfig.setType(orderExtensionDTO.getType()); + newGatewayConfig.setName(orderExtensionDTO.getName()); + newGatewayConfig.setValue(orderExtensionDTO.getValue()); + orderDetailDTO.setNewGatewayConfig(newGatewayConfig); + + GatewayConfigDO gatewayConfigDO = gatewayConfigService.getById(orderExtensionDTO.getId()); + if (ValidateUtils.isNull(gatewayConfigDO)) { + // 旧的配置不存在 + return orderDetailDTO; + } + + // 旧的配置 + OrderDetailGatewayConfigData oldGatewayConfig = new OrderDetailGatewayConfigData(); + newGatewayConfig.setId(gatewayConfigDO.getId()); + newGatewayConfig.setType(gatewayConfigDO.getType()); + newGatewayConfig.setName(gatewayConfigDO.getName()); + newGatewayConfig.setValue(gatewayConfigDO.getValue()); + newGatewayConfig.setVersion(gatewayConfigDO.getVersion()); + orderDetailDTO.setOldGatewayConfig(oldGatewayConfig); + + return orderDetailDTO; + } + + @Override + public ResultStatus handleOrderDetail(OrderDO orderDO, OrderHandleBaseDTO baseDTO, String username) { + return ResultStatus.SUCCESS; + } +} diff --git a/kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderApplyNotifyService.java b/kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderApplyNotifyService.java index cf4e5563b9317846f6a5a927f4fd668f47744b72..35caf21ff3846064fd0caa070b510598e3b3ee09 100644 --- a/kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderApplyNotifyService.java +++ b/kafka-manager-extends/kafka-manager-notify/src/main/java/com/xiaojukeji/kafka/manager/notify/OrderApplyNotifyService.java @@ -10,6 +10,7 @@ import com.xiaojukeji.kafka.manager.notify.common.OrderNotifyTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationListener; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; /** @@ -24,6 +25,7 @@ public class OrderApplyNotifyService implements ApplicationListener(), - OffsetLocationEnum.getOffsetStoreLocation(dto.getLocation()) - ); + ConsumerGroup consumerGroup = new ConsumerGroup(clusterDO.getId(), dto.getConsumerGroup(), OffsetLocationEnum.getOffsetStoreLocation(dto.getLocation())); return consumerService.resetConsumerOffset( clusterDO, dto.getTopicName(), - consumerGroupDTO, + consumerGroup, offsetDTOList ); } diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/config/SyncTopic2DBConfig.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/config/SyncTopic2DBConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..f5ec62a244bb9b09866e86ac4d801b30d34dbc2c --- /dev/null +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/config/SyncTopic2DBConfig.java @@ -0,0 +1,51 @@ +package com.xiaojukeji.kafka.manager.task.config; + +public class SyncTopic2DBConfig { + /** + * 默认的App + */ + private String defaultAppId; + + /** + * 进行同步的集群 + */ + private Long clusterId; + + /** + * 是否增加权限信息, 默认不增加 + */ + private boolean addAuthority; + + public String getDefaultAppId() { + return defaultAppId; + } + + public void setDefaultAppId(String defaultAppId) { + this.defaultAppId = defaultAppId; + } + + public Long getClusterId() { + return clusterId; + } + + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; + } + + public boolean isAddAuthority() { + return addAuthority; + } + + public void setAddAuthority(boolean addAuthority) { + this.addAuthority = addAuthority; + } + + @Override + public String toString() { + return "SyncTopic2DBConfig{" + + "defaultAppId='" + defaultAppId + '\'' + + ", clusterId=" + clusterId + + ", addAuthority=" + addAuthority + + '}'; + } +} diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCGData.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCGData.java index eef38ed5111c860fc9f8b575a09fe7ecc5ec5519..cc67428fd4b7781de33eea5dc4cdbc8fa29f809b 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCGData.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/metrics/collect/CollectAndPublishCGData.java @@ -2,7 +2,7 @@ package com.xiaojukeji.kafka.manager.task.dispatch.metrics.collect; import com.xiaojukeji.kafka.manager.common.bizenum.OffsetPosEnum; import com.xiaojukeji.kafka.manager.common.constant.LogConstant; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; import com.xiaojukeji.kafka.manager.common.entity.metrics.ConsumerMetrics; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.common.events.ConsumerMetricsCollectedEvent; @@ -105,7 +105,7 @@ public class CollectAndPublishCGData extends AbstractScheduledTask { private List getTopicConsumerMetrics(ClusterDO clusterDO, String topicName, long startTimeUnitMs) { - List consumerGroupDTOList = consumerService.getConsumerGroupList(clusterDO.getId(), topicName); + List consumerGroupDTOList = consumerService.getConsumerGroupList(clusterDO.getId(), topicName); if (ValidateUtils.isEmptyList(consumerGroupDTOList)) { // 重试 consumerGroupDTOList = consumerService.getConsumerGroupList(clusterDO.getId(), topicName); @@ -131,7 +131,7 @@ public class CollectAndPublishCGData extends AbstractScheduledTask { partitionOffsetMap.put(entry.getKey().partition(), entry.getValue()); } - for (ConsumerGroupDTO consumerGroupDTO: consumerGroupDTOList) { + for (ConsumerGroup consumerGroupDTO: consumerGroupDTOList) { try { ConsumerMetrics consumerMetrics = getTopicConsumerMetrics(clusterDO, topicName, consumerGroupDTO, partitionOffsetMap, startTimeUnitMs); @@ -150,20 +150,20 @@ public class CollectAndPublishCGData extends AbstractScheduledTask { private ConsumerMetrics getTopicConsumerMetrics(ClusterDO clusterDO, String topicName, - ConsumerGroupDTO consumerGroupDTO, + ConsumerGroup consumerGroup, Map partitionOffsetMap, long startTimeUnitMs) { Map consumerOffsetMap = - consumerService.getConsumerOffset(clusterDO, topicName, consumerGroupDTO); + consumerService.getConsumerOffset(clusterDO, topicName, consumerGroup); if (ValidateUtils.isEmptyMap(consumerOffsetMap)) { return null; } ConsumerMetrics metrics = new ConsumerMetrics(); metrics.setClusterId(clusterDO.getId()); metrics.setTopicName(topicName); - metrics.setConsumerGroup(consumerGroupDTO.getConsumerGroup()); - metrics.setLocation(consumerGroupDTO.getOffsetStoreLocation().location); + metrics.setConsumerGroup(consumerGroup.getConsumerGroup()); + metrics.setLocation(consumerGroup.getOffsetStoreLocation().location); metrics.setPartitionOffsetMap(partitionOffsetMap); metrics.setConsumeOffsetMap(consumerOffsetMap); metrics.setTimestampUnitMs(startTimeUnitMs); diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/AutoHandleTopicOrder.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/AutoHandleTopicOrder.java index 5db8a679900948fa6e7277b003c5526957f0d760..00b9d379f7bc73a5fc63491c7cf1c7301cea9960 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/AutoHandleTopicOrder.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/AutoHandleTopicOrder.java @@ -15,6 +15,7 @@ import com.xiaojukeji.kafka.manager.common.utils.ListUtils; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.*; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; +import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; import com.xiaojukeji.kafka.manager.service.service.*; import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask; import com.xiaojukeji.kafka.manager.task.component.CustomScheduled; @@ -109,6 +110,11 @@ public class AutoHandleTopicOrder extends AbstractScheduledTask { return false; } + if (PhysicalClusterMetadataManager.isTopicExist(physicalClusterId, dto.getTopicName())) { + rejectForRepeatedTopicName(orderDO); + return false; + } + if (ValidateUtils.isNull(dto.isPhysicalClusterId()) || !dto.isPhysicalClusterId()) { return handleApplyTopicOrderByLogicalClusterId(clusterDO, orderDO, dto, createConfig); } @@ -117,6 +123,13 @@ public class AutoHandleTopicOrder extends AbstractScheduledTask { return handleApplyTopicOrderByPhysicalClusterId(clusterDO, orderDO, dto, createConfig); } + private void rejectForRepeatedTopicName(OrderDO orderDO) { + orderDO.setApplicant(Constant.AUTO_HANDLE_USER_NAME); + orderDO.setStatus(OrderStatusEnum.REFUSED.getCode()); + orderDO.setOpinion("驳回：该 Topic 已被别人申请并生效"); + orderService.updateOrderById(orderDO); + } + /** * 逻辑集群申请单 */ diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/SyncTopic2DB.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/SyncTopic2DB.java new file mode 100644 index 0000000000000000000000000000000000000000..ae10a21dca9bdc1589599553b56d3f79e6c7f7e8 --- /dev/null +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/dispatch/op/SyncTopic2DB.java @@ -0,0 +1,163 @@ +package com.xiaojukeji.kafka.manager.task.dispatch.op; + +import com.xiaojukeji.kafka.manager.common.bizenum.TopicAuthorityEnum; +import com.xiaojukeji.kafka.manager.common.constant.KafkaConstant; +import com.xiaojukeji.kafka.manager.common.constant.LogConstant; +import com.xiaojukeji.kafka.manager.common.constant.TopicCreationConstant; +import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.TopicDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AppDO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.AuthorityDO; +import com.xiaojukeji.kafka.manager.common.utils.JsonUtils; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.common.zookeeper.znode.brokers.TopicMetadata; +import com.xiaojukeji.kafka.manager.service.cache.PhysicalClusterMetadataManager; +import com.xiaojukeji.kafka.manager.service.service.ClusterService; +import com.xiaojukeji.kafka.manager.service.service.ConfigService; +import com.xiaojukeji.kafka.manager.service.service.TopicManagerService; +import com.xiaojukeji.kafka.manager.service.service.gateway.AppService; +import com.xiaojukeji.kafka.manager.service.service.gateway.AuthorityService; +import com.xiaojukeji.kafka.manager.task.component.AbstractScheduledTask; +import com.xiaojukeji.kafka.manager.task.component.CustomScheduled; +import com.xiaojukeji.kafka.manager.task.component.EmptyEntry; +import com.xiaojukeji.kafka.manager.task.config.SyncTopic2DBConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * 定期将未落盘的Topic刷新到DB中, 仅存储对应的关系, 并不会增加权限等信息 + * @author zengqiao + * @date 19/12/29 + */ +@Component +@CustomScheduled(name = "syncTopic2DB", cron = "0 0/2 * * * ?", threadNum = 1) +@ConditionalOnProperty(prefix = "task.op", name = "sync-topic-enabled", havingValue = "true", matchIfMissing = false) +public class SyncTopic2DB extends AbstractScheduledTask { + private static final Logger LOGGER = LoggerFactory.getLogger(LogConstant.SCHEDULED_TASK_LOGGER); + + private static final String SYNC_TOPIC_2_DB_CONFIG_KEY = "SYNC_TOPIC_2_DB_CONFIG_KEY"; + + @Autowired + private AppService appService; + + @Autowired + private ConfigService configService; + + @Autowired + private ClusterService clusterService; + + @Autowired + private AuthorityService authorityService; + + @Autowired + private TopicManagerService topicManagerService; + + @Override + public List listAllTasks() { + EmptyEntry emptyEntry = new EmptyEntry(); + emptyEntry.setId(System.currentTimeMillis() / 1000); + return Arrays.asList(emptyEntry); + } + + @Override + public void processTask(EmptyEntry entryEntry) { + Map clusterIdConfigMap = getConfig(); + if (ValidateUtils.isEmptyMap(clusterIdConfigMap)) { + LOGGER.warn("class=SyncTopic2DB||method=processTask||msg=without config or config illegal"); + return; + } + LOGGER.info("class=SyncTopic2DB||method=processTask||data={}||msg=start sync", JsonUtils.toJSONString(clusterIdConfigMap)); + + List clusterDOList = clusterService.list(); + if (ValidateUtils.isEmptyList(clusterDOList)) { + return; + } + + for (ClusterDO clusterDO: clusterDOList) { + if (!clusterIdConfigMap.containsKey(clusterDO.getId())) { + continue; + } + try { + syncTopic2DB(clusterDO.getId(), clusterIdConfigMap.get(clusterDO.getId())); + } catch (Exception e) { + LOGGER.error("class=SyncTopic2DB||method=processTask||clusterId={}||errMsg={}||msg=sync failed", clusterDO.getId(), e.getMessage()); + } + } + } + + private void syncTopic2DB(Long clusterId, SyncTopic2DBConfig syncTopic2DBConfig) { + List doList = topicManagerService.getByClusterId(clusterId); + if (ValidateUtils.isNull(doList)) { + doList = new ArrayList<>(); + } + Set existedTopicNameSet = doList.stream().map(elem -> elem.getTopicName()).collect(Collectors.toSet()); + + for (String topicName: PhysicalClusterMetadataManager.getTopicNameList(clusterId)) { + if (existedTopicNameSet.contains(topicName) + || KafkaConstant.COORDINATOR_TOPIC_NAME.equals(topicName) + || KafkaConstant.TRANSACTION_TOPIC_NAME.equals(topicName)) { + continue; + } + + TopicMetadata topicMetadata = PhysicalClusterMetadataManager.getTopicMetadata(clusterId, topicName); + if (ValidateUtils.isNull(topicMetadata)) { + continue; + } + + // 新创建10分钟内的Topic不进行同步, 避免KM平台上新建的, 有所属应用的Topic被错误的同步了 + if (System.currentTimeMillis() - topicMetadata.getCreateTime() < 10 * 60 * 1000) { + continue; + } + + TopicDO topicDO = new TopicDO(); + topicDO.setAppId(syncTopic2DBConfig.getDefaultAppId()); + topicDO.setClusterId(clusterId); + topicDO.setTopicName(topicName); + topicDO.setDescription("定期同步至DB中的无主Topic"); + topicDO.setPeakBytesIn(TopicCreationConstant.DEFAULT_QUOTA); + topicManagerService.addTopic(topicDO); + + if (ValidateUtils.isNull(syncTopic2DBConfig.isAddAuthority()) || !syncTopic2DBConfig.isAddAuthority()) { + // 不增加权限信息, 则直接忽略 + return; + } + + // TODO 当前添加 Topic 和 添加 Authority 是非事务的, 中间出现异常之后, 会导致数据错误, 后续还需要优化一下 + AuthorityDO authorityDO = new AuthorityDO(); + authorityDO.setAppId(syncTopic2DBConfig.getDefaultAppId()); + authorityDO.setClusterId(clusterId); + authorityDO.setTopicName(topicName); + authorityDO.setAccess(TopicAuthorityEnum.READ_WRITE.getCode()); + authorityService.addAuthority(authorityDO); + } + } + + private Map getConfig() { + List configList = configService.getArrayByKey(SYNC_TOPIC_2_DB_CONFIG_KEY, SyncTopic2DBConfig.class); + if (ValidateUtils.isEmptyList(configList)) { + return Collections.EMPTY_MAP; + } + + Map clusterIdConfigMap = new HashMap<>(); + for (SyncTopic2DBConfig syncTopic2DBConfig: configList) { + if (ValidateUtils.isNullOrLessThanZero(syncTopic2DBConfig.getClusterId()) + || ValidateUtils.isBlank(syncTopic2DBConfig.getDefaultAppId())) { + continue; + } + + AppDO appDO = appService.getByAppId(syncTopic2DBConfig.getDefaultAppId()); + if (ValidateUtils.isNull(appDO)) { + continue; + } + + clusterIdConfigMap.put(syncTopic2DBConfig.getClusterId(), syncTopic2DBConfig); + } + return clusterIdConfigMap; + } +} diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushBKConsumerGroupMetadata.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushBKConsumerGroupMetadata.java index b02243ed40421db9d1fd1cbf56a1415dacd2f2d3..239c3ed027531b90a841cb6ef0e982374a44a64f 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushBKConsumerGroupMetadata.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushBKConsumerGroupMetadata.java @@ -50,8 +50,7 @@ public class FlushBKConsumerGroupMetadata { private void flush(Long clusterId) { // 获取消费组列表 Set consumerGroupSet = new HashSet<>(); - Map> consumerGroupAppIdMap = new HashMap<>(); - collectAndSaveConsumerGroup(clusterId, consumerGroupSet, consumerGroupAppIdMap); + collectAndSaveConsumerGroup(clusterId, consumerGroupSet); // 获取消费组summary信息 Map> topicNameConsumerGroupMap = new HashMap<>(); @@ -67,15 +66,12 @@ public class FlushBKConsumerGroupMetadata { new ConsumerMetadata( consumerGroupSet, topicNameConsumerGroupMap, - consumerGroupSummary, - consumerGroupAppIdMap + consumerGroupSummary ) ); } - private void collectAndSaveConsumerGroup(Long clusterId, - Set consumerGroupSet, - Map> consumerGroupAppIdMap) { + private void collectAndSaveConsumerGroup(Long clusterId, Set consumerGroupSet) { try { AdminClient adminClient = KafkaClientPool.getAdminClient(clusterId); @@ -83,20 +79,14 @@ public class FlushBKConsumerGroupMetadata { for (scala.collection.immutable.List brokerGroup : JavaConversions.asJavaMap(brokerGroupMap).values()) { List lists = JavaConversions.asJavaList(brokerGroup); for (kafka.coordinator.GroupOverview groupOverview : lists) { - String consumerGroup = groupOverview.groupId(); - List appIdList = new ArrayList<>(); if (consumerGroup != null && consumerGroup.contains("#")) { String[] splitArray = consumerGroup.split("#"); consumerGroup = splitArray[splitArray.length - 1]; - appIdList = Arrays.asList(splitArray).subList(0, splitArray.length - 1); } - consumerGroupAppIdMap.put(consumerGroup, appIdList); - consumerGroupSet.add(consumerGroup); } } - return ; } catch (Exception e) { LOGGER.error("collect consumerGroup failed, clusterId:{}.", clusterId, e); } diff --git a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushZKConsumerGroupMetadata.java b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushZKConsumerGroupMetadata.java index 5f89c9eeb2e6da619e52f6c0a9618b8d46023fe6..a7d196afca17bcf9cf0479ec3c6685f8a47871fd 100644 --- a/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushZKConsumerGroupMetadata.java +++ b/kafka-manager-task/src/main/java/com/xiaojukeji/kafka/manager/task/schedule/metadata/FlushZKConsumerGroupMetadata.java @@ -55,7 +55,7 @@ public class FlushZKConsumerGroupMetadata { collectTopicAndConsumerGroupMap(clusterId, new ArrayList<>(consumerGroupSet)); ConsumerMetadataCache.putConsumerMetadataInZK( clusterId, - new ConsumerMetadata(consumerGroupSet, topicNameConsumerGroupMap, new HashMap<>(0), new HashMap<>(0)) + new ConsumerMetadata(consumerGroupSet, topicNameConsumerGroupMap, new HashMap<>(0)) ); } diff --git a/kafka-manager-web/assembly.xml b/kafka-manager-web/assembly.xml deleted file mode 100644 index 8d2eeed6b0b5b61f88ac77896f3270770be074eb..0000000000000000000000000000000000000000 --- a/kafka-manager-web/assembly.xml +++ /dev/null @@ -1,44 +0,0 @@ - - bin - - dir - tar.gz - - - - - bin/* - - 0755 - - - - ../docs/install_guide - install - - * - - - - - src/main/resources/ - conf - - application.yml - logback-spring.xml - - - - - ${project.build.directory} - libs - - *.jar - - - - - \ No newline at end of file diff --git a/kafka-manager-web/pom.xml b/kafka-manager-web/pom.xml index e894af5d4d751e1e55c653b3d233ecdfffa651b0..f40e1c352f2e6449daec4bfc3e81bfe24023fbd4 100644 --- a/kafka-manager-web/pom.xml +++ b/kafka-manager-web/pom.xml @@ -122,26 +122,6 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java index 1f86c48ba88fb0a5d250110fbfc4c2a325a5980e..4fe01e228baf576ccddd70c687f088b7b1d9a93b 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/gateway/GatewayHeartbeatController.java @@ -47,7 +47,7 @@ public class GatewayHeartbeatController { List doList = null; try { - doList = JsonUtils.parseTopicConnections(clusterId, jsonObject); + doList = JsonUtils.parseTopicConnections(clusterId, jsonObject, System.currentTimeMillis()); } catch (Exception e) { LOGGER.error("class=GatewayHeartbeatController||method=receiveTopicConnections||clusterId={}||brokerId={}||msg=parse data failed||exception={}", clusterId, brokerId, e.getMessage()); return Result.buildFailure("fail"); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java index cae8c5374964aecf524f149d0277976f089d3415..34529616f731fb14be3950d1c0270423792bbf79 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalAppController.java @@ -76,7 +76,7 @@ public class NormalAppController { @RequestMapping(value = "apps/{appId}/basic-info", method = RequestMethod.GET) @ResponseBody public Result getAppBasicInfo(@PathVariable String appId) { - if (accountService.isAdminOrderHandler(SpringTool.getUserName())) { + if (accountService.isOpOrRd(SpringTool.getUserName())) { return new Result<>(AppConverter.convert2AppVO(appService.getByAppId(appId))); } @@ -101,7 +101,7 @@ public class NormalAppController { @RequestMapping(value = "apps/{appId}/topics", method = RequestMethod.GET) @ResponseBody public Result> getAppTopics(@PathVariable String appId, - @RequestParam(value = "mine") Boolean mine) { + @RequestParam(value = "mine", required = false) Boolean mine) { List dtoList = appService.getAppTopicDTOList(appId, mine); List voList = new ArrayList<>(); diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java index a0decf4966c9d242fd430cdd9385e229be327c3d..59b9598203ce18eb130d9f4e4406d2039be28ce8 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/normal/NormalConsumerController.java @@ -6,10 +6,10 @@ import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.PartitionOffsetDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; import com.xiaojukeji.kafka.manager.common.entity.dto.normal.TopicOffsetResetDTO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupDetailVO; -import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO; +import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupSummaryVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; import com.xiaojukeji.kafka.manager.service.cache.LogicalClusterMetadataManager; @@ -55,7 +55,7 @@ public class NormalConsumerController { @ApiOperation(value = "查询消费Topic的消费组", notes = "") @RequestMapping(value = "{clusterId}/consumers/{topicName}/consumer-groups", method = RequestMethod.GET) @ResponseBody - public Result> getConsumeGroups( + public Result> getConsumeGroups( @PathVariable Long clusterId, @PathVariable String topicName, @RequestParam(value = "isPhysicalClusterId", required = false) Boolean isPhysicalClusterId) { @@ -63,9 +63,9 @@ public class NormalConsumerController { if (ValidateUtils.isNull(physicalClusterId)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } - return new Result<>(ConsumerModelConverter.convert2ConsumerGroupVOList( - consumerService.getConsumerGroupList(physicalClusterId, topicName)) - ); + return new Result<>(ConsumerModelConverter.convert2ConsumerGroupSummaryVOList( + consumerService.getConsumerGroupSummaries(physicalClusterId, topicName) + )); } @ApiOperation(value = "查询消费组的消费详情", notes = "") @@ -95,15 +95,10 @@ public class NormalConsumerController { return Result.buildFrom(ResultStatus.CG_LOCATION_ILLEGAL); } - ConsumerGroupDTO consumeGroupDTO = new ConsumerGroupDTO( - clusterDO.getId(), - consumerGroup, - new ArrayList<>(), - offsetStoreLocation - ); + ConsumerGroup consumeGroup = new ConsumerGroup(clusterDO.getId(), consumerGroup, offsetStoreLocation); try { List consumeDetailDTOList = - consumerService.getConsumeDetail(clusterDO, topicName, consumeGroupDTO); + consumerService.getConsumeDetail(clusterDO, topicName, consumeGroup); return new Result<>( ConsumerModelConverter.convert2ConsumerGroupDetailVO( topicName, @@ -113,7 +108,7 @@ public class NormalConsumerController { ) ); } catch (Exception e) { - LOGGER.error("get consume detail failed, consumerGroup:{}.", consumeGroupDTO, e); + LOGGER.error("get consume detail failed, consumerGroup:{}.", consumeGroup, e); } return Result.buildFrom(ResultStatus.OPERATION_FAILED); @@ -139,16 +134,11 @@ public class NormalConsumerController { return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } - ConsumerGroupDTO consumerGroupDTO = new ConsumerGroupDTO( - physicalClusterId, - dto.getConsumerGroup(), - new ArrayList<>(), - OffsetLocationEnum.getOffsetStoreLocation(dto.getLocation()) - ); + ConsumerGroup consumerGroup = new ConsumerGroup(physicalClusterId, dto.getConsumerGroup(), OffsetLocationEnum.getOffsetStoreLocation(dto.getLocation())); List resultList = consumerService.resetConsumerOffset( clusterDO, dto.getTopicName(), - consumerGroupDTO, + consumerGroup, offsetDTOList ); for (Result result: resultList) { diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java new file mode 100644 index 0000000000000000000000000000000000000000..a97bb386558c2cd3b9521df3b970071724e86f40 --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpGatewayConfigController.java @@ -0,0 +1,52 @@ +package com.xiaojukeji.kafka.manager.web.api.versionone.op; + +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionAddGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionDeleteGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionModifyGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.service.service.gateway.GatewayConfigService; +import com.xiaojukeji.kafka.manager.web.converters.GatewayModelConverter; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + + +@Api(tags = "OP-Gateway配置相关接口(REST)") +@RestController +public class OpGatewayConfigController { + @Autowired + private GatewayConfigService gatewayConfigService; + + @ApiOperation(value = "创建Gateway配置", notes = "") + @RequestMapping(value = "gateway-configs", method = RequestMethod.POST) + @ResponseBody + public Result createGatewayConfig(@RequestBody OrderExtensionAddGatewayConfigDTO dto) { + if (ValidateUtils.isNull(dto) || !dto.legal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return gatewayConfigService.insert(GatewayModelConverter.convert2GatewayConfigDO(dto)); + } + + @ApiOperation(value = "修改Gateway配置", notes = "") + @RequestMapping(value = "gateway-configs", method = RequestMethod.PUT) + @ResponseBody + public Result modifyGatewayConfig(@RequestBody OrderExtensionModifyGatewayConfigDTO dto) { + if (ValidateUtils.isNull(dto) || !dto.legal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return gatewayConfigService.updateById(GatewayModelConverter.convert2GatewayConfigDO(dto)); + } + + @ApiOperation(value = "删除Gateway配置", notes = "") + @RequestMapping(value = "gateway-configs", method = RequestMethod.DELETE) + @ResponseBody + public Result deleteGatewayConfig(@RequestBody OrderExtensionDeleteGatewayConfigDTO dto) { + if (ValidateUtils.isNull(dto) || !dto.legal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return gatewayConfigService.deleteById(dto.getId()); + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java index bea20be9f20f64e14e7b4beb26ba9fca6586a23a..c7b36cba4c9a8626db4344e22b6248c7a2af7fd8 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/op/OpUtilsController.java @@ -194,15 +194,22 @@ public class OpUtilsController { if (ValidateUtils.isNull(clusterDO)) { return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); } - String operator = SpringTool.getUserName(); ResultStatus rs = null; if (RebalanceDimensionEnum.CLUSTER.getCode().equals(reqObj.getDimension())) { - rs = adminService.preferredReplicaElection(clusterDO, operator); + // 按照Cluster纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, SpringTool.getUserName()); } else if (RebalanceDimensionEnum.BROKER.getCode().equals(reqObj.getDimension())) { - rs = adminService.preferredReplicaElection(clusterDO, reqObj.getBrokerId(), operator); + // 按照Broker纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getBrokerId(), SpringTool.getUserName()); + } else if (RebalanceDimensionEnum.TOPIC.getCode().equals(reqObj.getDimension())) { + // 按照Topic纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), SpringTool.getUserName()); + } else if (RebalanceDimensionEnum.PARTITION.getCode().equals(reqObj.getDimension())) { + // 按照Partition纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), reqObj.getPartitionId(), SpringTool.getUserName()); } else { - // TODO: 19/7/8 Topic维度 & Region维度 优先副本选举 + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); } return Result.buildFrom(rs); } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdClusterController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdClusterController.java index 3147633ac618a841914db9603b207432b42ffb8d..69ba8c6d2e310d04c4e0afec2742cfbd8d85ae41 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdClusterController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdClusterController.java @@ -1,10 +1,11 @@ package com.xiaojukeji.kafka.manager.web.api.versionone.rd; import com.xiaojukeji.kafka.manager.common.bizenum.KafkaClientEnum; -import com.xiaojukeji.kafka.manager.common.bizenum.PeakFlowStatusEnum; import com.xiaojukeji.kafka.manager.common.constant.KafkaMetricsCollections; import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ControllerPreferredCandidate; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadataVO; +import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.ControllerPreferredCandidateVO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.RdClusterMetricsVO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.ClusterBrokerStatusVO; import com.xiaojukeji.kafka.manager.common.entity.ao.BrokerOverviewDTO; @@ -26,7 +27,6 @@ import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -168,4 +168,15 @@ public class RdClusterController { public Result> getTopicMetadatas(@PathVariable("clusterId") Long clusterId) { return new Result<>(ClusterModelConverter.convert2TopicMetadataVOList(clusterId)); } + + @ApiOperation(value = "Controller优先候选的Broker", notes = "滴滴内部引擎特性") + @RequestMapping(value = "clusters/{clusterId}/controller-preferred-candidates", method = RequestMethod.GET) + @ResponseBody + public Result> getControllerPreferredCandidates(@PathVariable("clusterId") Long clusterId) { + Result> candidateResult = clusterService.getControllerPreferredCandidates(clusterId); + if (candidateResult.failed()) { + return new Result(candidateResult.getCode(), candidateResult.getMessage()); + } + return Result.buildSuc(ClusterModelConverter.convert2ControllerPreferredCandidateVOList(candidateResult.getData())); + } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java new file mode 100644 index 0000000000000000000000000000000000000000..3748c3cabac798fbc03caac66cda1f1cc8a2504a --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/rd/RdGatewayConfigController.java @@ -0,0 +1,32 @@ +package com.xiaojukeji.kafka.manager.web.api.versionone.rd; + +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; +import com.xiaojukeji.kafka.manager.common.entity.vo.rd.GatewayConfigVO; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.service.service.gateway.GatewayConfigService; +import com.xiaojukeji.kafka.manager.web.converters.GatewayModelConverter; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@Api(tags = "RD-Gateway配置相关接口(REST)") +@RestController +public class RdGatewayConfigController { + @Autowired + private GatewayConfigService gatewayConfigService; + + @ApiOperation(value = "Gateway相关配置信息", notes = "") + @RequestMapping(value = "gateway-configs", method = RequestMethod.GET) + @ResponseBody + public Result> getGatewayConfigs() { + List doList = gatewayConfigService.list(); + if (ValidateUtils.isEmptyList(doList)) { + return Result.buildSuc(); + } + return Result.buildSuc(GatewayModelConverter.convert2GatewayConfigVOList(doList)); + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartConsumeController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartConsumeController.java index d07e35ca77d2b23c7ebaa2dddfa1e954846cf83d..aa8f7b2fcadb2113fef8b07650301ab0ba62dafc 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartConsumeController.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartConsumeController.java @@ -8,7 +8,7 @@ import com.xiaojukeji.kafka.manager.common.constant.SystemCodeConstant; import com.xiaojukeji.kafka.manager.common.entity.Result; import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; import com.xiaojukeji.kafka.manager.openapi.common.dto.ConsumeHealthDTO; import com.xiaojukeji.kafka.manager.openapi.common.dto.OffsetResetDTO; import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; @@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -152,15 +151,10 @@ public class ThirdPartConsumeController { return Result.buildFrom(ResultStatus.CG_LOCATION_ILLEGAL); } - ConsumerGroupDTO consumeGroupDTO = new ConsumerGroupDTO( - clusterDO.getId(), - consumerGroup, - new ArrayList<>(), - offsetStoreLocation - ); + ConsumerGroup consumeGroup = new ConsumerGroup(clusterDO.getId(), consumerGroup, offsetStoreLocation); try { List consumeDetailDTOList = - consumerService.getConsumeDetail(clusterDO, topicName, consumeGroupDTO); + consumerService.getConsumeDetail(clusterDO, topicName, consumeGroup); return new Result<>( ConsumerModelConverter.convert2ConsumerGroupDetailVO( topicName, @@ -170,7 +164,7 @@ public class ThirdPartConsumeController { ) ); } catch (Exception e) { - LOGGER.error("get consume detail failed, consumerGroup:{}.", consumeGroupDTO, e); + LOGGER.error("get consume detail failed, consumerGroup:{}.", consumeGroup, e); } return Result.buildFrom(ResultStatus.OPERATION_FAILED); } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpController.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpController.java new file mode 100644 index 0000000000000000000000000000000000000000..3d31a5f0b33b20dd30cae98a553904ccb38d04b6 --- /dev/null +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/api/versionone/thirdpart/ThirdPartOpController.java @@ -0,0 +1,63 @@ +package com.xiaojukeji.kafka.manager.web.api.versionone.thirdpart; + +import com.xiaojukeji.kafka.manager.common.bizenum.RebalanceDimensionEnum; +import com.xiaojukeji.kafka.manager.common.constant.ApiPrefix; +import com.xiaojukeji.kafka.manager.common.entity.Result; +import com.xiaojukeji.kafka.manager.common.entity.ResultStatus; +import com.xiaojukeji.kafka.manager.common.entity.dto.op.RebalanceDTO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.ClusterDO; +import com.xiaojukeji.kafka.manager.common.utils.SpringTool; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; +import com.xiaojukeji.kafka.manager.service.service.AdminService; +import com.xiaojukeji.kafka.manager.service.service.ClusterService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +/** + * @author zengqiao + * @date 20/9/23 + */ +@Api(tags = "开放接口-OP相关接口(REST)") +@RestController +@RequestMapping(ApiPrefix.API_V1_THIRD_PART_PREFIX) +public class ThirdPartOpController { + + @Autowired + private AdminService adminService; + + @Autowired + private ClusterService clusterService; + + @ApiOperation(value = "优先副本选举") + @RequestMapping(value = "op/rebalance", method = RequestMethod.POST) + @ResponseBody + public Result preferredReplicaElect(@RequestBody RebalanceDTO reqObj) { + if (!reqObj.paramLegal()) { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + ClusterDO clusterDO = clusterService.getById(reqObj.getClusterId()); + if (ValidateUtils.isNull(clusterDO)) { + return Result.buildFrom(ResultStatus.CLUSTER_NOT_EXIST); + } + + ResultStatus rs = null; + if (RebalanceDimensionEnum.CLUSTER.getCode().equals(reqObj.getDimension())) { + // 按照Cluster纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, SpringTool.getUserName()); + } else if (RebalanceDimensionEnum.BROKER.getCode().equals(reqObj.getDimension())) { + // 按照Broker纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getBrokerId(), SpringTool.getUserName()); + } else if (RebalanceDimensionEnum.TOPIC.getCode().equals(reqObj.getDimension())) { + // 按照Topic纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), SpringTool.getUserName()); + } else if (RebalanceDimensionEnum.PARTITION.getCode().equals(reqObj.getDimension())) { + // 按照Partition纬度均衡 + rs = adminService.preferredReplicaElection(clusterDO, reqObj.getTopicName(), reqObj.getPartitionId(), SpringTool.getUserName()); + } else { + return Result.buildFrom(ResultStatus.PARAM_ILLEGAL); + } + return Result.buildFrom(rs); + } +} diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ClusterModelConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ClusterModelConverter.java index e1ed0c9e104061c92e1aa45f1bae34c60317a6d6..9c76a8e5b69cfd21d4e2278a326410d1218c0b48 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ClusterModelConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ClusterModelConverter.java @@ -3,6 +3,7 @@ package com.xiaojukeji.kafka.manager.web.converters; import com.xiaojukeji.kafka.manager.common.entity.ao.BrokerOverviewDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.ClusterDetailDTO; import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ClusterBrokerStatus; +import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.ControllerPreferredCandidate; import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.LogicalCluster; import com.xiaojukeji.kafka.manager.common.entity.ao.cluster.LogicalClusterMetrics; import com.xiaojukeji.kafka.manager.common.entity.dto.rd.ClusterDTO; @@ -15,6 +16,7 @@ import com.xiaojukeji.kafka.manager.common.entity.vo.normal.cluster.TopicMetadat import com.xiaojukeji.kafka.manager.common.entity.vo.rd.KafkaControllerVO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.ClusterBrokerStatusVO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.ClusterDetailVO; +import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.ControllerPreferredCandidateVO; import com.xiaojukeji.kafka.manager.common.entity.vo.rd.cluster.RdClusterMetricsVO; import com.xiaojukeji.kafka.manager.common.utils.CopyUtils; import com.xiaojukeji.kafka.manager.common.utils.ListUtils; @@ -249,4 +251,21 @@ public class ClusterModelConverter { vo.setBrokerReplicaStatusList(clusterBrokerStatus.getBrokerReplicaStatusList()); return vo; } + + public static List convert2ControllerPreferredCandidateVOList(List candidateList) { + if (ValidateUtils.isEmptyList(candidateList)) { + return new ArrayList<>(); + } + + List voList = new ArrayList<>(); + for (ControllerPreferredCandidate candidate: candidateList) { + ControllerPreferredCandidateVO vo = new ControllerPreferredCandidateVO(); + vo.setBrokerId(candidate.getBrokerId()); + vo.setHost(candidate.getHost()); + vo.setStatus(candidate.getStatus()); + vo.setStartTime(candidate.getStartTime()); + voList.add(vo); + } + return voList; + } } diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ConsumerModelConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ConsumerModelConverter.java index 239bab165299607dba24a71897934190dbd0c383..0a66605705185b1a047b2ccf5094ac61640a3e88 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ConsumerModelConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/ConsumerModelConverter.java @@ -1,12 +1,16 @@ package com.xiaojukeji.kafka.manager.web.converters; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroup; +import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupSummary; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupDetailVO; import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumeDetailDTO; -import com.xiaojukeji.kafka.manager.common.entity.ao.consumer.ConsumerGroupDTO; +import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupSummaryVO; import com.xiaojukeji.kafka.manager.common.entity.vo.normal.consumer.ConsumerGroupVO; import com.xiaojukeji.kafka.manager.common.utils.ListUtils; +import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -41,18 +45,34 @@ public class ConsumerModelConverter { return consumerGroupDetailVOList; } - public static List convert2ConsumerGroupVOList(List consumeGroupDTOList) { - if (consumeGroupDTOList == null || consumeGroupDTOList.isEmpty()) { - return new ArrayList<>(); + public static List convert2ConsumerGroupVOList(List consumerGroupList) { + if (ValidateUtils.isEmptyList(consumerGroupList)) { + return Collections.emptyList(); } List consumerGroupVOList = new ArrayList<>(); - for (ConsumerGroupDTO consumeGroupDTO : consumeGroupDTOList) { + for (ConsumerGroup consumerGroup : consumerGroupList) { ConsumerGroupVO vo = new ConsumerGroupVO(); - vo.setConsumerGroup(consumeGroupDTO.getConsumerGroup()); - vo.setAppIds(ListUtils.strList2String(consumeGroupDTO.getAppIdList())); - vo.setLocation(consumeGroupDTO.getOffsetStoreLocation().location); + vo.setConsumerGroup(consumerGroup.getConsumerGroup()); + vo.setAppIds(""); + vo.setLocation(consumerGroup.getOffsetStoreLocation().location); consumerGroupVOList.add(vo); } return consumerGroupVOList; } + + public static List convert2ConsumerGroupSummaryVOList(List summaryList) { + if (ValidateUtils.isEmptyList(summaryList)) { + return Collections.emptyList(); + } + List voList = new ArrayList<>(); + for (ConsumerGroupSummary consumerGroupSummary : summaryList) { + ConsumerGroupSummaryVO vo = new ConsumerGroupSummaryVO(); + vo.setConsumerGroup(consumerGroupSummary.getConsumerGroup()); + vo.setAppIds(ListUtils.strList2String(consumerGroupSummary.getAppIdList())); + vo.setLocation(consumerGroupSummary.getOffsetStoreLocation().location); + vo.setState(consumerGroupSummary.getState()); + voList.add(vo); + } + return voList; + } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java index 7d5f3a7c5461495fca1408f65934a3b8230d4e12..f032e92198fd00bcbb5d1cf771ac4bff2398fbb8 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/GatewayModelConverter.java @@ -1,9 +1,13 @@ package com.xiaojukeji.kafka.manager.web.converters; +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionAddGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.bpm.common.entry.apply.gateway.OrderExtensionModifyGatewayConfigDTO; +import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.GatewayConfigDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.KafkaAclDO; import com.xiaojukeji.kafka.manager.common.entity.pojo.gateway.KafkaUserDO; import com.xiaojukeji.kafka.manager.common.entity.vo.gateway.KafkaAclVO; import com.xiaojukeji.kafka.manager.common.entity.vo.gateway.KafkaUserVO; +import com.xiaojukeji.kafka.manager.common.entity.vo.rd.GatewayConfigVO; import com.xiaojukeji.kafka.manager.common.utils.ValidateUtils; import java.util.ArrayList; @@ -49,4 +53,41 @@ public class GatewayModelConverter { } return voList; } + + public static List convert2GatewayConfigVOList(List doList) { + if (ValidateUtils.isNull(doList)) { + return new ArrayList<>(); + } + + List voList = new ArrayList<>(); + for (GatewayConfigDO configDO: doList) { + GatewayConfigVO vo = new GatewayConfigVO(); + vo.setId(configDO.getId()); + vo.setType(configDO.getType()); + vo.setName(configDO.getName()); + vo.setValue(configDO.getValue()); + vo.setVersion(configDO.getVersion()); + vo.setCreateTime(configDO.getCreateTime()); + vo.setModifyTime(configDO.getModifyTime()); + voList.add(vo); + } + return voList; + } + + public static GatewayConfigDO convert2GatewayConfigDO(OrderExtensionAddGatewayConfigDTO configDTO) { + GatewayConfigDO configDO = new GatewayConfigDO(); + configDO.setType(configDO.getType()); + configDO.setName(configDO.getName()); + configDO.setValue(configDO.getValue()); + return configDO; + } + + public static GatewayConfigDO convert2GatewayConfigDO(OrderExtensionModifyGatewayConfigDTO configDTO) { + GatewayConfigDO configDO = new GatewayConfigDO(); + configDO.setId(configDO.getId()); + configDO.setType(configDO.getType()); + configDO.setName(configDO.getName()); + configDO.setValue(configDO.getValue()); + return configDO; + } } \ No newline at end of file diff --git a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java index db9171eeb43b38005d7de43f5fbdef2b67fd7825..133ac0198dc89d7e33ffcd2edb2029b86a45571b 100644 --- a/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java +++ b/kafka-manager-web/src/main/java/com/xiaojukeji/kafka/manager/web/converters/TopicModelConverter.java @@ -37,6 +37,7 @@ public class TopicModelConverter { vo.setTopicCodeC(dto.getTopicCodeC()); vo.setDescription(dto.getDescription()); vo.setBootstrapServers(""); + vo.setRegionNameList(dto.getRegionNameList()); if (!ValidateUtils.isNull(clusterDO)) { vo.setBootstrapServers(clusterDO.getBootstrapServers()); } diff --git a/kafka-manager-web/src/main/resources/application.yml b/kafka-manager-web/src/main/resources/application.yml index b1ed6cd53a024a6b3ebbb60c838f89a7ab2da333..6d7d9bec90c6ccf52be154ac235ddcbb50edb61f 100644 --- a/kafka-manager-web/src/main/resources/application.yml +++ b/kafka-manager-web/src/main/resources/application.yml @@ -42,6 +42,11 @@ custom: topic-throttled-metrics: false save-days: 7 +# 任务相关的开关 +task: + op: + sync-topic-enabled: false # 未落盘的Topic定期同步到DB中 + account: ldap: