Skip to content

K
kafka-manager

DiDi / kafka-manager
9 个月 前同步成功

通知 58
Star 6372
Fork 1229
K
kafka-manager

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 9a25c22f 编写于 作者: H haoqi
浏览文件
操作

1. 调整docker-compose.yml中各个服务的镜像 

2. 经过@wangdongfang-aden大哥的调试将helm与docker镜像合二为一，于是删减掉各个镜像的Dockerfile与启动脚本，后续也不需要额外维护
上级 a7a5fa9a
隐藏空白更改
内联 并排
Showing with 16 addition and 941 deletion
km-dist/docker/docker-compose.yml
浏览文件 @ 9a25c22f
......@@ -3,7 +3,7 @@ version: "3"
services:
knowstreaming-manager:
image: haiti/knowstreaming-manager:0.1.0
image: knowstreaming/knowstreaming-manager:0.2.0-test
container_name: knowstreaming-manager
privileged: true
restart: always
......@@ -12,6 +12,9 @@ services:
- knowstreaming-mysql
expose:
- 80
command:
- /bin/sh
- /ks-start.sh
environment:
TZ: Asia/Shanghai
......@@ -27,9 +30,9 @@ services:
# extra_hosts:
# - "hostname:x.x.x.x"
# volumes:
# - /Users/haoqi/Downloads/ks/manage/log:/logs
# - /ks/manage/log:/logs
knowstreaming-ui:
image: haiti/knowstreaming-ui:0.1.0
image: knowstreaming/knowstreaming-ui:0.2.0-test1
container_name: knowstreaming-ui
restart: always
ports:
......@@ -56,27 +59,33 @@ services:
ES_JAVA_OPTS: -Xms512m -Xmx512m
discovery.type: single-node
# volumes:
# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
# - /ks/es/data:/usr/share/elasticsearch/data
knowstreaming-init:
image: haiti/knowstreaming-init:0.1.0
image: knowstreaming/knowstreaming-manager:0.2.0-test
container_name: knowstreaming_init
depends_on:
- elasticsearch-single
command:
- /bin/bash
- /es_template_create.sh
environment:
TZ: Asia/Shanghai
SERVER_ES_ADDRESS: elasticsearch-single:9200
knowstreaming-mysql:
image: haiti/knowstreaming-mysql:0.1.0
image: knowstreaming/knowstreaming-mysql:0.2.0-test
container_name: knowstreaming-mysql
restart: always
environment:
TZ: Asia/Shanghai
MYSQL_ROOT_PASSWORD: admin2022_
MYSQL_DATABASE: know_streaming
MYSQL_ROOT_HOST: '%'
expose:
- 3306
# ports:
# - '3306:3306'
# volumes:
# - /Users/haoqi/Downloads/ks/mysql/data:/data/mysql
\ No newline at end of file
# - /ks/mysql/data:/data/mysql
\ No newline at end of file
km-dist/docker/knowstreaming-init/Dockerfile 已删除 100644 → 0
浏览文件 @ a7a5fa9a
FROM debian:stable-20220912-slim
LABEL org.opencontainers.image.authors="haoqi123"
ARG INIT_ES_FILE=./init/*
COPY ${INIT_ES_FILE} /
ENV TZ=Asia/Shanghai
RUN apt-get update \
&& apt-get install -y curl
ENTRYPOINT ["/bin/bash","/template.sh","start"]
\ No newline at end of file
km-dist/docker/knowstreaming-init/init/template.sh 已删除 100644 → 0
浏览文件 @ a7a5fa9a
echo "Wait ElasticSearch Start...${SERVER_ES_ADDRESS}"
while true
do
curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes > /dev/null 2>&1
if [ "$?" != "0" ];then
sleep 1s
else
echo "ElasticSearch Start Initialize"
break
fi
done
curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_broker_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"NetworkProcessorAvgIdle" : {
"type" : "float"
},
"UnderReplicatedPartitions" : {
"type" : "float"
},
"BytesIn_min_15" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"RequestHandlerAvgIdle" : {
"type" : "float"
},
"connectionsCount" : {
"type" : "float"
},
"BytesIn_min_5" : {
"type" : "float"
},
"HealthScore" : {
"type" : "float"
},
"BytesOut" : {
"type" : "float"
},
"BytesOut_min_15" : {
"type" : "float"
},
"BytesIn" : {
"type" : "float"
},
"BytesOut_min_5" : {
"type" : "float"
},
"TotalRequestQueueSize" : {
"type" : "float"
},
"MessagesIn" : {
"type" : "float"
},
"TotalProduceRequests" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
},
"TotalResponseQueueSize" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}'
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_cluster_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"Connections" : {
"type" : "double"
},
"BytesIn_min_15" : {
"type" : "double"
},
"PartitionURP" : {
"type" : "double"
},
"HealthScore_Topics" : {
"type" : "double"
},
"EventQueueSize" : {
"type" : "double"
},
"ActiveControllerCount" : {
"type" : "double"
},
"GroupDeads" : {
"type" : "double"
},
"BytesIn_min_5" : {
"type" : "double"
},
"HealthCheckTotal_Topics" : {
"type" : "double"
},
"Partitions" : {
"type" : "double"
},
"BytesOut" : {
"type" : "double"
},
"Groups" : {
"type" : "double"
},
"BytesOut_min_15" : {
"type" : "double"
},
"TotalRequestQueueSize" : {
"type" : "double"
},
"HealthCheckPassed_Groups" : {
"type" : "double"
},
"TotalProduceRequests" : {
"type" : "double"
},
"HealthCheckPassed" : {
"type" : "double"
},
"TotalLogSize" : {
"type" : "double"
},
"GroupEmptys" : {
"type" : "double"
},
"PartitionNoLeader" : {
"type" : "double"
},
"HealthScore_Brokers" : {
"type" : "double"
},
"Messages" : {
"type" : "double"
},
"Topics" : {
"type" : "double"
},
"PartitionMinISR_E" : {
"type" : "double"
},
"HealthCheckTotal" : {
"type" : "double"
},
"Brokers" : {
"type" : "double"
},
"Replicas" : {
"type" : "double"
},
"HealthCheckTotal_Groups" : {
"type" : "double"
},
"GroupRebalances" : {
"type" : "double"
},
"MessageIn" : {
"type" : "double"
},
"HealthScore" : {
"type" : "double"
},
"HealthCheckPassed_Topics" : {
"type" : "double"
},
"HealthCheckTotal_Brokers" : {
"type" : "double"
},
"PartitionMinISR_S" : {
"type" : "double"
},
"BytesIn" : {
"type" : "double"
},
"BytesOut_min_5" : {
"type" : "double"
},
"GroupActives" : {
"type" : "double"
},
"MessagesIn" : {
"type" : "double"
},
"GroupReBalances" : {
"type" : "double"
},
"HealthCheckPassed_Brokers" : {
"type" : "double"
},
"HealthScore_Groups" : {
"type" : "double"
},
"TotalResponseQueueSize" : {
"type" : "double"
},
"Zookeepers" : {
"type" : "double"
},
"LeaderMessages" : {
"type" : "double"
},
"HealthScore_Cluster" : {
"type" : "double"
},
"HealthCheckPassed_Cluster" : {
"type" : "double"
},
"HealthCheckTotal_Cluster" : {
"type" : "double"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"type" : "date"
}
}
},
"aliases" : { }
}'
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_group_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"group" : {
"type" : "keyword"
},
"partitionId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"topic" : {
"type" : "keyword"
},
"metrics" : {
"properties" : {
"HealthScore" : {
"type" : "float"
},
"Lag" : {
"type" : "float"
},
"OffsetConsumed" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
}
}
},
"groupMetric" : {
"type" : "keyword"
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}'
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_partition_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"partitionId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"topic" : {
"type" : "keyword"
},
"metrics" : {
"properties" : {
"LogStartOffset" : {
"type" : "float"
},
"Messages" : {
"type" : "float"
},
"LogEndOffset" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}'
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_partition_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"partitionId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"clusterPhyId" : {
"type" : "long"
},
"topic" : {
"type" : "keyword"
},
"metrics" : {
"properties" : {
"LogStartOffset" : {
"type" : "float"
},
"Messages" : {
"type" : "float"
},
"LogEndOffset" : {
"type" : "float"
}
}
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}[root@10-255-0-23 template]# cat ks_kafka_replication_metric
PUT _template/ks_kafka_replication_metric
{
"order" : 10,
"index_patterns" : [
"ks_kafka_replication_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}'
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{
"order" : 10,
"index_patterns" : [
"ks_kafka_topic_metric*"
],
"settings" : {
"index" : {
"number_of_shards" : "10"
}
},
"mappings" : {
"properties" : {
"brokerId" : {
"type" : "long"
},
"routingValue" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"topic" : {
"type" : "keyword"
},
"clusterPhyId" : {
"type" : "long"
},
"metrics" : {
"properties" : {
"BytesIn_min_15" : {
"type" : "float"
},
"Messages" : {
"type" : "float"
},
"BytesRejected" : {
"type" : "float"
},
"PartitionURP" : {
"type" : "float"
},
"HealthCheckTotal" : {
"type" : "float"
},
"ReplicationCount" : {
"type" : "float"
},
"ReplicationBytesOut" : {
"type" : "float"
},
"ReplicationBytesIn" : {
"type" : "float"
},
"FailedFetchRequests" : {
"type" : "float"
},
"BytesIn_min_5" : {
"type" : "float"
},
"HealthScore" : {
"type" : "float"
},
"LogSize" : {
"type" : "float"
},
"BytesOut" : {
"type" : "float"
},
"BytesOut_min_15" : {
"type" : "float"
},
"FailedProduceRequests" : {
"type" : "float"
},
"BytesIn" : {
"type" : "float"
},
"BytesOut_min_5" : {
"type" : "float"
},
"MessagesIn" : {
"type" : "float"
},
"TotalProduceRequests" : {
"type" : "float"
},
"HealthCheckPassed" : {
"type" : "float"
}
}
},
"brokerAgg" : {
"type" : "keyword"
},
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"timestamp" : {
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
"index" : true,
"type" : "date",
"doc_values" : true
}
}
},
"aliases" : { }
}'
for i in {0..6};
do
logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \
exit 2
done
echo "ElasticSearch Initialize Success"
\ No newline at end of file
km-dist/docker/knowstreaming-manage/Dockerfile 已删除 100644 → 0
浏览文件 @ a7a5fa9a
FROM docker.io/amd64/eclipse-temurin:17.0.4.1_1-jdk-centos7
LABEL maintainer="haoqi123 <1148648445@qq.com>"
WORKDIR /knowstreaming
ARG VERSION_MAVEN=3.6.3
ARG VERSION_KM=3.0.0-beta.2
ARG INIT_FILE=./init/*
ARG CONFIG_FILE=./conf/*
COPY ${INIT_FILE} /
COPY ${CONFIG_FILE} /conf/
# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN yum install -y git
RUN curl -fsSLO --compressed "https://dlcdn.apache.org/maven/maven-3/${VERSION_MAVEN}/binaries/apache-maven-${VERSION_MAVEN}-bin.tar.gz" \
&& mkdir maven/ \
&& tar -zxf "apache-maven-${VERSION_MAVEN}-bin.tar.gz" -C maven/ --strip-components=1 --no-same-owner \
&& rm -rf /knowstreaming/*.tar.gz \
&& ln -s /knowstreaming/maven/bin/mvn /usr/local/bin/mvn \
# && curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/${VERSION_KM}.zip \
&& git clone https://github.com/didi/KnowStreaming.git \
# && unzip -o -q master.zip \
&& cd KnowStreaming && git checkout v${VERSION_KM} \
&& sed -i "/km-console/d" ./pom.xml \
&& mvn -U clean package -Dmaven.test.skip=true \
&& cd / && mkdir /app \
&& mv /knowstreaming/KnowStreaming/km-rest/target/ks-km.jar /app/app.jar \
&& rm -rf /knowstreaming/* /usr/local/bin/mvn \
&& yum erase -y git && yum clean all \
&& rm -rf /root/.m2 /tmp/*
ENTRYPOINT ["/bin/bash","/start.sh"]
HEALTHCHECK --interval=60s --timeout=60s CMD curl -f http://localhost/ks-km/api/v3/open/health || exit 1
km-dist/docker/knowstreaming-manage/conf/application.yml 已删除 100644 → 0
浏览文件 @ a7a5fa9a
server:
port: 80 # 服务端口
tomcat:
accept-count: 1000
max-connections: 10000
spring:
application:
name: know-streaming
profiles:
active: dev
main:
allow-bean-definition-overriding: true
jackson:
time-zone: GMT+8
datasource:
know-streaming: # know-streaming 自身数据库的配置
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username: SERVER_MYSQL_USER
password: SERVER_MYSQL_PASSWORD
driver-class-name: org.mariadb.jdbc.Driver
maximum-pool-size: 20
idle-timeout: 30000
connection-test-query: SELECT 1
logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username: SERVER_MYSQL_USER
password: SERVER_MYSQL_PASSWORD
driver-class-name: org.mariadb.jdbc.Driver
max-lifetime: 60000
init-sql: true
init-thread-num: 20
max-thread-num: 50
log-expire: 3 # 日志保存天数，以天为单位
app-name: know-streaming
claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual
logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可
jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username: SERVER_MYSQL_USER
password: SERVER_MYSQL_PASSWORD
driver-class-name: org.mariadb.jdbc.Driver
app-name: know-streaming
resource-extend-bean-name: myResourceExtendImpl
login-extend-bean-name: logiSecurityDefaultLoginExtendImpl
logging:
config: classpath:logback-spring.xml
# 线程池大小相关配置
thread-pool:
scheduled:
thread-num: 2 # @Scheduled任务的线程池大小，默认是一个
collector: # 采集模块的配置
future-util: # 采集模块线程池配置
num: 3 # 线程池个数
thread-num: 64 # 每个线程池核心线程数
queue-size: 10000 # 每个线程池队列大小
select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改
suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改
task: # 任务模块的配置
metrics: # metrics采集任务配置
thread-num: 18 # metrics采集任务线程池核心线程数
queue-size: 180 # metrics采集任务线程池队列大小
metadata: # metadata同步任务配置
thread-num: 27 # metadata同步任务线程池核心线程数
queue-size: 270 # metadata同步任务线程池队列大小
common: # 剩余其他任务配置
thread-num: 15 # 剩余其他任务线程池核心线程数
queue-size: 150 # 剩余其他任务线程池队列大小
# 客户端池大小相关配置
client-pool:
kafka-consumer:
min-idle-client-num: 2 # 最小空闲客户端数
max-idle-client-num: 20 # 最大空闲客户端数
max-total-client-num: 20 # 最大客户端数
borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒
# ES客户端配置
es:
client:
address: SERVER_ES_ADDRESS
client-cnt: 10
io-thread-cnt: 2
max-retry-cnt: 5
# 普罗米修斯指标导出相关配置
management:
endpoints:
web:
base-path: /metrics
exposure:
include: '*'
health:
elasticsearch:
enabled: false
metrics:
export:
prometheus:
descriptions: true
enabled: true
tags:
application: know-streaming
\ No newline at end of file
km-dist/docker/knowstreaming-manage/init/start.sh 已删除 100644 → 0
浏览文件 @ a7a5fa9a
sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml
sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml
sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml
sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml
sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml
java -server ${JAVA_OPTS} -XX:+UseStringDeduplication -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs -XX:ErrorFile=/logs/jvm/hs_err_pid%p.log -jar /app/app.jar --spring.config.location=/conf/application.yml
\ No newline at end of file
km-dist/docker/knowstreaming-mysql/Dockerfile 已删除 100644 → 0
浏览文件 @ a7a5fa9a
FROM mysql:5.7.39-oracle
LABEL org.opencontainers.image.authors="haoqi123"
EXPOSE 3306
ARG INIT_SOURCE_FILE=./sql/*.sql
ARG MYSQL_CNF_FILE=./conf/*.cnf
COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/
COPY ${MYSQL_CNF_FILE} /etc/mysql/conf.d/
\ No newline at end of file
km-dist/docker/knowstreaming-mysql/conf/mysql.cnf 已删除 100644 → 0
浏览文件 @ a7a5fa9a
[client]
default-character-set=utf8
[mysql]
default-character-set=utf8
[mysqld]
skip-host-cache
skip-name-resolve
datadir=/data/mysql
socket=/var/lib/mysql/mysql.sock
secure-file-priv=/var/lib/mysql-files
character-set-server=utf8
user=mysql
symbolic-links=0
pid-file=/var/run/mysqld/mysqld.pid
sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
\ No newline at end of file
km-dist/docker/knowstreaming-web/Dockerfile 已删除 100644 → 0
浏览文件 @ a7a5fa9a
FROM docker.io/amd64/nginx:1.22
LABEL maintainer="haoqi123 <1148648445@qq.com>"
WORKDIR /knowstreaming
ARG VERSION_NODE=14.20.0
ARG VERSION_KM=3.0.0-beta.2
# SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt update \
&& apt install -y git unzip
RUN echo 'server { \
listen 80; \
server_name localhost; \
gzip on; \
gzip_buffers 16 8k; \
gzip_comp_level 4; \
gzip_http_version 1.0; \
gzip_min_length 1280; \
gzip_types text/plain text/css text/xml application/x-javascript application/xml application/xml+rss application/json application/javascript text/*; \
gzip_vary on; \
root /pub; \
location / { \
root /pub; \
if ($request_filename ~* .*\.(?:htm|html|json)$) { \
add_header Cache-Control "private, no-store, no-cache, must-revalidate, proxy-revalidate"; \
} \
try_files $uri /layout/index.html; \
} \
location ~* \.(json)$ { \
add_header Cache-Control no-cache; \
} \
location @kmfallback { \
} \
#location ~ ^/(clusters|config|cluster|login) { \
# rewrite ^.*$ /; \
#} \
location ~ ^/ks-km/api/v3 { \
#rewrite ^/ks-km/api/v3/(.*)$ /ks-km/ks-km/api/v3/$1 break; \
proxy_pass http://knowstreaming-manager; \
#proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \
#proxy_cookie_path /ks-km/ /; \
#proxy_set_header Host $host; \
#proxy_set_header Referer $http_referer; \
#proxy_set_header Cookie $http_cookie; \
#proxy_set_header X-Real-Ip $remote_addr; \
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; \
} \
location ~ ^/logi-security/api/v1 { \
#rewrite ^/logi-security/api/v1/(.*)$ /ks-km/logi-security/api/v1/$1 break; \
proxy_pass http://knowstreaming-manager; \
#proxy_pass https://api-kylin-xg02.intra.xiaojukeji.com; \
} \
location ~ ^/(401|403|404|500){ \
rewrite ^.*$ /; \
} \
}' > /etc/nginx/conf.d/default.conf
RUN curl -fsSLO --compressed "https://npm.taobao.org/mirrors/node/v${VERSION_NODE}/node-v${VERSION_NODE}-linux-x64.tar.gz" \
&& mkdir node/ \
&& tar -zxf "node-v${VERSION_NODE}-linux-x64.tar.gz" -C node/ --strip-components=1 --no-same-owner \
&& rm -rf /knowstreaming/*.tar.gz \
&& ln -s /knowstreaming/node/bin/node /usr/local/bin/node \
&& ln -s /knowstreaming/node/bin/npm /usr/local/bin/npm \
&& npm config set strict-ssl false \
&& npm config set registry https://registry.npm.taobao.org \
&& npm info underscore \
&& node --version \
&& npm --version \
&& curl -fsSLO --compressed https://github.com/didi/KnowStreaming/archive/refs/tags/v${VERSION_KM}.zip \
&& unzip -o -q v${VERSION_KM}.zip \
&& cd KnowStreaming-${VERSION_KM}/km-console \
&& npm install -g lerna && npm run i && npm run build \
&& mv /knowstreaming/KnowStreaming-${VERSION_KM}/km-rest/src/main/resources/templates /pub \
&& npm uninstall -g lerna \
&& cd / \
&& rm -rf /knowstreaming/* /usr/local/bin/node /usr/local/bin/npm \
&& apt remove -y git unzip && apt autoremove -y && apt remove --purge --auto-remove -y \
&& rm -rf /var/lib/apt/lists/* /tmp /root/.npm
#ENTRYPOINT ["ls", "/ks_ui/km-console"]
#CMD ["nginx", "-g", "daemon off;"]
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或