Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
DiDi
kafka-manager
提交
0c14c641
K
kafka-manager
项目概览
DiDi
/
kafka-manager
10 个月 前同步成功
通知
58
Star
6372
Fork
1229
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
K
kafka-manager
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
0c14c641
编写于
9月 15, 2022
作者:
H
haoqi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
1. 添加docker-compose部署方式
2. 更改manage服务初始化方式 3. 更改es初始化方式
上级
61efdf49
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
157 addition
and
57 deletion
+157
-57
km-dist/docker/docker-compose.yml
km-dist/docker/docker-compose.yml
+16
-24
km-dist/docker/knowstreaming-es/Dockerfile
km-dist/docker/knowstreaming-es/Dockerfile
+0
-6
km-dist/docker/knowstreaming-manage/Dockerfile
km-dist/docker/knowstreaming-manage/Dockerfile
+5
-4
km-dist/docker/knowstreaming-manage/conf/application.yml
km-dist/docker/knowstreaming-manage/conf/application.yml
+107
-0
km-dist/docker/knowstreaming-manage/esinit/es.sh
km-dist/docker/knowstreaming-manage/esinit/es.sh
+8
-1
km-dist/docker/knowstreaming-manage/esinit/template.sh
km-dist/docker/knowstreaming-manage/esinit/template.sh
+20
-21
km-dist/docker/knowstreaming-mysql/Dockerfile
km-dist/docker/knowstreaming-mysql/Dockerfile
+1
-1
未找到文件。
km-dist/docker/docker-compose.yml
浏览文件 @
0c14c641
...
...
@@ -15,31 +15,17 @@ services:
-
kmysql
expose
:
-
80
#ports:
# - '80:80'
environment
:
TZ
:
Asia/Shanghai
server.port
:
80
spring.datasource.know-streaming.jdbc-url
:
jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
spring.datasource.know-streaming.username
:
root
spring.datasource.know-streaming.password
:
admin2022_
spring.logi-job.jdbc-url
:
jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
spring.logi-job.username
:
root
spring.logi-job.password
:
admin2022_
SERVER_MYSQL_ADDRESS
:
kmysql:3306
SERVER_MYSQL_DB
:
know_streaming
SERVER_MYSQL_USER
:
root
SERVER_MYSQL_PASSWORD
:
admin2022_
spring.logi-security.jdbc-url
:
jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
spring.logi-security.username
:
root
spring.logi-security.password
:
admin2022_
spring.logi-security.resource-extend-bean-name
:
myResourceExtendImpl
client-pool.kafka-consumer.max-total-client-num
:
20
es.client.address
:
elasticsearch_single:9200
ES_CLIENT_ADDRESS
:
elasticsearch_single:9200
log.path
:
/logs
SERVER_ES_ADDRESS
:
elasticsearch_single:9200
JAVA_OPTS
:
-Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08
networks
:
-
ks
volumes
:
...
...
@@ -63,15 +49,20 @@ services:
elasticsearch_single
:
image
:
docker.io/library/elasticsearch:7.6.2
container_name
:
elasticsearch_single
ports
:
-
'
9200:9200'
-
'
9300:9300'
restart
:
always
expose
:
-
9200
-
9300
# 对外访问
# ports:
# - '9200:9200'
# - '9300:9300'
environment
:
TZ
:
Asia/Shanghai
ES_JAVA_OPTS
:
-Xms512m -Xmx512m
discovery.type
:
single-node
volumes
:
-
/Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
#
volumes:
#
- /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data
networks
:
-
ks
...
...
@@ -79,6 +70,7 @@ services:
kmysql
:
image
:
ks_mysql:1.0
container_name
:
kmysql
restart
:
always
environment
:
TZ
:
Asia/Shanghai
MYSQL_ROOT_PASSWORD
:
admin2022_
...
...
km-dist/docker/knowstreaming-es/Dockerfile
已删除
100644 → 0
浏览文件 @
61efdf49
FROM
docker.io/library/elasticsearch:7.6.2
LABEL
author=haoqi6677
#EXPOSE 9200
#EXPOSE 9300
#ARG INIT_SOURCE_FILE=./esinit/*
#COPY ${INIT_SOURCE_FILE} /
\ No newline at end of file
km-dist/docker/knowstreaming-manage/Dockerfile
浏览文件 @
0c14c641
FROM
bitnami/java:11
ARG
JAVA_OPTS
ARG
JAR_FILE=./jar/*.jar
ARG
INIT_ES_FILE=./esinit
/*
ENV
JAR_FILE=./jar/*.jar
ENV
INIT_ES_FILE=./esinit/*
ENV
CONFIG_FILE=./conf
/*
COPY
${JAR_FILE} /app/app.jar
COPY
${INIT_ES_FILE} /
COPY
${CONFIG_FILE} /conf/
ENTRYPOINT
["/bin/bash","/es.sh"]
CMD
["
dumb-init","java","${JAVA_OPTS}", "-jar", "/app/app.jar
"]
CMD
["
/bin/bash","/es.sh
"]
#ENTRYPOINT java -jar /app/app.jar
\ No newline at end of file
km-dist/docker/knowstreaming-manage/conf/application.yml
0 → 100644
浏览文件 @
0c14c641
server
:
port
:
80
# 服务端口
tomcat
:
accept-count
:
1000
max-connections
:
10000
spring
:
application
:
name
:
know-streaming
profiles
:
active
:
dev
main
:
allow-bean-definition-overriding
:
true
jackson
:
time-zone
:
GMT+8
datasource
:
know-streaming
:
# know-streaming 自身数据库的配置
jdbc-url
:
jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username
:
SERVER_MYSQL_USER
password
:
SERVER_MYSQL_PASSWORD
driver-class-name
:
org.mariadb.jdbc.Driver
maximum-pool-size
:
20
idle-timeout
:
30000
connection-test-query
:
SELECT
1
logi-job
:
# know-streaming 依赖的 logi-job 模块的数据库的配置,默认与 know-streaming 的数据库配置保持一致即可
jdbc-url
:
jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username
:
SERVER_MYSQL_USER
password
:
SERVER_MYSQL_PASSWORD
driver-class-name
:
org.mariadb.jdbc.Driver
max-lifetime
:
60000
init-sql
:
true
init-thread-num
:
20
max-thread-num
:
50
log-expire
:
3
# 日志保存天数,以天为单位
app-name
:
know-streaming
claim-strategy
:
com.didiglobal.logi.job.core.consensual.RandomConsensual
logi-security
:
# know-streaming 依赖的 logi-security 模块的数据库的配置,默认与 know-streaming 的数据库配置保持一致即可
jdbc-url
:
jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true
username
:
SERVER_MYSQL_USER
password
:
SERVER_MYSQL_PASSWORD
driver-class-name
:
org.mariadb.jdbc.Driver
app-name
:
know-streaming
resource-extend-bean-name
:
myResourceExtendImpl
login-extend-bean-name
:
logiSecurityDefaultLoginExtendImpl
logging
:
config
:
classpath:logback-spring.xml
# 线程池大小相关配置
thread-pool
:
scheduled
:
thread-num
:
2
# @Scheduled任务的线程池大小,默认是一个
collector
:
# 采集模块的配置
future-util
:
# 采集模块线程池配置
num
:
3
# 线程池个数
thread-num
:
64
# 每个线程池核心线程数
queue-size
:
10000
# 每个线程池队列大小
select-suitable-enable
:
true
# 任务是否自动选择合适的线程池,非主要,可不修改
suitable-queue-size
:
1000
# 线程池理想的队列大小,非主要,可不修改
task
:
# 任务模块的配置
metrics
:
# metrics采集任务配置
thread-num
:
18
# metrics采集任务线程池核心线程数
queue-size
:
180
# metrics采集任务线程池队列大小
metadata
:
# metadata同步任务配置
thread-num
:
27
# metadata同步任务线程池核心线程数
queue-size
:
270
# metadata同步任务线程池队列大小
common
:
# 剩余其他任务配置
thread-num
:
15
# 剩余其他任务线程池核心线程数
queue-size
:
150
# 剩余其他任务线程池队列大小
# 客户端池大小相关配置
client-pool
:
kafka-consumer
:
min-idle-client-num
:
2
# 最小空闲客户端数
max-idle-client-num
:
20
# 最大空闲客户端数
max-total-client-num
:
20
# 最大客户端数
borrow-timeout-unit-ms
:
5000
# 租借超时时间,单位秒
# ES客户端配置
es
:
client
:
address
:
SERVER_ES_ADDRESS
client-cnt
:
10
io-thread-cnt
:
2
max-retry-cnt
:
5
# 普罗米修斯指标导出相关配置
management
:
endpoints
:
web
:
base-path
:
/metrics
exposure
:
include
:
'
*'
health
:
elasticsearch
:
enabled
:
false
metrics
:
export
:
prometheus
:
descriptions
:
true
enabled
:
true
tags
:
application
:
know-streaming
\ No newline at end of file
km-dist/docker/knowstreaming-manage/esinit/es.sh
浏览文件 @
0c14c641
/bin/bash /template.sh start
java
${
JAVA_OPTS
}
-jar
/app/app.jar
\ No newline at end of file
sed
-i
"s/SERVER_MYSQL_ADDRESS/
${
SERVER_MYSQL_ADDRESS
}
/g"
/conf/application.yml
sed
-i
"s/SERVER_MYSQL_DB/
${
SERVER_MYSQL_DB
}
/g"
/conf/application.yml
sed
-i
"s/SERVER_MYSQL_USER/
${
SERVER_MYSQL_USER
}
/g"
/conf/application.yml
sed
-i
"s/SERVER_MYSQL_PASSWORD/
${
SERVER_MYSQL_PASSWORD
}
/g"
/conf/application.yml
sed
-i
"s/SERVER_ES_ADDRESS/
${
SERVER_ES_ADDRESS
}
/g"
/conf/application.yml
java
${
JAVA_OPTS
}
-jar
/app/app.jar
--spring
.config.location
=
/conf/application.yml
\ No newline at end of file
km-dist/docker/knowstreaming-manage/esinit/template.sh
浏览文件 @
0c14c641
esAddrPort
=
${
ES_CLIENT_ADDRESS
}
echo
"Wait ElasticSearch Start..."
while
true
do
curl
-s
--connect-timeout
10
-o
/dev/null http://
${
esAddrPort
}
/_cat/nodes
>>
/dev/null 2>&1
curl
-s
--connect-timeout
10
-o
/dev/null http://
${
SERVER_ES_ADDRESS
}
/_cat/nodes
>>
/dev/null 2>&1
if
[
"
$?
"
!=
"0"
]
;
then
sleep
1s
else
echo
"ElasticSearch Start
Success
"
echo
"ElasticSearch Start
Initialize
"
break
fi
done
curl
-s
--connect-timeout
10
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_broker_metric
-d
'{
curl
-s
--connect-timeout
10
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_broker_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_broker_metric*"
...
...
@@ -113,7 +112,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -
"aliases" : { }
}'
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_cluster_metric
-d
'{
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_cluster_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_cluster_metric*"
...
...
@@ -298,9 +297,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
}
},
"aliases" : { }
}'
}'
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_group_metric
-d
'{
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_group_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_group_metric*"
...
...
@@ -373,9 +372,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
}
},
"aliases" : { }
}'
}'
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_partition_metric
-d
'{
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_partition_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_partition_metric*"
...
...
@@ -439,9 +438,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
}
},
"aliases" : { }
}'
}'
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_replication_metric
-d
'{
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_replication_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_partition_metric*"
...
...
@@ -528,9 +527,9 @@ PUT _template/ks_kafka_replication_metric
}
},
"aliases" : { }
}'
}'
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
esAddrPort
}
/_template/ks_kafka_topic_metric
-d
'{
curl
-s
-o
/dev/null
-X
POST
-H
'cache-control: no-cache'
-H
'content-type: application/json'
http://
${
SERVER_ES_ADDRESS
}
/_template/ks_kafka_topic_metric
-d
'{
"order" : 10,
"index_patterns" : [
"ks_kafka_topic_metric*"
...
...
@@ -645,17 +644,17 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
}
},
"aliases" : { }
}'
}'
for
i
in
`
seq
6
`
;
for
i
in
{
0..6
}
;
do
logdate
=
_
$(
date
-d
"
${
i
}
day ago"
+%Y-%m-%d
)
curl
-s
--connect-timeout
10
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_broker_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_cluster_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_group_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_partition_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_replication_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
esAddrPort
}
/ks_kafka_topic_metric
${
logdate
}
||
\
curl
-s
--connect-timeout
10
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_broker_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_cluster_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_group_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_partition_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_replication_metric
${
logdate
}
&&
\
curl
-s
-o
/dev/null
-X
PUT http://
${
SERVER_ES_ADDRESS
}
/ks_kafka_topic_metric
${
logdate
}
||
\
exit
2
done
...
...
km-dist/docker/knowstreaming-mysql/Dockerfile
浏览文件 @
0c14c641
FROM
mysql:5.7.39-oracle
EXPOSE
3306
ARG
INIT_SOURCE_FILE=./sql/*.sql
ENV
INIT_SOURCE_FILE=./sql/*.sql
#RUN mkdir /data/mysql
#RUN mkdir /var/lib/mysql-files
COPY
${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录