diff --git a/km-dist/docker/docker-compose.yml b/km-dist/docker/docker-compose.yml index ee4b346654d6e92762d7690d3af8678d33d75891..3e902498c7c0ae66544ab1e5ce77714b7204d09b 100644 --- a/km-dist/docker/docker-compose.yml +++ b/km-dist/docker/docker-compose.yml @@ -15,31 +15,17 @@ services: - kmysql expose: - 80 - #ports: - # - '80:80' environment: TZ: Asia/Shanghai - server.port: 80 - spring.datasource.know-streaming.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.datasource.know-streaming.username: root - spring.datasource.know-streaming.password: admin2022_ - spring.logi-job.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.logi-job.username: root - spring.logi-job.password: admin2022_ + SERVER_MYSQL_ADDRESS: kmysql:3306 + SERVER_MYSQL_DB: know_streaming + SERVER_MYSQL_USER: root + SERVER_MYSQL_PASSWORD: admin2022_ - spring.logi-security.jdbc-url: jdbc:mariadb://kmysql:3306/know_streaming?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true - spring.logi-security.username: root - spring.logi-security.password: admin2022_ - spring.logi-security.resource-extend-bean-name: myResourceExtendImpl - client-pool.kafka-consumer.max-total-client-num: 20 - - es.client.address: elasticsearch_single:9200 - ES_CLIENT_ADDRESS: elasticsearch_single:9200 - log.path: /logs + SERVER_ES_ADDRESS: elasticsearch_single:9200 JAVA_OPTS: -Xmx1g -Xms1g -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom -Duser.timezone=GMT+08 - networks: - ks volumes: @@ -63,15 +49,20 @@ services: elasticsearch_single: image: docker.io/library/elasticsearch:7.6.2 container_name: elasticsearch_single - ports: - - '9200:9200' - - '9300:9300' + restart: always + expose: + - 9200 + - 9300 +# 对外访问 +# ports: +# - '9200:9200' +# - '9300:9300' environment: TZ: Asia/Shanghai ES_JAVA_OPTS: -Xms512m -Xmx512m discovery.type: single-node - volumes: - - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data +# volumes: +# - /Users/haoqi/Downloads/ks/es/data:/usr/share/elasticsearch/data networks: - ks @@ -79,6 +70,7 @@ services: kmysql: image: ks_mysql:1.0 container_name: kmysql + restart: always environment: TZ: Asia/Shanghai MYSQL_ROOT_PASSWORD: admin2022_ diff --git a/km-dist/docker/knowstreaming-es/Dockerfile b/km-dist/docker/knowstreaming-es/Dockerfile deleted file mode 100644 index 6a6f6143fe790dd630699a1d17f3f79f13ec7ea3..0000000000000000000000000000000000000000 --- a/km-dist/docker/knowstreaming-es/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM docker.io/library/elasticsearch:7.6.2 -LABEL author=haoqi6677 -#EXPOSE 9200 -#EXPOSE 9300 -#ARG INIT_SOURCE_FILE=./esinit/* -#COPY ${INIT_SOURCE_FILE} / \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/Dockerfile b/km-dist/docker/knowstreaming-manage/Dockerfile index 021a592d3750af899900e3aa6f0799fc28b35048..05e28155f549ea7d8e3b181d7e0500ca7b8d518c 100644 --- a/km-dist/docker/knowstreaming-manage/Dockerfile +++ b/km-dist/docker/knowstreaming-manage/Dockerfile @@ -1,10 +1,11 @@ FROM bitnami/java:11 -ARG JAVA_OPTS -ARG JAR_FILE=./jar/*.jar -ARG INIT_ES_FILE=./esinit/* +ENV JAR_FILE=./jar/*.jar +ENV INIT_ES_FILE=./esinit/* +ENV CONFIG_FILE=./conf/* COPY ${JAR_FILE} /app/app.jar COPY ${INIT_ES_FILE} / +COPY ${CONFIG_FILE} /conf/ ENTRYPOINT ["/bin/bash","/es.sh"] -CMD ["dumb-init","java","${JAVA_OPTS}", "-jar", "/app/app.jar"] +CMD ["/bin/bash","/es.sh"] #ENTRYPOINT java -jar /app/app.jar \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/conf/application.yml b/km-dist/docker/knowstreaming-manage/conf/application.yml new file mode 100644 index 0000000000000000000000000000000000000000..80323cd0d67615bdbd92e1d5cffd545e59ef3b37 --- /dev/null +++ b/km-dist/docker/knowstreaming-manage/conf/application.yml @@ -0,0 +1,107 @@ +server: + port: 80 # 服务端口 + tomcat: + accept-count: 1000 + max-connections: 10000 + +spring: + application: + name: know-streaming + profiles: + active: dev + main: + allow-bean-definition-overriding: true + jackson: + time-zone: GMT+8 + datasource: + know-streaming: # know-streaming 自身数据库的配置 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + maximum-pool-size: 20 + idle-timeout: 30000 + connection-test-query: SELECT 1 + logi-job: # know-streaming 依赖的 logi-job 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + max-lifetime: 60000 + init-sql: true + init-thread-num: 20 + max-thread-num: 50 + log-expire: 3 # 日志保存天数，以天为单位 + app-name: know-streaming + claim-strategy: com.didiglobal.logi.job.core.consensual.RandomConsensual + logi-security: # know-streaming 依赖的 logi-security 模块的数据库的配置，默认与 know-streaming 的数据库配置保持一致即可 + jdbc-url: jdbc:mariadb://SERVER_MYSQL_ADDRESS/SERVER_MYSQL_DB?useUnicode=true&characterEncoding=utf8&jdbcCompliantTruncation=true&allowMultiQueries=true&useSSL=false&alwaysAutoGeneratedKeys=true&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true + username: SERVER_MYSQL_USER + password: SERVER_MYSQL_PASSWORD + driver-class-name: org.mariadb.jdbc.Driver + app-name: know-streaming + resource-extend-bean-name: myResourceExtendImpl + login-extend-bean-name: logiSecurityDefaultLoginExtendImpl + +logging: + config: classpath:logback-spring.xml + +# 线程池大小相关配置 +thread-pool: + scheduled: + thread-num: 2 # @Scheduled任务的线程池大小，默认是一个 + + collector: # 采集模块的配置 + future-util: # 采集模块线程池配置 + num: 3 # 线程池个数 + thread-num: 64 # 每个线程池核心线程数 + queue-size: 10000 # 每个线程池队列大小 + select-suitable-enable: true # 任务是否自动选择合适的线程池，非主要，可不修改 + suitable-queue-size: 1000 # 线程池理想的队列大小，非主要，可不修改 + + task: # 任务模块的配置 + metrics: # metrics采集任务配置 + thread-num: 18 # metrics采集任务线程池核心线程数 + queue-size: 180 # metrics采集任务线程池队列大小 + metadata: # metadata同步任务配置 + thread-num: 27 # metadata同步任务线程池核心线程数 + queue-size: 270 # metadata同步任务线程池队列大小 + common: # 剩余其他任务配置 + thread-num: 15 # 剩余其他任务线程池核心线程数 + queue-size: 150 # 剩余其他任务线程池队列大小 + + +# 客户端池大小相关配置 +client-pool: + kafka-consumer: + min-idle-client-num: 2 # 最小空闲客户端数 + max-idle-client-num: 20 # 最大空闲客户端数 + max-total-client-num: 20 # 最大客户端数 + borrow-timeout-unit-ms: 5000 # 租借超时时间，单位秒 + + +# ES客户端配置 +es: + client: + address: SERVER_ES_ADDRESS + client-cnt: 10 + io-thread-cnt: 2 + max-retry-cnt: 5 + +# 普罗米修斯指标导出相关配置 +management: + endpoints: + web: + base-path: /metrics + exposure: + include: '*' + health: + elasticsearch: + enabled: false + metrics: + export: + prometheus: + descriptions: true + enabled: true + tags: + application: know-streaming \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/es.sh b/km-dist/docker/knowstreaming-manage/esinit/es.sh index 9bd811bc3d9a8bc7985e38702cb9470935a6eaea..8d5383a9a974d0d29753637f9385acc7fa260a66 100644 --- a/km-dist/docker/knowstreaming-manage/esinit/es.sh +++ b/km-dist/docker/knowstreaming-manage/esinit/es.sh @@ -1,2 +1,9 @@ /bin/bash /template.sh start -java ${JAVA_OPTS} -jar /app/app.jar \ No newline at end of file + +sed -i "s/SERVER_MYSQL_ADDRESS/${SERVER_MYSQL_ADDRESS}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_DB/${SERVER_MYSQL_DB}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_USER/${SERVER_MYSQL_USER}/g" /conf/application.yml +sed -i "s/SERVER_MYSQL_PASSWORD/${SERVER_MYSQL_PASSWORD}/g" /conf/application.yml +sed -i "s/SERVER_ES_ADDRESS/${SERVER_ES_ADDRESS}/g" /conf/application.yml + +java ${JAVA_OPTS} -jar /app/app.jar --spring.config.location=/conf/application.yml \ No newline at end of file diff --git a/km-dist/docker/knowstreaming-manage/esinit/template.sh b/km-dist/docker/knowstreaming-manage/esinit/template.sh index bd69d1d10365b5d45568c970cb58c19dac5951d9..4309727942e9adbb09ee287dffc32ec2d21ed580 100644 --- a/km-dist/docker/knowstreaming-manage/esinit/template.sh +++ b/km-dist/docker/knowstreaming-manage/esinit/template.sh @@ -1,17 +1,16 @@ -esAddrPort=${ES_CLIENT_ADDRESS} echo "Wait ElasticSearch Start..." while true do - curl -s --connect-timeout 10 -o /dev/null http://${esAddrPort}/_cat/nodes >> /dev/null 2>&1 + curl -s --connect-timeout 10 -o /dev/null http://${SERVER_ES_ADDRESS}/_cat/nodes >> /dev/null 2>&1 if [ "$?" != "0" ];then sleep 1s else - echo "ElasticSearch Start Success" + echo "ElasticSearch Start Initialize" break fi done -curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_broker_metric -d '{ +curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_broker_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_broker_metric*" @@ -113,7 +112,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' - "aliases" : { } }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_cluster_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_cluster_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_cluster_metric*" @@ -298,9 +297,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_group_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_group_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_group_metric*" @@ -373,9 +372,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_partition_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_partition_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_partition_metric*" @@ -439,9 +438,9 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_replication_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_partition_metric*" @@ -528,9 +527,9 @@ PUT _template/ks_kafka_replication_metric } }, "aliases" : { } - }' + }' -curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esAddrPort}/_template/ks_kafka_topic_metric -d '{ +curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{ "order" : 10, "index_patterns" : [ "ks_kafka_topic_metric*" @@ -645,17 +644,17 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl } }, "aliases" : { } - }' + }' -for i in `seq 6`; +for i in {0..6}; do logdate=_$(date -d "${i} day ago" +%Y-%m-%d) - curl -s --connect-timeout 10 -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_broker_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_cluster_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_group_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_partition_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_replication_metric${logdate} && \ - curl -s -o /dev/null -X PUT http://${esAddrPort}/ks_kafka_topic_metric${logdate} || \ + curl -s --connect-timeout 10 -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_broker_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_cluster_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \ + curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \ exit 2 done diff --git a/km-dist/docker/knowstreaming-mysql/Dockerfile b/km-dist/docker/knowstreaming-mysql/Dockerfile index 1c5d8b9b13b61c014dc467f8f63b0fc961132248..edaf3ad8d080e6fdb976a5b8fba4d7568c9480af 100644 --- a/km-dist/docker/knowstreaming-mysql/Dockerfile +++ b/km-dist/docker/knowstreaming-mysql/Dockerfile @@ -1,6 +1,6 @@ FROM mysql:5.7.39-oracle EXPOSE 3306 -ARG INIT_SOURCE_FILE=./sql/*.sql +ENV INIT_SOURCE_FILE=./sql/*.sql #RUN mkdir /data/mysql #RUN mkdir /var/lib/mysql-files COPY ${INIT_SOURCE_FILE} /docker-entrypoint-initdb.d/