From b76839fe4ca7d02b3c09c7a66968c3d1e4323d71 Mon Sep 17 00:00:00 2001 From: Gao Hongtao Date: Thu, 16 May 2019 09:37:56 +0800 Subject: [PATCH] Update docker building tool (#2677) * Update docker building of oap server * generating appliation.yml according to enviroment variables * adding Makefile to build binary and docker image * Update ui docker * Update docker-compose * Update document * Update document * Merge from master * Reset skywalking-ui --- Makefile | 95 ++++++ docker/Makefile | 31 -- docker/README.md | 12 - docker/config/alarm-settings.yml | 72 ----- docker/config/application.yml | 117 ------- docker/config/component-libraries.yml | 311 ------------------- docker/config/datasource-settings.properties | 31 -- docker/config/log4j2.xml | 37 --- docker/docker-compose.yml | 13 +- docker/oap/{Dockerfile => Dockerfile.oap} | 8 +- docker/oap/docker-entrypoint.sh | 226 +++++++++++++- docker/ui/{Dockerfile => Dockerfile.ui} | 6 +- docker/ui/docker-entrypoint.sh | 9 +- docs/en/guides/How-to-build.md | 29 ++ 14 files changed, 372 insertions(+), 625 deletions(-) create mode 100644 Makefile delete mode 100644 docker/Makefile delete mode 100644 docker/README.md delete mode 100644 docker/config/alarm-settings.yml delete mode 100644 docker/config/application.yml delete mode 100644 docker/config/component-libraries.yml delete mode 100644 docker/config/datasource-settings.properties delete mode 100644 docker/config/log4j2.xml rename docker/oap/{Dockerfile => Dockerfile.oap} (89%) rename docker/ui/{Dockerfile => Dockerfile.ui} (90%) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..acb6a854dc --- /dev/null +++ b/Makefile @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +export SW_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + +export SW_OUT:=${SW_ROOT}/dist + +SKIP_TEST?=false + +.PHONY: build.all build.agent build.backend build.ui build.docker + +build.all: + cd $(SW_ROOT) && ./mvnw clean package -Dmaven.test.skip=$(SKIP_TEST) + +build.agent: + cd $(SW_ROOT) && ./mvnw clean package -Dmaven.test.skip=$(SKIP_TEST) -Pagent,dist + +build.backend: + cd $(SW_ROOT) && ./mvnw clean package -Dmaven.test.skip=$(SKIP_TEST) -Pbackend,dist + +build.ui: + cd $(SW_ROOT) && ./mvnw clean package -Dmaven.test.skip=$(SKIP_TEST) -Pui,dist + +build.docker: + cd $(SW_ROOT) && ./mvnw clean package -Dmaven.test.skip=$(SKIP_TEST) -Pbackend,ui,dist + +DOCKER_BUILD_TOP:=${SW_OUT}/docker_build + +HUB?=skywalking + +TAG?=latest + +.SECONDEXPANSION: #allow $@ to be used in dependency list + +.PHONY: docker docker.all docker.oap + +docker: build.docker docker.all + +DOCKER_TARGETS:=docker.oap docker.ui + +docker.all: $(DOCKER_TARGETS) + +docker.oap: $(SW_OUT)/apache-skywalking-apm-bin.tar.gz +docker.oap: $(SW_ROOT)/docker/oap/Dockerfile.oap +docker.oap: $(SW_ROOT)/docker/oap/docker-entrypoint.sh +docker.oap: $(SW_ROOT)/docker/oap/log4j2.xml + $(DOCKER_RULE) + +docker.ui: $(SW_OUT)/apache-skywalking-apm-bin.tar.gz +docker.ui: $(SW_ROOT)/docker/ui/Dockerfile.ui +docker.ui: $(SW_ROOT)/docker/ui/docker-entrypoint.sh +docker.ui: $(SW_ROOT)/docker/ui/logback.xml + $(DOCKER_RULE) + + +# $@ is the name of the target +# $^ the name of the dependencies for the target +# Rule Steps # +############## +# 1. Make a directory $(DOCKER_BUILD_TOP)/%@ +# 2. This rule uses cp to copy all dependency filenames into into $(DOCKER_BUILD_TOP/$@ +# 3. This rule then changes directories to $(DOCKER_BUID_TOP)/$@ +# 4. This rule runs $(BUILD_PRE) prior to any docker build and only if specified as a dependency variable +# 5. This rule finally runs docker build passing $(BUILD_ARGS) to docker if they are specified as a dependency variable + +DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .) + +# for each docker.XXX target create a push.docker.XXX target that pushes +# the local docker image to another hub +# a possible optimization is to use tag.$(TGT) as a dependency to do the tag for us +$(foreach TGT,$(DOCKER_TARGETS),$(eval push.$(TGT): | $(TGT) ; \ + time (docker push $(HUB)/$(subst docker.,,$(TGT)):$(TAG)))) + +# create a DOCKER_PUSH_TARGETS that's each of DOCKER_TARGETS with a push. prefix +DOCKER_PUSH_TARGETS:= +$(foreach TGT,$(DOCKER_TARGETS),$(eval DOCKER_PUSH_TARGETS+=push.$(TGT))) + +# Will build and push docker images. +docker.push: $(DOCKER_PUSH_TARGETS) + + diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index 055a95293f..0000000000 --- a/docker/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export DOCKER_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -oap: - cd $(DOCKER_ROOT)/oap; \ - cp ../../dist/apache-skywalking-apm-bin.tar.gz ./ ; \ - docker build -t skywalking/oap:latest . - -ui: - cd $(DOCKER_ROOT)/ui; \ - cp ../../dist/apache-skywalking-apm-bin.tar.gz ./ ; \ - docker build -t skywalking/ui:latest . - -build: oap ui - -.PHONY: oap ui \ No newline at end of file diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 15d4a2c75b..0000000000 --- a/docker/README.md +++ /dev/null @@ -1,12 +0,0 @@ - -# Build all images - -```shell -make build -``` - -# Run skywalking by docker-compose - -```shell -docker-compose up -``` \ No newline at end of file diff --git a/docker/config/alarm-settings.yml b/docker/config/alarm-settings.yml deleted file mode 100644 index 829246aefa..0000000000 --- a/docker/config/alarm-settings.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sample alarm rules. -rules: - # Rule unique name, must be ended with `_rule`. - service_resp_time_rule: - metrics-name: service_resp_time - op: ">" - threshold: 1000 - period: 10 - count: 3 - silence-period: 5 - message: Response time of service {name} is more than 1000ms in last 3 minutes. - service_sla_rule: - # Metrics value need to be long, double or int - metrics-name: service_sla - op: "<" - threshold: 8000 - # The length of time to evaluate the metric - period: 10 - # How many times after the metric match the condition, will trigger alarm - count: 2 - # How many times of checks, the alarm keeps silence after alarm triggered, default as same as period. - silence-period: 3 - message: Successful rate of service {name} is lower than 80% in last 2 minutes. - service_p90_sla_rule: - # Metrics value need to be long, double or int - metrics-name: service_p90 - op: ">" - threshold: 1000 - period: 10 - count: 3 - silence-period: 5 - message: 90% response time of service {name} is more than 1000ms in last 3 minutes - service_instance_resp_time_rule: - metrics-name: service_instance_resp_time - op: ">" - threshold: 1000 - period: 10 - count: 2 - silence-period: 5 - message: Response time of service instance {name} is more than 1000ms in last 2 minutes. -# Active endpoint related metric alarm will cost more memory than service and service instance metric alarm. -# Because the number of endpoint is much more than service and instance. -# -# endpoint_avg_rule: -# metrics-name: endpoint_avg -# op: ">" -# threshold: 1000 -# period: 10 -# count: 2 -# silence-period: 5 -# message: Response time of endpoint {name} is more than 1000ms in last 2 minutes. - -webhooks: -# - http://127.0.0.1/notify/ -# - http://127.0.0.1/go-wechat/ - diff --git a/docker/config/application.yml b/docker/config/application.yml deleted file mode 100644 index afbd3e7ad6..0000000000 --- a/docker/config/application.yml +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cluster: - standalone: - # Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+ - # library the oap-libs folder with your ZooKeeper 3.4.x library. -# zookeeper: -# nameSpace: ${SW_NAMESPACE:""} -# hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} -# #Retry Policy -# baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries -# maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry -# kubernetes: -# watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} -# namespace: ${SW_CLUSTER_K8S_NAMESPACE:default} -# labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} -# uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} -# consul: -# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} -# Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 -# hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} -# nacos: -# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} -# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} -core: - default: - # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate - # Receiver: Receive agent data, Level 1 aggregate - # Aggregator: Level 2 aggregate - role: ${SW_CORE_ROLE:Mixed} # Mixed/Receiver/Aggregator - restHost: ${SW_CORE_REST_HOST:0.0.0.0} - restPort: ${SW_CORE_REST_PORT:12800} - restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} - gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} - gRPCPort: ${SW_CORE_GRPC_PORT:11800} - downsampling: - - Hour - - Day - - Month - # Set a timeout on metric data. After the timeout has expired, the metric data will automatically be deleted. - recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute - minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute - hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour - dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day - monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month -storage: - elasticsearch: - nameSpace: ${SW_NAMESPACE:""} - clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:elasticsearch:9200} - user: ${SW_ES_USER:""} - password: ${SW_ES_PASSWORD:""} - indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2} - indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} - # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html - bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests - bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb - flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests - concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests - metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} - segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} -# h2: -# driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} -# url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} -# user: ${SW_STORAGE_H2_USER:sa} -# metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} -# mysql: -# metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} -receiver-sharing-server: - default: -receiver-register: - default: -receiver-trace: - default: - bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB - bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB - bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} - sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. - slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. -receiver-jvm: - default: -service-mesh: - default: - bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path - bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB - bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB - bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} -istio-telemetry: - default: -envoy-metric: - default: -# receiver_zipkin: -# default: -# host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} -# port: ${SW_RECEIVER_ZIPKIN_PORT:9411} -# contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} -query: - graphql: - path: ${SW_QUERY_GRAPHQL_PATH:/graphql} -alarm: - default: -telemetry: - none: diff --git a/docker/config/component-libraries.yml b/docker/config/component-libraries.yml deleted file mode 100644 index 1b71218766..0000000000 --- a/docker/config/component-libraries.yml +++ /dev/null @@ -1,311 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define all component libraries' names and IDs, used in monitored application. -# This is a bothway mapping, agent or SDK could use the value(ID) to represent the component name in uplink data. -# -# ###### -# id -# ###### -# We highly recommend DO NOT change the IDs in these file, just append new one, and make sure the ID unique. -# Any replacement will cause visualization and aggregation error. -# -# All IDs in this files are reserved, even some IDs removed by some reasons, those IDs will be abandoned. -# -# ###### -# languages -# ###### -# Languages declare which languages are using this component. Multi languages should be separated by `,` - -Tomcat: - id: 1 - languages: Java -HttpClient: - id: 2 - languages: Java,C#,Node.js -Dubbo: - id: 3 - languages: Java -H2: - id: 4 - languages: Java -Mysql: - id: 5 - languages: Java,C#,Node.js -ORACLE: - id: 6 - languages: Java -Redis: - id: 7 - languages: Java,C#,Node.js -Motan: - id: 8 - languages: Java -MongoDB: - id: 9 - languages: Java,C#,Node.js -Resin: - id: 10 - languages: Java -Feign: - id: 11 - languages: Java -OKHttp: - id: 12 - languages: Java -SpringRestTemplate: - id: 13 - languages: Java -SpringMVC: - id: 14 - languages: Java -Struts2: - id: 15 - languages: Java -NutzMVC: - id: 16 - languages: Java -NutzHttp: - id: 17 - languages: Java -JettyClient: - id: 18 - languages: Java -JettyServer: - id: 19 - languages: Java -Memcached: - id: 20 - languages: Java -ShardingJDBC: - id: 21 - languages: Java -PostgreSQL: - id: 22 - languages: Java,C#,Node.js -GRPC: - id: 23 - languages: Java -ElasticJob: - id: 24 - languages: Java -RocketMQ: - id: 25 - languages: Java -httpasyncclient: - id: 26 - languages: Java -Kafka: - id: 27 - languages: Java -ServiceComb: - id: 28 - languages: Java -Hystrix: - id: 29 - languages: Java -Jedis: - id: 30 - languages: Java -SQLite: - id: 31 - languages: Java,C# -h2-jdbc-driver: - id: 32 - languages: Java -mysql-connector-java: - id: 33 - languages: Java -ojdbc: - id: 34 - languages: Java -Spymemcached: - id: 35 - languages: Java -Xmemcached: - id: 36 - languages: Java -postgresql-jdbc-driver: - id: 37 - languages: Java -rocketMQ-producer: - id: 38 - languages: Java -rocketMQ-consumer: - id: 39 - languages: Java -kafka-producer: - id: 40 - languages: Java -kafka-consumer: - id: 41 - languages: Java -mongodb-driver: - id: 42 - languages: Java -SOFARPC: - id: 43 - languages: Java -ActiveMQ: - id: 44 - languages: Java -activemq-producer: - id: 45 - languages: Java -activemq-consumer: - id: 46 - languages: Java -Elasticsearch: - id: 47 - languages: Java -transport-client: - id: 48 - languages: Java -http: - id: 49 - languages: Java,C#,Node.js -rpc: - id: 50 - languages: Java,C#,Node.js -RabbitMQ: - id: 51 - languages: Java -rabbitmq-producer: - id: 52 - languages: Java -rabbitmq-consumer: - id: 53 - languages: Java -Canal: - id: 54 - languages: Java -Gson: - id: 55 - languages: Java -Redisson: - id: 56 - languages: Java -Lettuce: - id: 57 - languages: Java -Zookeeper: - id: 58 - languages: Java -Vertx: - id: 59 - languages: Java -ShardingSphere: - id: 60 - languages: Java - -# .NET/.NET Core components -# [3000, 4000) for C#/.NET only -AspNetCore: - id: 3001 - languages: C# -EntityFrameworkCore: - id: 3002 - languages: C# -SqlClient: - id: 3003 - languages: C# -CAP: - id: 3004 - languages: C# -StackExchange.Redis: - id: 3005 - languages: C# -SqlServer: - id: 3006 - languages: C# -Npgsql: - id: 3007 - languages: C# -MySqlConnector: - id: 3008 - languages: C# -EntityFrameworkCore.InMemory: - id: 3009 - languages: C# -EntityFrameworkCore.SqlServer: - id: 3010 - languages: C# -EntityFrameworkCore.Sqlite: - id: 3011 - languages: C# -Pomelo.EntityFrameworkCore.MySql: - id: 3012 - languages: C# -Npgsql.EntityFrameworkCore.PostgreSQL: - id: 3013 - languages: C# -InMemoryDatabase: - id: 3014 - languages: C# -AspNet: - id: 3015 - languages: C# -SmartSql: - id: 3016 - languages: C# - -# NoeJS components -# [4000, 5000) for Node.js agent -HttpServer: - id: 4001 - languages: Node.js -express: - id: 4002 - languages: Node.js -Egg: - id: 4003 - languages: Node.js -Koa: - id: 4004 - languages: Node.js - -# Component Server mapping defines the server display names of some components -# e.g. -# Jedis is a client library in Java for Redis server -Component-Server-Mappings: - mongodb-driver: MongoDB - rocketMQ-producer: RocketMQ - rocketMQ-consumer: RocketMQ - kafka-producer: Kafka - kafka-consumer: Kafka - activemq-producer: ActiveMQ - activemq-consumer: ActiveMQ - rabbitmq-producer: RabbitMQ - rabbitmq-consumer: RabbitMQ - postgresql-jdbc-driver: PostgreSQL - Xmemcached: Memcached - Spymemcached: Memcached - h2-jdbc-driver: H2 - mysql-connector-java: Mysql - Jedis: Redis - Redisson: Redis - Lettuce: Redis - Zookeeper: Zookeeper - StackExchange.Redis: Redis - SqlClient: SqlServer - Npgsql: PostgreSQL - MySqlConnector: Mysql - EntityFrameworkCore.InMemory: InMemoryDatabase - EntityFrameworkCore.SqlServer: SqlServer - EntityFrameworkCore.Sqlite: SQLite - Pomelo.EntityFrameworkCore.MySql: Mysql - Npgsql.EntityFrameworkCore.PostgreSQL: PostgreSQL - transport-client: Elasticsearch diff --git a/docker/config/datasource-settings.properties b/docker/config/datasource-settings.properties deleted file mode 100644 index 70b009915e..0000000000 --- a/docker/config/datasource-settings.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -jdbcUrl=jdbc:mysql://localhost:3306/swtest -dataSource.user=root -dataSource.password=root@1234 -dataSource.cachePrepStmts=true -dataSource.prepStmtCacheSize=250 -dataSource.prepStmtCacheSqlLimit=2048 -dataSource.useServerPrepStmts=true -dataSource.useLocalSessionState=true -dataSource.rewriteBatchedStatements=true -dataSource.cacheResultSetMetadata=true -dataSource.cacheServerConfiguration=true -dataSource.elideSetAutoCommits=true -dataSource.maintainTimeStats=false \ No newline at end of file diff --git a/docker/config/log4j2.xml b/docker/config/log4j2.xml deleted file mode 100644 index 346d18ba14..0000000000 --- a/docker/config/log4j2.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 88b40f7374..7b5f8217cd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,7 +17,7 @@ version: '3.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 + image: docker.elastic.co/elasticsearch/elasticsearch:6.4.3 container_name: elasticsearch restart: always ports: @@ -25,6 +25,10 @@ services: - 9300:9300 environment: discovery.type: single-node + ulimits: + memlock: + soft: -1 + hard: -1 oap: image: skywalking/oap container_name: oap @@ -36,8 +40,9 @@ services: ports: - 11800:11800 - 12800:12800 - volumes: - - ./config:/skywalking/config:ro + environment: + SW_STORAGE: elasticsearch + SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200 ui: image: skywalking/ui container_name: ui @@ -49,4 +54,4 @@ services: ports: - 8080:8080 environment: - collectorListOfServers: oap:12800 + SW_OAP_ADDRESS: oap:12800 diff --git a/docker/oap/Dockerfile b/docker/oap/Dockerfile.oap similarity index 89% rename from docker/oap/Dockerfile rename to docker/oap/Dockerfile.oap index 4b3e9f3993..43ec2ddb67 100644 --- a/docker/oap/Dockerfile +++ b/docker/oap/Dockerfile.oap @@ -17,7 +17,9 @@ FROM openjdk:8u181-jdk-stretch ENV DIST_NAME=apache-skywalking-apm-bin \ - JAVA_OPTS=" -Xms256M " + JAVA_OPTS=" -Xms256M " \ + SW_CLUSTER="standalone" \ + SW_STORAGE="h2" COPY "$DIST_NAME.tar.gz" / @@ -33,6 +35,6 @@ WORKDIR skywalking COPY log4j2.xml config/ COPY docker-entrypoint.sh . -EXPOSE 12800 11800 +EXPOSE 12800 11800 1234 -ENTRYPOINT ["sh", "docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "docker-entrypoint.sh"] \ No newline at end of file diff --git a/docker/oap/docker-entrypoint.sh b/docker/oap/docker-entrypoint.sh index 593ad1c7af..764cf01458 100755 --- a/docker/oap/docker-entrypoint.sh +++ b/docker/oap/docker-entrypoint.sh @@ -16,7 +16,227 @@ #!/bin/bash -set -ex +set -e + +var_application_file="config/application.yml" + +generateClusterStandalone() { + echo "cluster:" >> ${var_application_file} + echo " standalone:" >> ${var_application_file} +} + +generateClusterZookeeper() { + cat <> ${var_application_file} +cluster: + zookeeper: + nameSpace: \${SW_NAMESPACE:""} + hostPort: \${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} + #Retry Policy + baseSleepTimeMs: \${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries + maxRetries: \${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry +EOT +} + +generateClusterK8s() { + cat <> ${var_application_file} +cluster: + kubernetes: + watchTimeoutSeconds: \${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} + namespace: \${SW_CLUSTER_K8S_NAMESPACE:default} + labelSelector: \${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} + uidEnvName: \${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} +EOT +} + +generateClusterConsul() { + cat <> ${var_application_file} +cluster: + consul: + serviceName: \${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} + Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 + hostPort: \${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +EOT +} + +generateStorageElastisearch() { + cat <> ${var_application_file} +storage: + elasticsearch: + nameSpace: \${SW_NAMESPACE:""} + clusterNodes: \${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} + user: \${SW_ES_USER:""} + password: \${SW_ES_PASSWORD:""} + indexShardsNumber: \${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2} + indexReplicasNumber: \${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} + # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html + bulkActions: \${SW_STORAGE_ES_BULK_ACTIONS:2000} # Execute the bulk every 2000 requests + bulkSize: \${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb + flushInterval: \${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests + concurrentRequests: \${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests + metadataQueryMaxSize: \${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} + segmentQueryMaxSize: \${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} +EOT +} + +generateStorageH2() { + cat <> ${var_application_file} +storage: + h2: + driver: \${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} + url: \${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} + user: \${SW_STORAGE_H2_USER:sa} + metadataQueryMaxSize: \${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} +EOT +} + +generateStorageMySQL() { + cat <> ${var_application_file} +storage: + mysql: + metadataQueryMaxSize: \${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} +EOT +} + +validateVariables() { + name=$1; value=$2; list=$3 + valid=false + for c in ${list} ; do + if [[ "$c" = "$value" ]]; then + valid=true + fi + done + + if ! ${valid}; then + echo "Error: $name=$value please specify $name = $list" + exit 1 + fi +} + +generateApplicationYaml() { + # validate + [[ -z "$SW_CLUSTER" ]] && [[ -z "$SW_STORAGE" ]] && { echo "Error: please specify \"SW_CLUSTER\" \"SW_STORAGE\""; exit 1; } + + validateVariables "SW_CLUSTER" "$SW_CLUSTER" "standalone zookeeper kubernetes consul" + + validateVariables "SW_STORAGE" "$SW_STORAGE" "elasticsearch h2 mysql" + + echo "# Generated by 'docker-entrypoint.sh'" > ${var_application_file} + #generate cluster + case ${SW_CLUSTER} in + standalone) generateClusterStandalone;; + zookeeper) generateClusterZookeeper;; + kubernetes) generateClusterK8s;; + consul) generateClusterConsul;; + esac + + #generate core + cat <> ${var_application_file} +core: + default: + # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate + # Receiver: Receive agent data, Level 1 aggregate + # Aggregator: Level 2 aggregate + role: \${SW_CORE_ROLE:Mixed} # Mixed/Receiver/Aggregator + restHost: \${SW_CORE_REST_HOST:0.0.0.0} + restPort: \${SW_CORE_REST_PORT:12800} + restContextPath: \${SW_CORE_REST_CONTEXT_PATH:/} + gRPCHost: \${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: \${SW_CORE_GRPC_PORT:11800} + downsampling: + - Hour + - Day + - Month + # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. + recordDataTTL: \${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: \${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: \${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: \${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: \${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month +EOT + + # generate storage + case ${SW_STORAGE} in + elasticsearch) generateStorageElastisearch;; + h2) generateStorageH2;; + mysql) generateStorageMySQL;; + esac + + cat <> ${var_application_file} +receiver-sharing-server: + default: +receiver-register: + default: +receiver-trace: + default: + bufferPath: \${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: \${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: \${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: \${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} + sampleRate: \${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. + slowDBAccessThreshold: \${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. +receiver-jvm: + default: +receiver-clr: + default: +service-mesh: + default: + bufferPath: \${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: \${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: \${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: \${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} +istio-telemetry: + default: +envoy-metric: + default: +query: + graphql: + path: \${SW_QUERY_GRAPHQL_PATH:/graphql} +alarm: + default: +telemetry: + prometheus: + host: \${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0} + port: \${SW_TELEMETRY_PROMETHEUS_PORT:1234} +EOT + + if [[ "$SW_RECEIVER_ZIPKIN_ENABLED" = "true" ]]; then + cat <> ${var_application_file} +receiver_zipkin: + default: + host: \${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} + port: \${SW_RECEIVER_ZIPKIN_PORT:9411} + contextPath: \${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} +EOT + fi + + if [[ "$SW_RECEIVER_JAEGER_ENABLED" = "true" ]]; then + cat <> ${var_application_file} +receiver_jaeger: + default: + gRPCHost: \${SW_RECEIVER_JAEGER_HOST:0.0.0.0} + gRPCPort: \${SW_RECEIVER_JAEGER_PORT:14250} +EOT + fi + + if [[ "$SW_EXPORTER_ENABLED" = "true" ]]; then + cat <> \${var_application_file} +exporter: + grpc: + targetHost: \${SW_EXPORTER_GRPC_HOST:127.0.0.1} + targetPort: \${SW_EXPORTER_GRPC_PORT:9870} +EOT + fi +} + +echo "[Entrypoint] Apache SkyWalking Docker Image" + +if [[ -z "$SW_L0AD_CONFIG_FILE_FROM_VOLUME" ]] || [[ "$SW_L0AD_CONFIG_FILE_FROM_VOLUME" != "true" ]]; then + generateApplicationYaml + echo "Generated application.yml" + echo "-------------------------" + cat ${var_application_file} + echo "-------------------------" +fi CLASSPATH="config:$CLASSPATH" for i in oap-libs/*.jar @@ -24,5 +244,5 @@ do CLASSPATH="$i:$CLASSPATH" done -exec java -XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ - ${JAVA_OPTS} -classpath $CLASSPATH org.apache.skywalking.oap.server.starter.OAPServerStartUp "$@" +exec java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ + ${JAVA_OPTS} -classpath ${CLASSPATH} org.apache.skywalking.oap.server.starter.OAPServerStartUp "$@" diff --git a/docker/ui/Dockerfile b/docker/ui/Dockerfile.ui similarity index 90% rename from docker/ui/Dockerfile rename to docker/ui/Dockerfile.ui index 4efaa5a6a2..a52a148b88 100644 --- a/docker/ui/Dockerfile +++ b/docker/ui/Dockerfile.ui @@ -17,7 +17,9 @@ FROM openjdk:8u181-jdk-stretch ENV DIST_NAME=apache-skywalking-apm-bin \ - JAVA_OPTS=" -Xms256M " + JAVA_OPTS=" -Xms256M " \ + SW_OAP_ADDRESS="127.0.0.1:12800" \ + SW_TIMEOUT="20000" COPY "$DIST_NAME.tar.gz" / @@ -35,4 +37,4 @@ COPY logback.xml webapp/ EXPOSE 8080 -ENTRYPOINT ["sh", "docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "docker-entrypoint.sh"] \ No newline at end of file diff --git a/docker/ui/docker-entrypoint.sh b/docker/ui/docker-entrypoint.sh index 881368f4ad..6771e22f2c 100755 --- a/docker/ui/docker-entrypoint.sh +++ b/docker/ui/docker-entrypoint.sh @@ -16,6 +16,11 @@ #!/bin/bash -set -ex +set -e -exec java -XX:+PrintFlagsFinal -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -jar webapp/skywalking-webapp.jar --logging.config=webapp/logback.xml --collector.ribbon.listOfServers=${collectorListOfServers} "$@" +export LOGGING_CONFIG="webapp/logback.xml" + +[[ ! -z "$SW_OAP_ADDRESS" ]] && export COLLECTOR_RIBBON_LISTOFSERVERS=${SW_OAP_ADDRESS} && echo "COLLECTOR_RIBBON_LISTOFSERVERS=$COLLECTOR_RIBBON_LISTOFSERVERS" +[[ ! -z "$SW_TIMEOUT" ]] && export COLLECTOR_RIBBON_READTIMEOUT=${SW_TIMEOUT} && echo "COLLECTOR_RIBBON_READTIMEOUT=$COLLECTOR_RIBBON_READTIMEOUT" + +exec java -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -jar webapp/skywalking-webapp.jar "$@" diff --git a/docs/en/guides/How-to-build.md b/docs/en/guides/How-to-build.md index 8739de1077..f403b05208 100644 --- a/docs/en/guides/How-to-build.md +++ b/docs/en/guides/How-to-build.md @@ -28,12 +28,41 @@ SkyWalking is a complex maven project, including many modules, which could cause If you just want to recompile part of the project, you have following options - Compile agent and package > ./mvnw package -Pagent,dist + +or + +> make build.agent + - Compile backend and package > ./mvnw package -Pbackend,dist + +or + +> make build.backend + - Compile UI and package > ./mvnw package -Pui,dist +or + +> make build.ui + + +### Build docker images +We can build docker images of `backend` and `ui` with `Makefile` located in root folder. + +- Build all docker images +> make docker.all + +- Build oap server docker image +> make docker.oap + +- Build ui docker image +> make docker.ui +`HUB` and `TAG` variables ares used to setup `REPOSITORY` and `TAG` of a docker image. To get +a oap image with name `bar/oap:foo`, run the following command +> HUB=bar TAG=foo make docker.oap ## Setup your IntelliJ IDEA 1. Import the project as a maven project -- GitLab