Support SSL for prometheus telemetry and prometheus fetcher (#5431)

* Support SSL for prometheus telemetry and prometheus fetcher
* Documents
* Clear up abandunt dependencies
* Update license
Signed-off-by: NGao Hongtao <hanahmily@gmail.com>
Co-authored-by: 吴晟 Wu Sheng <wu.sheng@foxmail.com>

dist-material/release-docs/LICENSE

@@ -311,7 +311,7 @@ The text of each license is the standard Apache 2.0 license.
     nacos 1.3.1: https://github.com/alibaba/nacos, Apache 2.0
     consul-client 1.2.6: https://github.com/rickfast/consul-client, Apache 2.0
     okhttp 3.9.0: https://github.com/square/okhttp, Apache 2.0
-    prometheus client_java 0.6.0: https://github.com/prometheus/client_java, Apache 2.0
+    prometheus client_java(simpleclient) 0.6.0: https://github.com/prometheus/client_java, Apache 2.0
     proto files from istio/istio: https://github.com/istio/istio  Apache 2.0
     proto files from istio/api: https://github.com/istio/api      Apache 2.0
     proto files from envoyproxy/data-plane-api: https://github.com/envoyproxy/data-plane-api  Apache 2.0

docs/en/setup/backend/backend-fetcher.md

@@ -42,7 +42,7 @@ metricsPath: <path>
 staticConfig:
   # The targets specified by the static config.
   targets:
-    [ - <host> ]
+    [ - <target> ]
   # Labels assigned to all metrics fetched from the targets.
   labels:
     [ <labelname>: <labelvalue> ... ]
@@ -51,6 +51,15 @@ metricsRules:
    [ - <metric_rules> ]
 ```
 
+#### <target>
+
+```yaml
+# The url of target exporter. the format should be complied with "java.net.URI"
+url: <string>
+# The path of root CA file.
+sslCaFilePath: <string>
+```
+
 #### <metric_rules>
 
 ```yaml

docs/en/setup/backend/backend-telemetry.md

@@ -8,9 +8,12 @@ telemetry:
   prometheus:
     host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
     port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
+    sslEnabled: ${SW_TELEMETRY_PROMETHEUS_SSL_ENABLED:false}
+    sslKeyPath: ${SW_TELEMETRY_PROMETHEUS_SSL_KEY_PATH:""}
+    sslCertChainPath: ${SW_TELEMETRY_PROMETHEUS_SSL_CERT_CHAIN_PATH:""}
 ```
 
-but you can set one of `prometheus` or `so11y` to enable them, for more information, refer to the details below.
+but you can set one of `prometheus` to enable them, for more information, refer to the details below.
 
 ## Prometheus
 Prometheus is supported as telemetry implementor. 
@@ -32,6 +35,19 @@ telemetry:
     port: 1543
 ```
 
+Set SSL relevant settings to expose a secure endpoint. Notice private key file and cert chain file could be uploaded once
+changes are applied to them.
+```yaml
+telemetry:
+  selector: ${SW_TELEMETRY:prometheus}
+  prometheus:
+    host: 127.0.0.1
+    port: 1543
+    sslEnabled: true
+    sslKeyPath: /etc/ssl/key.pem
+    sslCertChainPath: /etc/ssl/cert-chain.pem
+```
+
 ### Grafana Visualization
 Provide the grafana dashboard settings. Check [SkyWalking Telemetry dashboard](grafana.json) config.
 

oap-server/server-bootstrap/src/main/resources/application.yml

@@ -299,6 +299,9 @@ telemetry:
   prometheus:
     host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
     port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
+    sslEnabled: ${SW_TELEMETRY_PROMETHEUS_SSL_ENABLED:false}
+    sslKeyPath: ${SW_TELEMETRY_PROMETHEUS_SSL_KEY_PATH:""}
+    sslCertChainPath: ${SW_TELEMETRY_PROMETHEUS_SSL_CERT_CHAIN_PATH:""}
 
 configuration:
   selector: ${SW_CONFIGURATION:none}

oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml

@@ -34,7 +34,8 @@ metricsPath: /metrics
 staticConfig:
   # targets will be labeled as "instance"
   targets:
-    - localhost:1234
+    - url: http://localhost:1234
+      sslCaFilePath:
   labels:
     service: oap-server
 metricsRules:

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/metric/promethues/rule/StaticConfig.java

@@ -26,6 +26,6 @@ import lombok.NoArgsConstructor;
 @Data
 @NoArgsConstructor
 public class StaticConfig {
-    private List<String> targets;
+    private List<Target> targets;
     private Map<String, String> labels;
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/metric/promethues/rule/Target.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.metric.promethues.rule;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class Target {
+    private String url;
+    private String sslCaFilePath;
+}

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/pom.xml

@@ -37,9 +37,5 @@
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
         </dependency>
-         <dependency>
-            <groupId>com.squareup.okhttp3</groupId>
-            <artifactId>okhttp</artifactId>
-         </dependency>
     </dependencies>
 </project>
\ No newline at end of file

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/http/HttpClient.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.fetcher.prometheus.http;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpVersion;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Objects;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+import lombok.Builder;
+import org.apache.skywalking.oap.server.library.server.ssl.HttpDynamicSslContext;
+
+@Builder
+public class HttpClient {
+
+    private final String url;
+
+    private final String caFilePath;
+
+    public String request() throws URISyntaxException, InterruptedException {
+        URI uri = new URI(url);
+        String scheme = uri.getScheme() == null ? "http" : uri.getScheme();
+        String host = uri.getHost() == null ? "127.0.0.1" : uri.getHost();
+        int port = uri.getPort();
+
+        // Configure SSL context if necessary.
+        final boolean ssl = "https".equalsIgnoreCase(scheme);
+        final HttpDynamicSslContext sslCtx = ssl ? HttpDynamicSslContext.forClient(caFilePath) : null;
+
+        // Configure the client.
+        EventLoopGroup group = new NioEventLoopGroup();
+        BlockingQueue<String> channel = new SynchronousQueue<>();
+        try {
+            Bootstrap b = new Bootstrap();
+            b.group(group)
+                .channel(NioSocketChannel.class)
+                .handler(new HttpClientInitializer(sslCtx, channel));
+
+            // Make the connection attempt.
+            Channel ch = b.connect(host, port).sync().channel();
+
+            // Prepare the HTTP request.
+            HttpRequest request = new DefaultFullHttpRequest(
+                HttpVersion.HTTP_1_1, HttpMethod.GET, uri.getRawPath(), Unpooled.EMPTY_BUFFER);
+            request.headers().set(HttpHeaderNames.HOST, host);
+            request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
+            request.headers().set(HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.TEXT_PLAIN);
+
+            // Send the HTTP request.
+            ch.writeAndFlush(request);
+
+            return Objects.requireNonNull(channel.poll(10, TimeUnit.SECONDS), "Request timeout");
+        } finally {
+            // Shut down executor threads to exit.
+            group.shutdownGracefully();
+        }
+    }
+
+}

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/http/HttpClientHandler.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.fetcher.prometheus.http;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.util.CharsetUtil;
+import java.util.concurrent.BlockingQueue;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@RequiredArgsConstructor
+@Slf4j
+public class HttpClientHandler extends SimpleChannelInboundHandler<HttpObject> {
+
+    private final BlockingQueue<String> channel;
+
+    private final StringBuilder buf = new StringBuilder();
+
+    @Override
+    public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) {
+        if (msg instanceof HttpResponse) {
+            buf.setLength(0);
+        }
+        if (msg instanceof HttpContent) {
+            HttpContent content = (HttpContent) msg;
+            buf.append(content.content().toString(CharsetUtil.UTF_8));
+            if (content instanceof LastHttpContent) {
+                try {
+                    channel.put(buf.toString());
+                } catch (InterruptedException e) {
+                    ctx.fireExceptionCaught(e);
+                }
+                ctx.close();
+            }
+        }
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+        log.error("HTTP request error", cause);
+        ctx.close();
+    }
+}

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/http/HttpClientInitializer.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.fetcher.prometheus.http;
+
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http.HttpClientCodec;
+import io.netty.handler.codec.http.HttpContentDecompressor;
+import io.netty.handler.ssl.SslContext;
+import java.util.concurrent.BlockingQueue;
+
+public class HttpClientInitializer extends ChannelInitializer<SocketChannel> {
+
+    private final SslContext sslCtx;
+    private final BlockingQueue<String> channel;
+
+    public HttpClientInitializer(SslContext sslCtx, BlockingQueue<String> channel) {
+        this.sslCtx = sslCtx;
+        this.channel = channel;
+    }
+
+    @Override
+    public void initChannel(SocketChannel ch) {
+        ChannelPipeline p = ch.pipeline();
+
+        // Enable HTTPS if necessary.
+        if (sslCtx != null) {
+            p.addLast(sslCtx.newHandler(ch.alloc()));
+        }
+
+        p.addLast(new HttpClientCodec());
+
+        // Remove the following line if you don't want automatic content decompression.
+        p.addLast(new HttpContentDecompressor());
+
+        p.addLast(new HttpClientHandler(channel));
+    }
+}

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/provider/PrometheusFetcherProvider.java

@@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.fetcher.prometheus.provider;
 
 import com.google.common.collect.Maps;
 import io.vavr.CheckedFunction1;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
 import java.time.Duration;
 import java.util.Collection;
 import java.util.LinkedList;
@@ -29,15 +31,14 @@ import java.util.Objects;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.Response;
+import org.apache.commons.codec.Charsets;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem;
 import org.apache.skywalking.oap.server.core.metric.promethues.PrometheusMetricConverter;
 import org.apache.skywalking.oap.server.core.metric.promethues.rule.Rule;
 import org.apache.skywalking.oap.server.core.metric.promethues.rule.Rules;
 import org.apache.skywalking.oap.server.core.metric.promethues.rule.StaticConfig;
+import org.apache.skywalking.oap.server.fetcher.prometheus.http.HttpClient;
 import org.apache.skywalking.oap.server.fetcher.prometheus.module.PrometheusFetcherModule;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleDefine;
@@ -51,7 +52,6 @@ import org.apache.skywalking.oap.server.library.util.prometheus.metrics.MetricFa
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.util.Objects.requireNonNull;
 import static java.util.stream.Collectors.toList;
 
 public class PrometheusFetcherProvider extends ModuleProvider {
@@ -59,8 +59,6 @@ public class PrometheusFetcherProvider extends ModuleProvider {
 
     private final PrometheusFetcherConfig config;
 
-    private final OkHttpClient client = new OkHttpClient();
-
     private List<Rule> rules;
 
     private ScheduledExecutorService ses;
@@ -115,13 +113,11 @@ public class PrometheusFetcherProvider extends ModuleProvider {
                     StaticConfig sc = r.getStaticConfig();
                     long now = System.currentTimeMillis();
                     converter.toMeter(sc.getTargets().stream()
-                        .map(CheckedFunction1.liftTry(url -> {
-                            Request request = new Request.Builder()
-                                .url(String.format("http://%s%s", url, r.getMetricsPath().startsWith("/") ? r.getMetricsPath() : "/" + r.getMetricsPath()))
-                                .build();
+                        .map(CheckedFunction1.liftTry(target -> {
                             List<Metric> result = new LinkedList<>();
-                            try (Response response = client.newCall(request).execute()) {
-                                Parser p = Parsers.text(requireNonNull(response.body()).byteStream());
+                            String content = HttpClient.builder().url(target.getUrl()).caFilePath(target.getSslCaFilePath()).build().request();
+                            try (InputStream targetStream = new ByteArrayInputStream(content.getBytes(Charsets.UTF_8))) {
+                                Parser p = Parsers.text(targetStream);
                                 MetricFamily mf;
 
                                 while ((mf = p.parse(now)) != null) {
@@ -131,7 +127,7 @@ public class PrometheusFetcherProvider extends ModuleProvider {
                                                 return;
                                             }
                                             Map<String, String> extraLabels = Maps.newHashMap(sc.getLabels());
-                                            extraLabels.put("instance", url);
+                                            extraLabels.put("instance", target.getUrl());
                                             extraLabels.forEach((key, value) -> {
                                                 if (metric.getLabels().containsKey(key)) {
                                                     metric.getLabels().put("exported_" + key, metric.getLabels().get(key));

oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/test/resources/fetcher-prom-rules/localhost.yaml

@@ -19,7 +19,7 @@ metricsPath: /metrics
 staticConfig:
   # targets will be labeled as "instance"
   targets:
-    - localhost:1234
+    - url: http://localhost:1234
   labels:
     app: test-oap
 metricsRules:

oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/grpc/ssl/DynamicSslContext.java

@@ -19,27 +19,20 @@
 package org.apache.skywalking.oap.server.library.server.grpc.ssl;
 
 import io.grpc.netty.GrpcSslContexts;
-import io.netty.buffer.ByteBufAllocator;
-import io.netty.handler.ssl.ApplicationProtocolNegotiator;
-import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.SslProvider;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.security.GeneralSecurityException;
-import java.util.List;
-import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLSessionContext;
-import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor;
+import org.apache.skywalking.oap.server.library.server.ssl.AbstractSslContext;
+import org.apache.skywalking.oap.server.library.server.ssl.PrivateKeyUtil;
 
 /**
  * Load SslContext dynamically.
  */
-public class DynamicSslContext extends SslContext {
-    private final MultipleFilesChangeMonitor monitor;
-    private volatile SslContext ctx;
+public class DynamicSslContext extends AbstractSslContext {
 
     public static DynamicSslContext forServer(final String privateKeyFile, final String certChainFile) {
         return new DynamicSslContext(privateKeyFile, certChainFile);
@@ -49,86 +42,33 @@ public class DynamicSslContext extends SslContext {
         return new DynamicSslContext(caFile);
     }
 
-    private DynamicSslContext(final String privateKeyFile, final String certChainFile) {
-        updateContext(privateKeyFile, certChainFile);
-        monitor = new MultipleFilesChangeMonitor(
-            10,
-            readableContents -> updateContext(privateKeyFile, certChainFile),
-            certChainFile,
-            privateKeyFile);
+    protected DynamicSslContext(String privateKeyFile, String certChainFile) {
+        super(privateKeyFile, certChainFile);
     }
 
-    private DynamicSslContext(final String caFile) {
-        updateContext(caFile);
-        monitor = new MultipleFilesChangeMonitor(
-            10,
-            readableContents -> updateContext(caFile),
-            caFile);
+    protected DynamicSslContext(String caFile) {
+        super(caFile);
     }
 
-    private void updateContext(String caFile) {
+    protected void updateContext(String caFile) {
         try {
-            ctx = GrpcSslContexts.forClient().trustManager(Paths.get(caFile).toFile()).build();
+            setCtx(GrpcSslContexts.forClient().trustManager(Paths.get(caFile).toFile()).build());
         } catch (SSLException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
-    private void updateContext(final String privateKeyFile, final String certChainFile) {
+    protected void updateContext(final String privateKeyFile, final String certChainFile) {
         try {
-            ctx = GrpcSslContexts
+            setCtx(GrpcSslContexts
                 .configure(SslContextBuilder
                     .forServer(
                         new FileInputStream(Paths.get(certChainFile).toFile()),
                         PrivateKeyUtil.loadDecryptionKey(privateKeyFile)),
                     SslProvider.OPENSSL)
-                .build();
+                .build());
         } catch (GeneralSecurityException | IOException e) {
             throw new IllegalArgumentException(e);
         }
     }
-
-    public void start() {
-       monitor.start();
-    }
-
-    @Override
-    public final boolean isClient() {
-        return ctx.isClient();
-    }
-
-    @Override
-    public final List<String> cipherSuites() {
-        return ctx.cipherSuites();
-    }
-
-    @Override
-    public final long sessionCacheSize() {
-        return ctx.sessionCacheSize();
-    }
-
-    @Override
-    public final long sessionTimeout() {
-        return ctx.sessionTimeout();
-    }
-
-    @Override
-    public final ApplicationProtocolNegotiator applicationProtocolNegotiator() {
-        return ctx.applicationProtocolNegotiator();
-    }
-
-    @Override
-    public final SSLEngine newEngine(ByteBufAllocator alloc) {
-        return ctx.newEngine(alloc);
-    }
-
-    @Override
-    public final SSLEngine newEngine(ByteBufAllocator alloc, String peerHost, int peerPort) {
-        return ctx.newEngine(alloc, peerHost, peerPort);
-    }
-
-    @Override
-    public final SSLSessionContext sessionContext() {
-        return ctx.sessionContext();
-    }
 }

oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/ssl/AbstractSslContext.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.server.ssl;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.handler.ssl.ApplicationProtocolNegotiator;
+import io.netty.handler.ssl.SslContext;
+import java.util.List;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLSessionContext;
+import lombok.AccessLevel;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.library.util.MultipleFilesChangeMonitor;
+
+public abstract class AbstractSslContext extends SslContext {
+    private final MultipleFilesChangeMonitor monitor;
+
+    @Setter(AccessLevel.PROTECTED)
+    private volatile SslContext ctx;
+
+    protected AbstractSslContext(final String privateKeyFile, final String certChainFile) {
+        updateContext(privateKeyFile, certChainFile);
+        monitor = new MultipleFilesChangeMonitor(
+            10,
+            readableContents -> updateContext(privateKeyFile, certChainFile),
+            certChainFile,
+            privateKeyFile);
+    }
+
+    protected AbstractSslContext(final String caFile) {
+        updateContext(caFile);
+        monitor = new MultipleFilesChangeMonitor(
+            10,
+            readableContents -> updateContext(caFile),
+            caFile);
+    }
+
+    protected abstract void updateContext(String caFile);
+
+    protected abstract void updateContext(final String privateKeyFile, final String certChainFile);
+
+    public void start() {
+        monitor.start();
+    }
+
+    @Override
+    public final boolean isClient() {
+        return ctx.isClient();
+    }
+
+    @Override
+    public final List<String> cipherSuites() {
+        return ctx.cipherSuites();
+    }
+
+    @Override
+    public final long sessionCacheSize() {
+        return ctx.sessionCacheSize();
+    }
+
+    @Override
+    public final long sessionTimeout() {
+        return ctx.sessionTimeout();
+    }
+
+    @Override
+    public final ApplicationProtocolNegotiator applicationProtocolNegotiator() {
+        return ctx.applicationProtocolNegotiator();
+    }
+
+    @Override
+    public final SSLEngine newEngine(ByteBufAllocator alloc) {
+        return ctx.newEngine(alloc);
+    }
+
+    @Override
+    public final SSLEngine newEngine(ByteBufAllocator alloc, String peerHost, int peerPort) {
+        return ctx.newEngine(alloc, peerHost, peerPort);
+    }
+
+    @Override
+    public final SSLSessionContext sessionContext() {
+        return ctx.sessionContext();
+    }
+}

oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/ssl/HttpDynamicSslContext.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.library.server.ssl;
+
+import io.netty.handler.ssl.SslContextBuilder;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.GeneralSecurityException;
+import javax.net.ssl.SSLException;
+
+public class HttpDynamicSslContext extends AbstractSslContext {
+
+    public static HttpDynamicSslContext forServer(String privateKeyFile, String certChainFile) {
+        return new HttpDynamicSslContext(privateKeyFile, certChainFile);
+    }
+
+    public static HttpDynamicSslContext forClient(String caFile) {
+        return new HttpDynamicSslContext(caFile);
+    }
+
+    protected HttpDynamicSslContext(String privateKeyFile, String certChainFile) {
+        super(privateKeyFile, certChainFile);
+    }
+
+    protected HttpDynamicSslContext(String caFile) {
+        super(caFile);
+    }
+
+    protected void updateContext(String caFile) {
+        try {
+            setCtx(SslContextBuilder.forClient().trustManager(Paths.get(caFile).toFile()).build());
+        } catch (SSLException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    protected void updateContext(final String privateKeyFile, final String certChainFile) {
+        try {
+            setCtx(SslContextBuilder
+                .forServer(
+                    new FileInputStream(Paths.get(certChainFile).toFile()),
+                    PrivateKeyUtil.loadDecryptionKey(privateKeyFile)).build());
+        } catch (GeneralSecurityException | IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+}

oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/grpc/ssl/PrivateKeyUtil.java → oap-server/server-library/library-server/src/main/java/org/apache/skywalking/oap/server/library/server/ssl/PrivateKeyUtil.java

@@ -16,7 +16,7 @@
  *
  */
 
-package org.apache.skywalking.oap.server.library.server.grpc.ssl;
+package org.apache.skywalking.oap.server.library.server.ssl;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -30,7 +30,7 @@ import java.util.Base64;
 /**
  * Util intends to parse PKCS#1 and PKCS#8 at same time.
  */
-class PrivateKeyUtil {
+public class PrivateKeyUtil {
     private static final String PKCS_1_PEM_HEADER = "-----BEGIN RSA PRIVATE KEY-----";
     private static final String PKCS_1_PEM_FOOTER = "-----END RSA PRIVATE KEY-----";
     private static final String PKCS_8_PEM_HEADER = "-----BEGIN PRIVATE KEY-----";
@@ -39,7 +39,7 @@ class PrivateKeyUtil {
     /**
      * Load a RSA decryption key from a file (PEM or DER).
      */
-    static InputStream loadDecryptionKey(String keyFilePath) throws GeneralSecurityException, IOException {
+    public static InputStream loadDecryptionKey(String keyFilePath) throws GeneralSecurityException, IOException {
         byte[] keyDataBytes = Files.readAllBytes(Paths.get(keyFilePath));
         String keyDataString = new String(keyDataBytes, StandardCharsets.UTF_8);
 

oap-server/server-telemetry/telemetry-prometheus/pom.xml

@@ -29,6 +29,11 @@
     <packaging>jar</packaging>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>library-server</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.skywalking</groupId>
             <artifactId>telemetry-api</artifactId>
@@ -40,16 +45,15 @@
             <artifactId>simpleclient</artifactId>
             <version>0.6.0</version>
         </dependency>
-        <!-- Hotspot JVM metrics-->
         <dependency>
             <groupId>io.prometheus</groupId>
-            <artifactId>simpleclient_hotspot</artifactId>
+            <artifactId>simpleclient_common</artifactId>
             <version>0.6.0</version>
         </dependency>
-        <!-- Exposition HTTPServer-->
+        <!-- Hotspot JVM metrics-->
         <dependency>
             <groupId>io.prometheus</groupId>
-            <artifactId>simpleclient_httpserver</artifactId>
+            <artifactId>simpleclient_hotspot</artifactId>
             <version>0.6.0</version>
         </dependency>
     </dependencies>

oap-server/server-telemetry/telemetry-prometheus/src/main/java/org/apache/skywalking/oap/server/telemetry/prometheus/PrometheusConfig.java

@@ -30,4 +30,7 @@ import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 public class PrometheusConfig extends ModuleConfig {
     private String host = "0.0.0.0";
     private int port = 1234;
+    private boolean sslEnabled = false;
+    private String sslKeyPath;
+    private String sslCertChainPath;
 }

oap-server/server-telemetry/telemetry-prometheus/src/main/java/org/apache/skywalking/oap/server/telemetry/prometheus/PrometheusTelemetryProvider.java

@@ -18,9 +18,7 @@
 
 package org.apache.skywalking.oap.server.telemetry.prometheus;
 
-import io.prometheus.client.exporter.HTTPServer;
 import io.prometheus.client.hotspot.DefaultExports;
-import java.io.IOException;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleDefine;
 import org.apache.skywalking.oap.server.library.module.ModuleProvider;
@@ -29,6 +27,7 @@ import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedExcepti
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
 import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
 import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+import org.apache.skywalking.oap.server.telemetry.prometheus.httpserver.HttpServer;
 
 /**
  * Start the Prometheus
@@ -60,8 +59,8 @@ public class PrometheusTelemetryProvider extends ModuleProvider {
         this.registerServiceImplementation(MetricsCreator.class, new PrometheusMetricsCreator());
         this.registerServiceImplementation(MetricsCollector.class, new PrometheusMetricsCollector());
         try {
-            new HTTPServer(config.getHost(), config.getPort());
-        } catch (IOException e) {
+            new HttpServer(config).start();
+        } catch (InterruptedException e) {
             throw new ModuleStartException(e.getMessage(), e);
         }
 

oap-server/server-telemetry/telemetry-prometheus/src/main/java/org/apache/skywalking/oap/server/telemetry/prometheus/httpserver/HttpServer.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.prometheus.httpserver;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import java.util.Optional;
+import java.util.concurrent.ThreadFactory;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.library.server.ssl.HttpDynamicSslContext;
+import org.apache.skywalking.oap.server.telemetry.prometheus.PrometheusConfig;
+
+/**
+ * An HTTP server that sends back the content of the received HTTP request
+ * in a pretty plaintext form.
+ */
+@RequiredArgsConstructor
+@Slf4j
+public final class HttpServer {
+
+    private final PrometheusConfig config;
+
+    public void start() throws InterruptedException {
+        // Configure SSL.
+        final HttpDynamicSslContext sslCtx;
+        if (config.isSslEnabled()) {
+            sslCtx = HttpDynamicSslContext.forServer(config.getSslKeyPath(), config.getSslCertChainPath());
+        } else {
+            sslCtx = null;
+        }
+
+        // Configure the server.
+        ThreadFactory tf = new ThreadFactoryBuilder().setDaemon(true).build();
+        EventLoopGroup bossGroup = new NioEventLoopGroup(1, tf);
+        EventLoopGroup workerGroup = new NioEventLoopGroup(0, tf);
+        ServerBootstrap b = new ServerBootstrap();
+        b.group(bossGroup, workerGroup)
+            .channel(NioServerSocketChannel.class)
+            .handler(new LoggingHandler(LogLevel.INFO))
+            .childHandler(new HttpServerInitializer(sslCtx));
+
+        b.bind(config.getHost(), config.getPort()).sync();
+        Optional.ofNullable(sslCtx).ifPresent(HttpDynamicSslContext::start);
+
+        log.info("Prometheus exporter endpoint:" +
+            (config.isSslEnabled() ? "https" : "http") + "://" + config.getHost() + ":" + config.getPort() + '/');
+    }
+}

oap-server/server-telemetry/telemetry-prometheus/src/main/java/org/apache/skywalking/oap/server/telemetry/prometheus/httpserver/HttpServerHandler.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.prometheus.httpserver;
+
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpUtil;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.util.CharsetUtil;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.exporter.common.TextFormat;
+import java.io.IOException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.logging.log4j.core.util.StringBuilderWriter;
+
+import static io.netty.channel.ChannelFutureListener.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE;
+import static io.netty.handler.codec.http.HttpHeaderValues.TEXT_PLAIN;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.OK;
+
+@Slf4j
+public class HttpServerHandler  extends SimpleChannelInboundHandler<HttpObject> {
+
+    private final CollectorRegistry registry = CollectorRegistry.defaultRegistry;
+    private final StringBuilderWriter buf = new StringBuilderWriter();
+
+    @Override
+    public void channelReadComplete(ChannelHandlerContext ctx) {
+        ctx.flush();
+    }
+
+    @Override
+    public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) {
+        if (msg instanceof HttpRequest) {
+            HttpRequest req = (HttpRequest) msg;
+
+            boolean keepAlive = HttpUtil.isKeepAlive(req);
+
+            buf.getBuilder().setLength(0);
+            try {
+                TextFormat.write004(buf, registry.metricFamilySamples());
+            } catch (IOException e) {
+                ctx.fireExceptionCaught(e);
+                return;
+            }
+            FullHttpResponse response = new DefaultFullHttpResponse(req.protocolVersion(), OK,
+                Unpooled.copiedBuffer(buf.getBuilder().toString(), CharsetUtil.UTF_8));
+            response.headers()
+                .set(CONTENT_TYPE, TEXT_PLAIN)
+                .setInt(CONTENT_LENGTH, response.content().readableBytes());
+
+            if (keepAlive) {
+                if (!req.protocolVersion().isKeepAliveDefault()) {
+                    response.headers().set(CONNECTION, KEEP_ALIVE);
+                }
+            } else {
+                // Tell the client we're going to close the connection.
+                response.headers().set(CONNECTION, CLOSE);
+            }
+
+            ChannelFuture f = ctx.write(response);
+
+            if (!keepAlive) {
+                f.addListener(CLOSE);
+            }
+        }
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+        log.error("Prometheus exporter error", cause);
+        FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
+            INTERNAL_SERVER_ERROR,
+            Unpooled.wrappedBuffer(cause.getMessage().getBytes()));
+        ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
+        ctx.close();
+    }
+}
\ No newline at end of file

oap-server/server-telemetry/telemetry-prometheus/src/main/java/org/apache/skywalking/oap/server/telemetry/prometheus/httpserver/HttpServerInitializer.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.telemetry.prometheus.httpserver;
+
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http.HttpServerCodec;
+import io.netty.handler.codec.http.HttpServerExpectContinueHandler;
+import io.netty.handler.ssl.SslContext;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
+
+    private final SslContext sslCtx;
+
+    @Override
+    public void initChannel(SocketChannel ch) {
+        ChannelPipeline p = ch.pipeline();
+        if (sslCtx != null) {
+            p.addLast(sslCtx.newHandler(ch.alloc()));
+        }
+        p.addLast(new HttpServerCodec());
+        p.addLast(new HttpServerExpectContinueHandler());
+        p.addLast(new HttpServerHandler());
+    }
+}

tools/dependencies/known-oap-backend-dependencies-es7.txt

@@ -157,7 +157,6 @@ retrofit-2.3.0.jar
 simpleclient-0.6.0.jar
 simpleclient_common-0.6.0.jar
 simpleclient_hotspot-0.6.0.jar
-simpleclient_httpserver-0.6.0.jar
 slf4j-api-1.7.25.jar
 snakeyaml-1.18.jar
 sundr-codegen-0.21.0.jar

tools/dependencies/known-oap-backend-dependencies.txt

@@ -155,7 +155,6 @@ retrofit-2.3.0.jar
 simpleclient-0.6.0.jar
 simpleclient_common-0.6.0.jar
 simpleclient_hotspot-0.6.0.jar
-simpleclient_httpserver-0.6.0.jar
 slf4j-api-1.7.25.jar
 snakeyaml-1.18.jar
 sundr-codegen-0.21.0.jar