support Slack alarm hooks (#5364)

docs/en/setup/backend/backend-alarm.md

@@ -162,6 +162,24 @@ message AlarmMessage {
 }
 ```
 
+## Slack Chat Hook
+To do this you need to follow the [Getting Started with Incoming Webhooks guide](https://api.slack.com/messaging/webhooks) and create new Webhooks.
+
+The alarm message will send through HTTP post by `application/json` content type if you configured Slack Incoming Webhooks as following:
+```yml
+slackHooks:
+  textTemplate: |-
+    {
+      "type": "section",
+      "text": {
+        "type": "mrkdwn",
+        "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
+      }
+    }
+  webhooks:
+    - https://hooks.slack.com/services/x/y/z
+```
+
 ## Update the settings dynamically
 Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
 which will override the settings in `alarm-settings.yml`.

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java

@@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
 import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.alarm.AlarmModule;
 import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
 import org.apache.skywalking.oap.server.library.module.ModuleProvider;
 
 /**
@@ -109,4 +110,8 @@ public class AlarmRulesWatcher extends ConfigChangeWatcher {
     public GRPCAlarmSetting getGrpchookSetting() {
         return this.rules.getGrpchookSetting();
     }
+
+    public SlackSettings getSlackSettings() {
+        return this.rules.getSlacks();
+    }
 }

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java

@@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.alarm.MetricsNotify;
 import org.apache.skywalking.oap.server.core.alarm.ServiceInstanceMetaInAlarm;
 import org.apache.skywalking.oap.server.core.alarm.ServiceMetaInAlarm;
 import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCCallback;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackhookCallback;
 import org.apache.skywalking.oap.server.core.analysis.IDManager;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo;
@@ -158,6 +159,7 @@ public class NotifyHandler implements MetricsNotify {
         List<AlarmCallback> allCallbacks = new ArrayList<>(Arrays.asList(callbacks));
         allCallbacks.add(new WebhookCallback(alarmRulesWatcher));
         allCallbacks.add(new GRPCCallback(alarmRulesWatcher));
+        allCallbacks.add(new SlackhookCallback(alarmRulesWatcher));
         core.start(allCallbacks);
     }
 }

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java

@@ -24,6 +24,7 @@ import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
 import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
 
 @Setter
 @Getter
@@ -32,6 +33,7 @@ public class Rules {
     private List<AlarmRule> rules;
     private List<String> webhooks;
     private GRPCAlarmSetting grpchookSetting;
+    private SlackSettings slacks;
 
     public Rules() {
         this.rules = new ArrayList<>();

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
 import org.yaml.snakeyaml.Yaml;
 
 /**
@@ -65,8 +66,10 @@ public class RulesReader {
                         alarmRule.setExcludeNames((ArrayList) settings.getOrDefault("exclude-names", new ArrayList(0)));
                         alarmRule.setIncludeNamesRegex((String) settings.getOrDefault("include-names-regex", ""));
                         alarmRule.setExcludeNamesRegex((String) settings.getOrDefault("exclude-names-regex", ""));
-                        alarmRule.setIncludeLabels((ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
-                        alarmRule.setExcludeLabels((ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
+                        alarmRule.setIncludeLabels(
+                            (ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
+                        alarmRule.setExcludeLabels(
+                            (ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
                         alarmRule.setIncludeLabelsRegex((String) settings.getOrDefault("include-labels-regex", ""));
                         alarmRule.setExcludeLabelsRegex((String) settings.getOrDefault("exclude-labels-regex", ""));
                         alarmRule.setThreshold(settings.get("threshold").toString());
@@ -74,8 +77,9 @@ public class RulesReader {
                         alarmRule.setPeriod((Integer) settings.getOrDefault("period", 1));
                         alarmRule.setCount((Integer) settings.getOrDefault("count", 1));
                         alarmRule.setSilencePeriod((Integer) settings.getOrDefault("silence-period", -1));
-                        alarmRule.setMessage((String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
-                            .getAlarmRuleName()));
+                        alarmRule.setMessage(
+                            (String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
+                                .getAlarmRuleName()));
 
                         rules.getRules().add(alarmRule);
                     }
@@ -104,6 +108,22 @@ public class RulesReader {
 
                 rules.setGrpchookSetting(grpcAlarmSetting);
             }
+
+            Map slacks = (Map) yamlData.get("slackHooks");
+            if (slacks != null) {
+                SlackSettings slackSettings = new SlackSettings();
+                Object textTemplate = slacks.getOrDefault("textTemplate", "");
+                slackSettings.setTextTemplate((String) textTemplate);
+
+                List<String> slackWebhooks = (List<String>) slacks.get("webhooks");
+                if (slackWebhooks != null) {
+                    slackWebhooks.forEach(
+                        url -> slackSettings.getWebhooks().add(url)
+                    );
+                }
+
+                rules.setSlacks(slackSettings);
+            }
         }
 
         return rules;

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java

@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
 import org.apache.http.StatusLine;
@@ -35,14 +36,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
 import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Use SkyWalking alarm webhook API call a remote endpoints.
  */
+@Slf4j
 public class WebhookCallback implements AlarmCallback {
-    private static final Logger LOGGER = LoggerFactory.getLogger(WebhookCallback.class);
     private static final int HTTP_CONNECT_TIMEOUT = 1000;
     private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
     private static final int HTTP_SOCKET_TIMEOUT = 10000;
@@ -62,7 +61,7 @@ public class WebhookCallback implements AlarmCallback {
 
     @Override
     public void doAlarm(List<AlarmMessage> alarmMessage) {
-        if (alarmRulesWatcher.getWebHooks().size() == 0) {
+        if (alarmRulesWatcher.getWebHooks().isEmpty()) {
             return;
         }
 
@@ -81,19 +80,19 @@ public class WebhookCallback implements AlarmCallback {
                     CloseableHttpResponse httpResponse = httpClient.execute(post);
                     StatusLine statusLine = httpResponse.getStatusLine();
                     if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
-                        LOGGER.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
+                        log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
                     }
                 } catch (UnsupportedEncodingException e) {
-                    LOGGER.error("Alarm to JSON error, " + e.getMessage(), e);
+                    log.error("Alarm to JSON error, " + e.getMessage(), e);
                 } catch (IOException e) {
-                    LOGGER.error("send alarm to " + url + " failure.", e);
+                    log.error("send alarm to " + url + " failure.", e);
                 }
             });
         } finally {
             try {
                 httpClient.close();
             } catch (IOException e) {
-                LOGGER.error(e.getMessage(), e);
+                log.error(e.getMessage(), e);
             }
         }
     }

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.alarm.provider.slack;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Setter
+@Getter
+@ToString
+public class SlackSettings {
+
+    private String textTemplate;
+
+    @Builder.Default
+    private List<String> webhooks = new ArrayList<>();
+}

oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.alarm.provider.slack;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
+import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
+import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher;
+
+/**
+ * Use SkyWalking alarm slack webhook API call a remote endpoints.
+ */
+@Slf4j
+public class SlackhookCallback implements AlarmCallback {
+    private static final int HTTP_CONNECT_TIMEOUT = 1000;
+    private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
+    private static final int HTTP_SOCKET_TIMEOUT = 10000;
+    private static final Gson GSON = new Gson();
+    private AlarmRulesWatcher alarmRulesWatcher;
+    private RequestConfig requestConfig;
+
+    public SlackhookCallback(final AlarmRulesWatcher alarmRulesWatcher) {
+        this.alarmRulesWatcher = alarmRulesWatcher;
+        this.requestConfig = RequestConfig.custom()
+                                          .setConnectTimeout(HTTP_CONNECT_TIMEOUT)
+                                          .setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
+                                          .setSocketTimeout(HTTP_SOCKET_TIMEOUT)
+                                          .build();
+    }
+
+    @Override
+    public void doAlarm(List<AlarmMessage> alarmMessages) {
+        if (this.alarmRulesWatcher.getSlackSettings().getWebhooks().isEmpty()) {
+            return;
+        }
+
+        CloseableHttpClient httpClient = HttpClients.custom().build();
+        try {
+            this.alarmRulesWatcher.getSlackSettings().getWebhooks().forEach(url -> {
+                HttpPost post = new HttpPost(url);
+                post.setConfig(requestConfig);
+                post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString());
+                post.setHeader(HttpHeaders.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON.toString());
+
+                StringEntity entity;
+                try {
+
+                    JsonObject jsonObject = new JsonObject();
+                    JsonArray jsonElements = new JsonArray();
+
+                    alarmMessages.forEach(item -> {
+                        jsonElements.add(GSON.fromJson(
+                            String.format(
+                                this.alarmRulesWatcher.getSlackSettings().getTextTemplate(), item.getAlarmMessage()
+                            ), JsonObject.class));
+                    });
+
+                    jsonObject.add("blocks", jsonElements);
+
+                    entity = new StringEntity(GSON.toJson(jsonObject), ContentType.APPLICATION_JSON);
+
+                    post.setEntity(entity);
+                    CloseableHttpResponse httpResponse = httpClient.execute(post);
+                    StatusLine statusLine = httpResponse.getStatusLine();
+                    if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
+                        log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
+                    }
+                } catch (UnsupportedEncodingException e) {
+                    log.error("Alarm to JSON error, " + e.getMessage(), e);
+                } catch (IOException e) {
+                    log.error("send alarm to " + url + " failure.", e);
+                }
+            });
+        } finally {
+            try {
+                httpClient.close();
+            } catch (IOException e) {
+                log.error(e.getMessage(), e);
+            }
+        }
+    }
+}

oap-server/server-bootstrap/src/main/resources/alarm-settings.yml

@@ -47,3 +47,15 @@ gRPCHook:
 #  target_host: 127.0.0.1
 #  target_port: 9888
 
+slackHooks:
+  textTemplate: |-
+    {
+      "type": "section",
+      "text": {
+        "type": "mrkdwn",
+        "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
+      }
+    }
+  webhooks:
+#    - https://hooks.slack.com/services/x/y/zssss
+