From 3dfa40ebb59da7624d495938ffb2795e11f8b61b Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sat, 22 Aug 2020 17:18:09 +0800 Subject: [PATCH] support Slack alarm hooks (#5364) --- docs/en/setup/backend/backend-alarm.md | 18 +++ .../alarm/provider/AlarmRulesWatcher.java | 5 + .../core/alarm/provider/NotifyHandler.java | 2 + .../oap/server/core/alarm/provider/Rules.java | 2 + .../core/alarm/provider/RulesReader.java | 28 ++++- .../core/alarm/provider/WebhookCallback.java | 15 ++- .../alarm/provider/slack/SlackSettings.java | 42 +++++++ .../provider/slack/SlackhookCallback.java | 115 ++++++++++++++++++ .../src/main/resources/alarm-settings.yml | 12 ++ 9 files changed, 227 insertions(+), 12 deletions(-) create mode 100644 oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java create mode 100644 oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java diff --git a/docs/en/setup/backend/backend-alarm.md b/docs/en/setup/backend/backend-alarm.md index 37317bef3b..90189be37a 100644 --- a/docs/en/setup/backend/backend-alarm.md +++ b/docs/en/setup/backend/backend-alarm.md @@ -162,6 +162,24 @@ message AlarmMessage { } ``` +## Slack Chat Hook +To do this you need to follow the [Getting Started with Incoming Webhooks guide](https://api.slack.com/messaging/webhooks) and create new Webhooks. + +The alarm message will send through HTTP post by `application/json` content type if you configured Slack Incoming Webhooks as following: +```yml +slackHooks: + textTemplate: |- + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**." + } + } + webhooks: + - https://hooks.slack.com/services/x/y/z +``` + ## Update the settings dynamically Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md), which will override the settings in `alarm-settings.yml`. diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java index af4f3ae1ea..51a535387a 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java @@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.alarm.AlarmModule; import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting; +import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings; import org.apache.skywalking.oap.server.library.module.ModuleProvider; /** @@ -109,4 +110,8 @@ public class AlarmRulesWatcher extends ConfigChangeWatcher { public GRPCAlarmSetting getGrpchookSetting() { return this.rules.getGrpchookSetting(); } + + public SlackSettings getSlackSettings() { + return this.rules.getSlacks(); + } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java index bf77c38a76..2209d19d20 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java @@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.alarm.MetricsNotify; import org.apache.skywalking.oap.server.core.alarm.ServiceInstanceMetaInAlarm; import org.apache.skywalking.oap.server.core.alarm.ServiceMetaInAlarm; import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCCallback; +import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackhookCallback; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo; @@ -158,6 +159,7 @@ public class NotifyHandler implements MetricsNotify { List allCallbacks = new ArrayList<>(Arrays.asList(callbacks)); allCallbacks.add(new WebhookCallback(alarmRulesWatcher)); allCallbacks.add(new GRPCCallback(alarmRulesWatcher)); + allCallbacks.add(new SlackhookCallback(alarmRulesWatcher)); core.start(allCallbacks); } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java index 87dd507861..0f95820bc9 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting; +import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings; @Setter @Getter @@ -32,6 +33,7 @@ public class Rules { private List rules; private List webhooks; private GRPCAlarmSetting grpchookSetting; + private SlackSettings slacks; public Rules() { this.rules = new ArrayList<>(); diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java index e1c1adf4c3..9f5f9478c1 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting; +import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings; import org.yaml.snakeyaml.Yaml; /** @@ -65,8 +66,10 @@ public class RulesReader { alarmRule.setExcludeNames((ArrayList) settings.getOrDefault("exclude-names", new ArrayList(0))); alarmRule.setIncludeNamesRegex((String) settings.getOrDefault("include-names-regex", "")); alarmRule.setExcludeNamesRegex((String) settings.getOrDefault("exclude-names-regex", "")); - alarmRule.setIncludeLabels((ArrayList) settings.getOrDefault("include-labels", new ArrayList(0))); - alarmRule.setExcludeLabels((ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0))); + alarmRule.setIncludeLabels( + (ArrayList) settings.getOrDefault("include-labels", new ArrayList(0))); + alarmRule.setExcludeLabels( + (ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0))); alarmRule.setIncludeLabelsRegex((String) settings.getOrDefault("include-labels-regex", "")); alarmRule.setExcludeLabelsRegex((String) settings.getOrDefault("exclude-labels-regex", "")); alarmRule.setThreshold(settings.get("threshold").toString()); @@ -74,8 +77,9 @@ public class RulesReader { alarmRule.setPeriod((Integer) settings.getOrDefault("period", 1)); alarmRule.setCount((Integer) settings.getOrDefault("count", 1)); alarmRule.setSilencePeriod((Integer) settings.getOrDefault("silence-period", -1)); - alarmRule.setMessage((String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule - .getAlarmRuleName())); + alarmRule.setMessage( + (String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule + .getAlarmRuleName())); rules.getRules().add(alarmRule); } @@ -104,6 +108,22 @@ public class RulesReader { rules.setGrpchookSetting(grpcAlarmSetting); } + + Map slacks = (Map) yamlData.get("slackHooks"); + if (slacks != null) { + SlackSettings slackSettings = new SlackSettings(); + Object textTemplate = slacks.getOrDefault("textTemplate", ""); + slackSettings.setTextTemplate((String) textTemplate); + + List slackWebhooks = (List) slacks.get("webhooks"); + if (slackWebhooks != null) { + slackWebhooks.forEach( + url -> slackSettings.getWebhooks().add(url) + ); + } + + rules.setSlacks(slackSettings); + } } return rules; diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java index a9d5333ab0..c997d6a25b 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.List; +import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpHeaders; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; @@ -35,14 +36,12 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.skywalking.oap.server.core.alarm.AlarmCallback; import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Use SkyWalking alarm webhook API call a remote endpoints. */ +@Slf4j public class WebhookCallback implements AlarmCallback { - private static final Logger LOGGER = LoggerFactory.getLogger(WebhookCallback.class); private static final int HTTP_CONNECT_TIMEOUT = 1000; private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000; private static final int HTTP_SOCKET_TIMEOUT = 10000; @@ -62,7 +61,7 @@ public class WebhookCallback implements AlarmCallback { @Override public void doAlarm(List alarmMessage) { - if (alarmRulesWatcher.getWebHooks().size() == 0) { + if (alarmRulesWatcher.getWebHooks().isEmpty()) { return; } @@ -81,19 +80,19 @@ public class WebhookCallback implements AlarmCallback { CloseableHttpResponse httpResponse = httpClient.execute(post); StatusLine statusLine = httpResponse.getStatusLine(); if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) { - LOGGER.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode()); + log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode()); } } catch (UnsupportedEncodingException e) { - LOGGER.error("Alarm to JSON error, " + e.getMessage(), e); + log.error("Alarm to JSON error, " + e.getMessage(), e); } catch (IOException e) { - LOGGER.error("send alarm to " + url + " failure.", e); + log.error("send alarm to " + url + " failure.", e); } }); } finally { try { httpClient.close(); } catch (IOException e) { - LOGGER.error(e.getMessage(), e); + log.error(e.getMessage(), e); } } } diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java new file mode 100644 index 0000000000..9c3df1cbbe --- /dev/null +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm.provider.slack; + +import java.util.ArrayList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Setter +@Getter +@ToString +public class SlackSettings { + + private String textTemplate; + + @Builder.Default + private List webhooks = new ArrayList<>(); +} diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java new file mode 100644 index 0000000000..f7eb93eae7 --- /dev/null +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.alarm.provider.slack; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.netty.handler.codec.http.HttpHeaderValues; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.skywalking.oap.server.core.alarm.AlarmCallback; +import org.apache.skywalking.oap.server.core.alarm.AlarmMessage; +import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher; + +/** + * Use SkyWalking alarm slack webhook API call a remote endpoints. + */ +@Slf4j +public class SlackhookCallback implements AlarmCallback { + private static final int HTTP_CONNECT_TIMEOUT = 1000; + private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000; + private static final int HTTP_SOCKET_TIMEOUT = 10000; + private static final Gson GSON = new Gson(); + private AlarmRulesWatcher alarmRulesWatcher; + private RequestConfig requestConfig; + + public SlackhookCallback(final AlarmRulesWatcher alarmRulesWatcher) { + this.alarmRulesWatcher = alarmRulesWatcher; + this.requestConfig = RequestConfig.custom() + .setConnectTimeout(HTTP_CONNECT_TIMEOUT) + .setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT) + .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .build(); + } + + @Override + public void doAlarm(List alarmMessages) { + if (this.alarmRulesWatcher.getSlackSettings().getWebhooks().isEmpty()) { + return; + } + + CloseableHttpClient httpClient = HttpClients.custom().build(); + try { + this.alarmRulesWatcher.getSlackSettings().getWebhooks().forEach(url -> { + HttpPost post = new HttpPost(url); + post.setConfig(requestConfig); + post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString()); + post.setHeader(HttpHeaders.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON.toString()); + + StringEntity entity; + try { + + JsonObject jsonObject = new JsonObject(); + JsonArray jsonElements = new JsonArray(); + + alarmMessages.forEach(item -> { + jsonElements.add(GSON.fromJson( + String.format( + this.alarmRulesWatcher.getSlackSettings().getTextTemplate(), item.getAlarmMessage() + ), JsonObject.class)); + }); + + jsonObject.add("blocks", jsonElements); + + entity = new StringEntity(GSON.toJson(jsonObject), ContentType.APPLICATION_JSON); + + post.setEntity(entity); + CloseableHttpResponse httpResponse = httpClient.execute(post); + StatusLine statusLine = httpResponse.getStatusLine(); + if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) { + log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode()); + } + } catch (UnsupportedEncodingException e) { + log.error("Alarm to JSON error, " + e.getMessage(), e); + } catch (IOException e) { + log.error("send alarm to " + url + " failure.", e); + } + }); + } finally { + try { + httpClient.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + } +} diff --git a/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml b/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml index c161092e12..771bc2af4e 100755 --- a/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml +++ b/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml @@ -47,3 +47,15 @@ gRPCHook: # target_host: 127.0.0.1 # target_port: 9888 +slackHooks: + textTemplate: |- + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**." + } + } + webhooks: +# - https://hooks.slack.com/services/x/y/zssss + -- GitLab