提交 e9b9c9df 编写于 作者: Y youyong205

fix the metric alert problem

......@@ -2,6 +2,7 @@ package com.dianping.cat.consumer.metric;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
......@@ -119,7 +120,7 @@ public class MetricConfigManager implements Initializable {
return getMetricConfig().findMetricItemConfig(id);
}
public List<MetricItemConfig> queryMetricItemConfigs(Set<String> domains) {
public List<MetricItemConfig> queryMetricItemConfigs(Collection<String> domains) {
List<MetricItemConfig> configs = new ArrayList<MetricItemConfig>();
Map<String, MetricItemConfig> metricConfig = getMetricConfig().getMetricItemConfigs();
......
......@@ -52,8 +52,8 @@ public class CatHomeModule extends AbstractModule {
SwitchAlert swtichAlert = ctx.lookup(SwitchAlert.class);
ExceptionAlert exceptionAlert = ctx.lookup(ExceptionAlert.class);
Threads.forGroup("Cat").start(metricAlert);
Threads.forGroup("Cat").start(swtichAlert);
Threads.forGroup("Cat").start(metricAlert);
Threads.forGroup("Cat").start(exceptionAlert);
}
executeAlarmModule(ctx);
......
......@@ -52,6 +52,7 @@ import com.dianping.cat.report.task.metric.AlertConfig;
import com.dianping.cat.report.task.metric.AlertInfo;
import com.dianping.cat.report.task.metric.MetricAlert;
import com.dianping.cat.report.task.metric.RemoteMetricReportService;
import com.dianping.cat.report.task.metric.SwitchAlert;
import com.dianping.cat.report.view.DomainNavManager;
import com.dianping.cat.system.config.BugConfigManager;
import com.dianping.cat.system.config.ConfigReloadTask;
......@@ -59,6 +60,7 @@ import com.dianping.cat.system.config.DomainGroupConfigManager;
import com.dianping.cat.system.config.ExceptionThresholdConfigManager;
import com.dianping.cat.system.config.MetricAggregationConfigManager;
import com.dianping.cat.system.config.MetricGroupConfigManager;
import com.dianping.cat.system.config.MetricRuleConfigManager;
import com.dianping.cat.system.config.UtilizationConfigManager;
import com.dianping.cat.system.tool.DefaultMailImpl;
import com.dianping.cat.system.tool.MailSMS;
......@@ -156,6 +158,10 @@ public class ComponentsConfigurator extends AbstractResourceConfigurator {
BaselineService.class, MailSMS.class, AlertConfig.class, AlertInfo.class)//
.req(RemoteMetricReportService.class));
all.add(C(SwitchAlert.class).req(MetricConfigManager.class, ProductLineConfigManager.class,
BaselineService.class, MailSMS.class, AlertConfig.class, AlertInfo.class)//
.req(RemoteMetricReportService.class, MetricRuleConfigManager.class));
all.add(C(ExceptionAlert.class).req(ProjectDao.class, AlertConfig.class, MailSMS.class,
ExceptionThresholdConfigManager.class).req(ModelService.class, TopAnalyzer.ID));
......
......@@ -2,7 +2,6 @@ package com.dianping.cat.report.chart;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
......@@ -131,7 +130,7 @@ public class AggregationGraphCreator extends GraphCreatorBase {
protected boolean isProductLineInGroup(String productLine, List<MetricAggregation> metricAggregations) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine);
List<MetricItemConfig> metricConfigs = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
List<MetricItemConfig> metricConfigs = m_metricConfigManager.queryMetricItemConfigs(domains);
for(MetricItemConfig metricConfig : metricConfigs){
String domain = metricConfig.getDomain();
String type = metricConfig.getType();
......
......@@ -5,7 +5,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
......@@ -98,7 +97,7 @@ public class GraphCreator extends GraphCreatorBase{
protected boolean isProductLineInGroup(String productLine, List<MetricKeyConfig> configs) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine);
List<MetricItemConfig> metricConfig = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
List<MetricItemConfig> metricConfig = m_metricConfigManager.queryMetricItemConfigs(domains);
for (MetricKeyConfig metric : configs) {
String domain = metric.getMetricDomain();
......@@ -140,7 +139,7 @@ public class GraphCreator extends GraphCreatorBase{
private boolean showInDashboard(String productline) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productline);
List<MetricItemConfig> configs = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
List<MetricItemConfig> configs = m_metricConfigManager.queryMetricItemConfigs(domains);
for (MetricItemConfig config : configs) {
if (config.isShowAvgDashboard() || config.isShowCountDashboard() || config.isShowSumDashboard()) {
return true;
......
......@@ -9,8 +9,8 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.Set;
import org.codehaus.plexus.logging.LogEnabled;
import org.codehaus.plexus.logging.Logger;
......@@ -233,7 +233,7 @@ public abstract class GraphCreatorBase implements LogEnabled {
private Map<String, double[]> queryMetricValueByDate(String productLine, long start) {
MetricReport metricReport = m_metricReportService.queryMetricReport(productLine, new Date(start));
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine);
List<MetricItemConfig> metricConfigs = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
List<MetricItemConfig> metricConfigs = m_metricConfigManager.queryMetricItemConfigs(domains);
Collections.sort(metricConfigs, new Comparator<MetricItemConfig>() {
@Override
......
......@@ -22,8 +22,6 @@ public class AlertConfig {
private static final Long ONE_MINUTE_MILLSEC = 60000L;
private static final int JUDGE_DEFAULT_MINUTE = 3;
public List<String> buildExceptionSMSReceivers(ProductLine productLine) {
List<String> phones = new ArrayList<String>();
......@@ -73,10 +71,7 @@ public class AlertConfig {
}
public Pair<Boolean, String> checkData(MetricItemConfig config, double[] value, double[] baseline, MetricType type) {
int listLength = value.length;
int length = JUDGE_DEFAULT_MINUTE > listLength ? listLength : JUDGE_DEFAULT_MINUTE;
double[] valueTrim = getLastMinutes(value, length);
double[] baseLineTrim = getLastMinutes(baseline, length);
int length = value.length;
StringBuilder baselines = new StringBuilder();
StringBuilder values = new StringBuilder();
double decreasePercent = config.getDecreasePercentage();
......@@ -93,18 +88,17 @@ public class AlertConfig {
}
for (int i = 0; i < length; i++) {
baselines.append(df.format(baseLineTrim[i])).append(" ");
values.append(df.format(valueTrim[i])).append(" ");
valueSum = valueSum + valueTrim[i];
baselineSum = baselineSum + baseLineTrim[i];
baselines.append(df.format(baseline[i])).append(" ");
values.append(df.format(value[i])).append(" ");
valueSum = valueSum + value[i];
baselineSum = baselineSum + baseline[i];
if (baseLineTrim[i] <= 0) {
baseLineTrim[i] = 100;
if (baseline[i] <= 0) {
baseline[i] = 100;
return new Pair<Boolean, String>(false, "");
}
if (type == MetricType.COUNT || type == MetricType.SUM) {
if (valueTrim[i] / baseLineTrim[i] > (1 - decreasePercent / 100)
|| (baseLineTrim[i] - valueTrim[i]) < decreaseValue) {
if (value[i] / baseline[i] > (1 - decreasePercent / 100) || (baseline[i] - value[i]) < decreaseValue) {
return new Pair<Boolean, String>(false, "");
}
}
......@@ -122,18 +116,9 @@ public class AlertConfig {
public Pair<Boolean, String> checkData(MetricItemConfig config, double[] value, double[] baseline, MetricType type,
List<Config> configs) {
int valLength = value.length;
for (Config con : configs) {
int dataLength = getMaxMinute(con);
if (dataLength > valLength) {
continue;
}
Pair<Boolean, String> tmpResult = checkDataByConfig(value, baseline, type, con);
double[] validVal = getLastMinutes(value, dataLength);
double[] validBase = getLastMinutes(baseline, dataLength);
Pair<Boolean, String> tmpResult = checkDataByConfig(config, validVal, validBase, type, con);
if (tmpResult.getKey() == true) {
return tmpResult;
}
......@@ -141,19 +126,7 @@ public class AlertConfig {
return new Pair<Boolean, String>(false, "");
}
private int getMaxMinute(Config con) {
int maxMinute = 0;
for (Condition condition : con.getConditions()) {
int tmpMinute = condition.getMinute();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
return maxMinute;
}
private Pair<Boolean, String> checkDataByConfig(MetricItemConfig config, double[] value, double[] baseline,
MetricType type, Config con) {
private Pair<Boolean, String> checkDataByConfig(double[] value, double[] baseline, MetricType type, Config con) {
int length = value.length;
StringBuilder baselines = new StringBuilder();
StringBuilder values = new StringBuilder();
......@@ -187,16 +160,6 @@ public class AlertConfig {
return new Pair<Boolean, String>(true, sb.toString());
}
private double[] getLastMinutes(double[] doubleList, int remainCount) {
double[] result = new double[remainCount];
int startIndex = doubleList.length - remainCount;
for (int i = 0; i < remainCount; i++) {
result[i] = doubleList[startIndex + i];
}
return result;
}
private Long getMillsByString(String time) throws Exception {
String[] times = time.split(":");
int hour = Integer.parseInt(times[0]);
......@@ -233,7 +196,7 @@ public class AlertConfig {
int minute = condition.getMinute();
if (minute == 0) {
minute = JUDGE_DEFAULT_MINUTE;
minute = 3;
}
if (index < length - minute) {
......
package com.dianping.cat.report.task.metric;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.advanced.metric.config.entity.MetricItemConfig;
import com.dianping.cat.consumer.company.model.entity.ProductLine;
import com.dianping.cat.consumer.metric.MetricAnalyzer;
import com.dianping.cat.consumer.metric.MetricConfigManager;
import com.dianping.cat.consumer.metric.ProductLineConfigManager;
import com.dianping.cat.consumer.metric.model.entity.MetricReport;
import com.dianping.cat.consumer.metric.model.entity.Segment;
import com.dianping.cat.report.baseline.BaselineService;
import com.dianping.cat.service.ModelPeriod;
import com.dianping.cat.service.ModelRequest;
public abstract class BaseAlert {
@Inject
protected MetricConfigManager m_metricConfigManager;
@Inject
protected ProductLineConfigManager m_productLineConfigManager;
@Inject
protected BaselineService m_baselineService;
@Inject
protected RemoteMetricReportService m_service;
protected static final int DATA_AREADY_MINUTE = 1;
protected Map<String, MetricReport> m_currentReports = new HashMap<String, MetricReport>();
protected Map<String, MetricReport> m_lastReports = new HashMap<String, MetricReport>();
protected MetricReport fetchMetricReport(String product, ModelPeriod period) {
if (period == ModelPeriod.CURRENT) {
MetricReport report = m_currentReports.get(product);
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.CURRENT.getStartTime()).setProperty(
"requireAll", "ture");
report = m_service.invoke(request);
if (report != null) {
m_currentReports.put(product, report);
}
return report;
}
} else if (period == ModelPeriod.LAST) {
MetricReport report = m_lastReports.get(product);
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.LAST.getStartTime()).setProperty("requireAll",
"ture");
report = m_service.invoke(request);
if (report != null) {
m_lastReports.put(product, report);
}
return report;
}
} else {
throw new RuntimeException("internal error, this can't be reached.");
}
}
protected double[] mergerArray(double[] from, double[] to) {
int fromLength = from.length;
int toLength = to.length;
double[] result = new double[fromLength + toLength];
int index = 0;
for (int i = 0; i < fromLength; i++) {
result[i] = from[i];
index++;
}
for (int i = 0; i < toLength; i++) {
result[i + index] = to[i];
}
return result;
}
protected abstract void processMetricItemConfig(MetricItemConfig config, int minute, ProductLine productLine);
protected void processProductLine(ProductLine productLine) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine.getId());
List<MetricItemConfig> configs = m_metricConfigManager.queryMetricItemConfigs(domains);
long current = (System.currentTimeMillis()) / 1000 / 60;
int minute = (int) (current % (60)) - DATA_AREADY_MINUTE;
for (MetricItemConfig config : configs) {
processMetricItemConfig(config, minute, productLine);
}
}
protected double[] queryBaseLine(int start, int end, String baseLineKey, Date date, MetricType type) {
double[] baseline = m_baselineService.queryHourlyBaseline(MetricAnalyzer.ID, baseLineKey + ":" + type, date);
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(baseline, start, result, 0, length);
return result;
}
protected double[] queryRealData(int start, int end, String metricKey, MetricReport report, MetricType type) {
double[] all = new double[60];
Map<Integer, Segment> map = report.findOrCreateMetricItem(metricKey).getSegments();
for (Entry<Integer, Segment> entry : map.entrySet()) {
Integer minute = entry.getKey();
Segment seg = entry.getValue();
if (type == MetricType.AVG) {
all[minute] = seg.getAvg();
} else if (type == MetricType.COUNT) {
all[minute] = (double) seg.getCount();
} else if (type == MetricType.SUM) {
all[minute] = seg.getSum();
}
}
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(all, start, result, 0, length);
return result;
}
}
package com.dianping.cat.report.task.metric;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.codehaus.plexus.logging.LogEnabled;
import org.codehaus.plexus.logging.Logger;
......@@ -17,24 +15,19 @@ import org.unidal.tuple.Pair;
import com.dianping.cat.Cat;
import com.dianping.cat.advanced.metric.config.entity.MetricItemConfig;
import com.dianping.cat.consumer.company.model.entity.ProductLine;
import com.dianping.cat.consumer.metric.MetricAnalyzer;
import com.dianping.cat.consumer.metric.MetricConfigManager;
import com.dianping.cat.consumer.metric.ProductLineConfigManager;
import com.dianping.cat.consumer.metric.model.entity.MetricReport;
import com.dianping.cat.consumer.metric.model.entity.Segment;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.home.monitorrules.entity.Condition;
import com.dianping.cat.home.monitorrules.entity.Config;
import com.dianping.cat.home.monitorrules.entity.Subcondition;
import com.dianping.cat.message.Event;
import com.dianping.cat.message.Transaction;
import com.dianping.cat.report.baseline.BaselineService;
import com.dianping.cat.service.ModelPeriod;
import com.dianping.cat.service.ModelRequest;
import com.dianping.cat.system.config.MetricRuleConfigManager;
import com.dianping.cat.system.tool.MailSMS;
public class MetricAlert implements Task, LogEnabled {
public class MetricAlert extends BaseAlert implements Task, LogEnabled {
@Inject
private MetricConfigManager m_metricConfigManager;
......@@ -42,18 +35,9 @@ public class MetricAlert implements Task, LogEnabled {
@Inject
private ProductLineConfigManager m_productLineConfigManager;
@Inject
private MetricRuleConfigManager m_metricRuleConfigManager;
@Inject
private BaselineService m_baselineService;
@Inject
private MailSMS m_mailSms;
@Inject
private RemoteMetricReportService m_service;
@Inject
private AlertConfig m_alertConfig;
......@@ -64,98 +48,25 @@ public class MetricAlert implements Task, LogEnabled {
private static final int DATA_CHECK_MINUTE = 3;
private static final int DATA_AREADY_MINUTE = 1;
private Map<String, MetricReport> m_currentReports = new HashMap<String, MetricReport>();
private Map<String, MetricReport> m_lastReports = new HashMap<String, MetricReport>();
private Logger m_logger;
private void addDescMetricIfNotExist(MetricItemConfig config, List<Config> configs) {
double descPer = config.getDecreasePercentage();
double descVal = config.getDecreaseValue();
boolean isDescPerExist = false;
boolean isDescValExist = false;
String dayBeginTime = "00:00";
String dayEndTime = "24:00";
if (descPer == 0) {
descPer = 50;
}
if (descVal == 0) {
descVal = 100;
}
for (Config con : configs) {
String startTime = con.getStarttime();
String endTime = con.getEndtime();
if (startTime == null || !startTime.equals(dayBeginTime) || endTime == null || !endTime.equals(dayEndTime)) {
continue;
}
for (Condition c : con.getConditions()) {
List<Subcondition> subCons = c.getSubconditions();
if (subCons.size() != 2) {
continue;
}
for (Subcondition sub : subCons) {
RuleType type = RuleType.getByTypeId(sub.getType());
switch (type) {
case DecreasePercentage:
isDescPerExist = true;
break;
case DecreaseValue:
isDescValExist = true;
break;
default:
break;
}
}
if (isDescPerExist && isDescValExist) {
break;
} else {
isDescPerExist = false;
isDescValExist = false;
}
}
}
if (isDescPerExist && isDescValExist) {
return;
} else {
addNewCondition(configs, "DescPer", descPer, "DescVal", descVal, dayBeginTime, dayEndTime);
}
}
private void addNewCondition(List<Config> configs, String type, double val, String type2, double val2,
String dayBeginTime, String dayEndTime) {
configs.add(new Config()
.setStarttime(dayBeginTime)
.setEndtime(dayEndTime)
.addCondition(
new Condition().setTitle("default rule")
.addSubcondition(new Subcondition().setType(type).setText(String.valueOf(val)))
.addSubcondition(new Subcondition().setType(type2).setText(String.valueOf(val2)))));
}
private Pair<Boolean, String> checkDataByJudge(MetricItemConfig config, double[] value, double[] baseline,
MetricType type, List<Config> configs) {
MetricType type) {
Pair<Boolean, String> originResult = m_alertConfig.checkData(config, value, baseline, type);
addDescMetricIfNotExist(config, configs);
Pair<Boolean, String> ruleJudgeResult = m_alertConfig.checkData(config, value, baseline, type, configs);
if (originResult.getKey() != ruleJudgeResult.getKey()) {
Cat.logError("rule execute error!", new Exception());
try {
List<Config> configs = convert(config);
Pair<Boolean, String> ruleJudgeResult = m_alertConfig.checkData(config, value, baseline, type, configs);
if (originResult.getKey() != ruleJudgeResult.getKey()) {
String metricKey = m_metricConfigManager.buildMetricKey(config.getDomain(), config.getType(),
config.getMetricKey());
m_logger.error(String.format("Error judge result, config: %s, value: %s, baseline: %s", metricKey,
printArray(value), printArray(baseline)));
}
} catch (Exception e) {
Cat.logError(e);
}
return originResult;
}
......@@ -165,31 +76,29 @@ public class MetricAlert implements Task, LogEnabled {
String domain = config.getDomain();
String key = config.getMetricKey();
String metricKey = m_metricConfigManager.buildMetricKey(domain, config.getType(), key);
List<Config> configs = m_metricRuleConfigManager.getConfigs(product, domain, key, metricKey);
int maxMinute = queryMaxMinute(configs);
if (minute >= maxMinute - 1) {
if (minute >= DATA_CHECK_MINUTE - 1) {
MetricReport report = fetchMetricReport(product, ModelPeriod.CURRENT);
if (report != null) {
int start = minute + 1 - maxMinute;
int start = minute + 1 - DATA_CHECK_MINUTE;
int end = minute;
value = queryRealData(start, end, metricKey, report, type);
baseline = queryBaseLine(start, end, metricKey, new Date(ModelPeriod.CURRENT.getStartTime()), type);
return checkDataByJudge(config, value, baseline, type, configs);
return checkDataByJudge(config, value, baseline, type);
}
} else if (minute < 0) {
MetricReport lastReport = fetchMetricReport(product, ModelPeriod.LAST);
if (lastReport != null) {
int start = 60 + minute + 1 - (maxMinute);
int start = 60 + minute + 1 - (DATA_CHECK_MINUTE);
int end = 60 + minute;
value = queryRealData(start, end, metricKey, lastReport, type);
baseline = queryBaseLine(start, end, metricKey, new Date(ModelPeriod.LAST.getStartTime()), type);
return checkDataByJudge(config, value, baseline, type, configs);
return checkDataByJudge(config, value, baseline, type);
}
} else {
MetricReport currentReport = fetchMetricReport(product, ModelPeriod.CURRENT);
......@@ -201,7 +110,7 @@ public class MetricAlert implements Task, LogEnabled {
double[] currentBaseline = queryBaseLine(currentStart, currentEnd, metricKey,
new Date(ModelPeriod.CURRENT.getStartTime()), type);
int lastStart = 60 + 1 - (maxMinute - minute);
int lastStart = 60 + 1 - (DATA_CHECK_MINUTE - minute);
int lastEnd = 59;
double[] lastValue = queryRealData(lastStart, lastEnd, metricKey, lastReport, type);
double[] lastBaseline = queryBaseLine(lastStart, lastEnd, metricKey,
......@@ -209,65 +118,41 @@ public class MetricAlert implements Task, LogEnabled {
value = mergerArray(lastValue, currentValue);
baseline = mergerArray(lastBaseline, currentBaseline);
return checkDataByJudge(config, value, baseline, type, configs);
return checkDataByJudge(config, value, baseline, type);
}
}
return null;
}
@Override
public void enableLogging(Logger logger) {
m_logger = logger;
}
private List<Config> convert(MetricItemConfig metricItemConfig) {
List<Config> configs = new ArrayList<Config>();
Config config = new Config();
Condition condition = new Condition();
Subcondition subcondition1 = new Subcondition();
Subcondition subcondition2 = new Subcondition();
private MetricReport fetchMetricReport(String product, ModelPeriod period) {
if (period == ModelPeriod.CURRENT) {
MetricReport report = m_currentReports.get(product);
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.CURRENT.getStartTime()).setProperty(
"requireAll", "ture");
report = m_service.invoke(request);
if (report != null) {
m_currentReports.put(product, report);
}
return report;
}
} else if (period == ModelPeriod.LAST) {
MetricReport report = m_lastReports.get(product);
double decreasePercent = metricItemConfig.getDecreasePercentage();
double decreaseValue = metricItemConfig.getDecreaseValue();
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.LAST.getStartTime()).setProperty("requireAll",
"ture");
report = m_service.invoke(request);
if (report != null) {
m_lastReports.put(product, report);
}
return report;
}
} else {
throw new RuntimeException("internal error, this can't be reached.");
if (decreasePercent == 0) {
decreasePercent = 50;
}
if (decreaseValue == 0) {
decreaseValue = 100;
}
}
private int queryMaxMinute(List<Config> configs) {
int maxMinute = DATA_CHECK_MINUTE;
subcondition1.setType("DescPer").setText(String.valueOf(decreasePercent));
subcondition2.setType("DescVal").setText(String.valueOf(decreaseValue));
for (Config config : configs) {
for (Condition con : config.getConditions()) {
int tmpMinute = con.getMinute();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
}
return maxMinute;
condition.addSubcondition(subcondition1).addSubcondition(subcondition2);
config.addCondition(condition);
configs.add(config);
return configs;
}
@Override
public void enableLogging(Logger logger) {
m_logger = logger;
}
@Override
......@@ -275,28 +160,21 @@ public class MetricAlert implements Task, LogEnabled {
return "metric-alert";
}
private double[] mergerArray(double[] from, double[] to) {
int fromLength = from.length;
int toLength = to.length;
double[] result = new double[fromLength + toLength];
int index = 0;
private String printArray(double[] value) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fromLength; i++) {
result[i] = from[i];
index++;
}
for (int i = 0; i < toLength; i++) {
result[i + index] = to[i];
for (double d : value) {
sb.append(d).append(" ");
}
return result;
return sb.toString();
}
private void processMetricItemConfig(MetricItemConfig config, int minute, ProductLine productLine) {
protected void processMetricItemConfig(MetricItemConfig config, int minute, ProductLine productLine) {
if ((!config.getAlarm() && !config.isShowAvgDashboard() && !config.isShowSumDashboard() && !config
.isShowCountDashboard())) {
return;
}
String product =productLine.getId();
String product = productLine.getId();
Pair<Boolean, String> alert = null;
if (config.isShowAvg()) {
......@@ -317,57 +195,14 @@ public class MetricAlert implements Task, LogEnabled {
}
}
private void processProductLine(ProductLine productLine) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine.getId());
List<MetricItemConfig> configs = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
long current = (System.currentTimeMillis()) / 1000 / 60;
int minute = (int) (current % (60)) - DATA_AREADY_MINUTE;
for (MetricItemConfig config : configs) {
processMetricItemConfig(config, minute, productLine);
}
}
private double[] queryBaseLine(int start, int end, String baseLineKey, Date date, MetricType type) {
double[] baseline = m_baselineService.queryHourlyBaseline(MetricAnalyzer.ID, baseLineKey + ":" + type, date);
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(baseline, start, result, 0, length);
return result;
}
private double[] queryRealData(int start, int end, String metricKey, MetricReport report, MetricType type) {
double[] all = new double[60];
Map<Integer, Segment> map = report.findOrCreateMetricItem(metricKey).getSegments();
for (Entry<Integer, Segment> entry : map.entrySet()) {
Integer minute = entry.getKey();
Segment seg = entry.getValue();
if (type == MetricType.AVG) {
all[minute] = seg.getAvg();
} else if (type == MetricType.COUNT) {
all[minute] = (double) seg.getCount();
} else if (type == MetricType.SUM) {
all[minute] = seg.getSum();
}
}
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(all, start, result, 0, length);
return result;
}
@Override
public void run() {
boolean active = true;
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
active = false;
}
boolean active = true;
while (active) {
int minute = Calendar.getInstance().get(Calendar.MINUTE);
String minuteStr = String.valueOf(minute);
......@@ -385,7 +220,9 @@ public class MetricAlert implements Task, LogEnabled {
for (ProductLine productLine : productLines.values()) {
try {
processProductLine(productLine);
if (productLine.isMetricDashboard()) {
processProductLine(productLine);
}
} catch (Exception e) {
Cat.logError(e);
}
......
......@@ -3,10 +3,8 @@ package com.dianping.cat.report.task.metric;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.codehaus.plexus.logging.LogEnabled;
import org.codehaus.plexus.logging.Logger;
......@@ -17,43 +15,24 @@ import org.unidal.tuple.Pair;
import com.dianping.cat.Cat;
import com.dianping.cat.advanced.metric.config.entity.MetricItemConfig;
import com.dianping.cat.consumer.company.model.entity.ProductLine;
import com.dianping.cat.consumer.metric.MetricAnalyzer;
import com.dianping.cat.consumer.metric.MetricConfigManager;
import com.dianping.cat.consumer.metric.ProductLineConfigManager;
import com.dianping.cat.consumer.metric.model.entity.MetricReport;
import com.dianping.cat.consumer.metric.model.entity.Segment;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.home.monitorrules.entity.Condition;
import com.dianping.cat.home.monitorrules.entity.Config;
import com.dianping.cat.home.monitorrules.entity.Subcondition;
import com.dianping.cat.message.Event;
import com.dianping.cat.message.Transaction;
import com.dianping.cat.report.baseline.BaselineService;
import com.dianping.cat.service.ModelPeriod;
import com.dianping.cat.service.ModelRequest;
import com.dianping.cat.system.config.MetricRuleConfigManager;
import com.dianping.cat.system.tool.MailSMS;
public class SwitchAlert implements Task, LogEnabled {
@Inject
private MetricConfigManager m_metricConfigManager;
@Inject
private ProductLineConfigManager m_productLineConfigManager;
public class SwitchAlert extends BaseAlert implements Task, LogEnabled {
@Inject
private MetricRuleConfigManager m_metricRuleConfigManager;
@Inject
private BaselineService m_baselineService;
@Inject
private MailSMS m_mailSms;
@Inject
private RemoteMetricReportService m_service;
@Inject
private SwitchAlertConfig m_alertConfig;
......@@ -62,109 +41,19 @@ public class SwitchAlert implements Task, LogEnabled {
private static final long DURATION = TimeUtil.ONE_MINUTE;
private static final int DATA_AREADY_MINUTE = 1;
private Map<String, MetricReport> m_currentReports = new HashMap<String, MetricReport>();
private Map<String, MetricReport> m_lastReports = new HashMap<String, MetricReport>();
private Logger m_logger;
private void addDescMetricIfNotExist(MetricItemConfig config, List<Config> configs) {
double descPer = config.getDecreasePercentage();
double descVal = config.getDecreaseValue();
boolean isDescPerExist = false;
boolean isDescValExist = false;
String dayBeginTime = "00:00";
String dayEndTime = "24:00";
if (descPer == 0) {
descPer = 50;
}
if (descVal == 0) {
descVal = 100;
}
for (Config con : configs) {
String startTime = con.getStarttime();
String endTime = con.getEndtime();
if (startTime == null || !startTime.equals(dayBeginTime) || endTime == null || !endTime.equals(dayEndTime)) {
continue;
}
for (Condition c : con.getConditions()) {
List<Subcondition> subCons = c.getSubconditions();
if (subCons.size() != 2) {
continue;
}
for (Subcondition sub : subCons) {
RuleType type = RuleType.getByTypeId(sub.getType());
switch (type) {
case DecreasePercentage:
isDescPerExist = true;
break;
case DecreaseValue:
isDescValExist = true;
break;
default:
break;
}
}
if (isDescPerExist && isDescValExist) {
break;
} else {
isDescPerExist = false;
isDescValExist = false;
}
}
}
if (isDescPerExist && isDescValExist) {
return;
} else {
addNewCondition(configs, "DescPer", descPer, "DescVal", descVal, dayBeginTime, dayEndTime);
}
}
private void addNewCondition(List<Config> configs, String type, double val, String type2, double val2,
String dayBeginTime, String dayEndTime) {
configs.add(new Config()
.setStarttime(dayBeginTime)
.setEndtime(dayEndTime)
.addCondition(
new Condition().setTitle("default rule")
.addSubcondition(new Subcondition().setType(type).setText(String.valueOf(val)))
.addSubcondition(new Subcondition().setType(type2).setText(String.valueOf(val2)))));
}
private Pair<Boolean, String> checkDataByJudge(MetricItemConfig config, double[] value, double[] baseline,
MetricType type, List<Config> configs) {
Pair<Boolean, String> originResult = m_alertConfig.checkData(config, value, baseline, type);
addDescMetricIfNotExist(config, configs);
Pair<Boolean, String> ruleJudgeResult = m_alertConfig.checkData(config, value, baseline, type, configs);
if (originResult.getKey() != ruleJudgeResult.getKey()) {
Cat.logError("rule execute error!", new Exception());
}
return originResult;
}
private Pair<Boolean, String> computeAlertInfo(int minute, String product, MetricItemConfig config, MetricType type) {
double[] value = null;
double[] baseline = null;
String domain = config.getDomain();
String key = config.getMetricKey();
String metricKey = m_metricConfigManager.buildMetricKey(domain, config.getType(), key);
List<Config> configs = m_metricRuleConfigManager.getConfigs(product, domain, key, metricKey);
List<Config> configs = m_metricRuleConfigManager.buildConfigs(product, domain, key, metricKey);
int maxMinute = queryCheckMinute(configs);
if (minute >= maxMinute - 1) {
......@@ -177,7 +66,7 @@ public class SwitchAlert implements Task, LogEnabled {
value = queryRealData(start, end, metricKey, report, type);
baseline = queryBaseLine(start, end, metricKey, new Date(ModelPeriod.CURRENT.getStartTime()), type);
return checkDataByJudge(config, value, baseline, type, configs);
return m_alertConfig.checkData(config, value, baseline, type, configs);
}
} else if (minute < 0) {
MetricReport lastReport = fetchMetricReport(product, ModelPeriod.LAST);
......@@ -188,7 +77,7 @@ public class SwitchAlert implements Task, LogEnabled {
value = queryRealData(start, end, metricKey, lastReport, type);
baseline = queryBaseLine(start, end, metricKey, new Date(ModelPeriod.LAST.getStartTime()), type);
return checkDataByJudge(config, value, baseline, type, configs);
return m_alertConfig.checkData(config, value, baseline, type, configs);
}
} else {
MetricReport currentReport = fetchMetricReport(product, ModelPeriod.CURRENT);
......@@ -208,7 +97,7 @@ public class SwitchAlert implements Task, LogEnabled {
value = mergerArray(lastValue, currentValue);
baseline = mergerArray(lastBaseline, currentBaseline);
return checkDataByJudge(config, value, baseline, type, configs);
return m_alertConfig.checkData(config, value, baseline, type, configs);
}
}
return null;
......@@ -219,82 +108,15 @@ public class SwitchAlert implements Task, LogEnabled {
m_logger = logger;
}
private MetricReport fetchMetricReport(String product, ModelPeriod period) {
if (period == ModelPeriod.CURRENT) {
MetricReport report = m_currentReports.get(product);
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.CURRENT.getStartTime()).setProperty(
"requireAll", "ture");
report = m_service.invoke(request);
if (report != null) {
m_currentReports.put(product, report);
}
return report;
}
} else if (period == ModelPeriod.LAST) {
MetricReport report = m_lastReports.get(product);
if (report != null) {
return report;
} else {
ModelRequest request = new ModelRequest(product, ModelPeriod.LAST.getStartTime()).setProperty("requireAll",
"ture");
report = m_service.invoke(request);
if (report != null) {
m_lastReports.put(product, report);
}
return report;
}
} else {
throw new RuntimeException("internal error, this can't be reached.");
}
}
private int queryCheckMinute(List<Config> configs) {
int maxMinute = 0;
for (Config config : configs) {
for (Condition con : config.getConditions()) {
int tmpMinute = con.getMinute();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
}
return maxMinute;
}
@Override
public String getName() {
return "metric-alert";
}
private double[] mergerArray(double[] from, double[] to) {
int fromLength = from.length;
int toLength = to.length;
double[] result = new double[fromLength + toLength];
int index = 0;
for (int i = 0; i < fromLength; i++) {
result[i] = from[i];
index++;
}
for (int i = 0; i < toLength; i++) {
result[i + index] = to[i];
}
return result;
}
private void processMetricItemConfig(MetricItemConfig config, int minute, ProductLine productLine) {
protected void processMetricItemConfig(MetricItemConfig config, int minute, ProductLine productLine) {
String product = productLine.getId();
Pair<Boolean, String> alert = null;
if (config.isShowAvg()) {
alert = computeAlertInfo(minute, product, config, MetricType.AVG);
}
......@@ -313,57 +135,30 @@ public class SwitchAlert implements Task, LogEnabled {
}
}
private void processProductLine(ProductLine productLine) {
List<String> domains = m_productLineConfigManager.queryDomainsByProductLine(productLine.getId());
List<MetricItemConfig> configs = m_metricConfigManager.queryMetricItemConfigs(new HashSet<String>(domains));
long current = (System.currentTimeMillis()) / 1000 / 60;
int minute = (int) (current % (60)) - DATA_AREADY_MINUTE;
for (MetricItemConfig config : configs) {
processMetricItemConfig(config, minute, productLine);
}
}
private double[] queryBaseLine(int start, int end, String baseLineKey, Date date, MetricType type) {
double[] baseline = m_baselineService.queryHourlyBaseline(MetricAnalyzer.ID, baseLineKey + ":" + type, date);
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(baseline, start, result, 0, length);
return result;
}
private double[] queryRealData(int start, int end, String metricKey, MetricReport report, MetricType type) {
double[] all = new double[60];
Map<Integer, Segment> map = report.findOrCreateMetricItem(metricKey).getSegments();
private int queryCheckMinute(List<Config> configs) {
int maxMinute = 0;
for (Entry<Integer, Segment> entry : map.entrySet()) {
Integer minute = entry.getKey();
Segment seg = entry.getValue();
for (Config config : configs) {
for (Condition con : config.getConditions()) {
int tmpMinute = con.getMinute();
if (type == MetricType.AVG) {
all[minute] = seg.getAvg();
} else if (type == MetricType.COUNT) {
all[minute] = (double) seg.getCount();
} else if (type == MetricType.SUM) {
all[minute] = seg.getSum();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
}
int length = end - start + 1;
double[] result = new double[length];
System.arraycopy(all, start, result, 0, length);
return result;
return maxMinute;
}
@Override
public void run() {
boolean active = true;
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
active = false;
}
boolean active = true;
while (active) {
int minute = Calendar.getInstance().get(Calendar.MINUTE);
String minuteStr = String.valueOf(minute);
......@@ -410,14 +205,13 @@ public class SwitchAlert implements Task, LogEnabled {
private void sendAlertInfo(ProductLine productLine, MetricItemConfig config, String content) {
List<String> emails = m_alertConfig.buildMailReceivers(productLine);
List<String> phones = m_alertConfig.buildSMSReceivers(productLine);
String title = m_alertConfig.buildMailTitle(productLine, config);
m_logger.info(title + " " + content + " " + emails);
m_mailSms.sendEmail(title, content, emails);
m_mailSms.sendSms(title + " " + content, content, phones);
Cat.logEvent("MetricAlert", productLine.getId(), Event.SUCCESS, title + " " + content);
Cat.logEvent("SwitchAlert", productLine.getId(), Event.SUCCESS, title + " " + content);
}
@Override
......
......@@ -65,90 +65,24 @@ public class SwitchAlertConfig {
return phones;
}
public Pair<Boolean, String> checkData(MetricItemConfig config, double[] value, double[] baseline, MetricType type) {
int listLength = value.length;
int length = JUDGE_DEFAULT_MINUTE > listLength ? listLength : JUDGE_DEFAULT_MINUTE;
double[] valueTrim = getLastMinutes(value, length);
double[] baseLineTrim = getLastMinutes(baseline, length);
StringBuilder baselines = new StringBuilder();
StringBuilder values = new StringBuilder();
double decreasePercent = config.getDecreasePercentage();
double decreaseValue = config.getDecreaseValue();
double valueSum = 0;
double baselineSum = 0;
DecimalFormat df = new DecimalFormat("0.0");
if (decreasePercent == 0) {
decreasePercent = 50;
}
if (decreaseValue == 0) {
decreaseValue = 100;
}
for (int i = 0; i < length; i++) {
baselines.append(df.format(baseLineTrim[i])).append(" ");
values.append(df.format(valueTrim[i])).append(" ");
valueSum = valueSum + valueTrim[i];
baselineSum = baselineSum + baseLineTrim[i];
if (baseLineTrim[i] <= 0) {
baseLineTrim[i] = 100;
return new Pair<Boolean, String>(false, "");
}
if (type == MetricType.COUNT || type == MetricType.SUM) {
if (valueTrim[i] / baseLineTrim[i] > (1 - decreasePercent / 100)
|| (baseLineTrim[i] - valueTrim[i]) < decreaseValue) {
return new Pair<Boolean, String>(false, "");
}
}
}
double percent = (1 - valueSum / baselineSum) * 100;
StringBuilder sb = new StringBuilder();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sb.append("[基线值:").append(baselines.toString()).append("] ");
sb.append("[实际值:").append(values.toString()).append("] ");
sb.append("[下降:").append(df.format(percent)).append("%").append("]");
sb.append("[告警时间:").append(sdf.format(new Date()) + "]");
return new Pair<Boolean, String>(true, sb.toString());
}
public Pair<Boolean, String> checkData(MetricItemConfig config, double[] value, double[] baseline, MetricType type,
List<Config> configs) {
int valLength = value.length;
for (Config con : configs) {
int dataLength = getMaxMinute(con);
if (dataLength > valLength) {
continue;
}
int dataLength = queryMaxMinute(con);
double[] validVal = getLastMinutes(value, dataLength);
double[] validBase = getLastMinutes(baseline, dataLength);
Pair<Boolean, String> tmpResult = checkDataByConfig(config, validVal, validBase, type, con);
if (tmpResult.getKey() == true) {
return tmpResult;
Pair<Boolean, String> result = checkDataByConfig(config, validVal, validBase, type, con);
if (result.getKey() == true) {
return result;
}
}
return new Pair<Boolean, String>(false, "");
}
private int getMaxMinute(Config con) {
int maxMinute = 0;
for (Condition condition : con.getConditions()) {
int tmpMinute = condition.getMinute();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
return maxMinute;
}
private Pair<Boolean, String> checkDataByConfig(MetricItemConfig config, double[] value, double[] baseline,
MetricType type, Config con) {
int length = value.length;
StringBuilder baselines = new StringBuilder();
StringBuilder values = new StringBuilder();
......@@ -261,4 +195,15 @@ public class SwitchAlertConfig {
return isRuleTriggered;
}
private int queryMaxMinute(Config con) {
int maxMinute = 0;
for (Condition condition : con.getConditions()) {
int tmpMinute = condition.getMinute();
if (tmpMinute > maxMinute) {
maxMinute = tmpMinute;
}
}
return maxMinute;
}
}
......@@ -68,7 +68,7 @@ public class MetricRuleConfigManager implements Initializable {
return configs;
}
public List<com.dianping.cat.home.monitorrules.entity.Config> getConfigs(String product, String domain, String key,
public List<com.dianping.cat.home.monitorrules.entity.Config> buildConfigs(String product, String domain, String key,
String metricKey) {
List<com.dianping.cat.home.monitorrules.entity.Config> configs = new ArrayList<com.dianping.cat.home.monitorrules.entity.Config>();
......
......@@ -20,7 +20,7 @@
</entity>
<entity name="condition">
<attribute name="title" value-type="String" />
<attribute name="minute" value-type="int" default-value="2"/>
<attribute name="minute" value-type="int" default-value="3"/>
<entity-ref name="subcondition" type="list" names="subconditions" />
</entity>
<entity name="subcondition">
......
......@@ -1719,6 +1719,36 @@
</requirement>
</requirements>
</component>
<component>
<role>com.dianping.cat.report.task.metric.SwitchAlert</role>
<implementation>com.dianping.cat.report.task.metric.SwitchAlert</implementation>
<requirements>
<requirement>
<role>com.dianping.cat.consumer.metric.MetricConfigManager</role>
</requirement>
<requirement>
<role>com.dianping.cat.consumer.metric.ProductLineConfigManager</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.baseline.BaselineService</role>
</requirement>
<requirement>
<role>com.dianping.cat.system.tool.MailSMS</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.task.metric.AlertConfig</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.task.metric.AlertInfo</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.task.metric.RemoteMetricReportService</role>
</requirement>
<requirement>
<role>com.dianping.cat.system.config.MetricRuleConfigManager</role>
</requirement>
</requirements>
</component>
<component>
<role>com.dianping.cat.report.task.exceptionAlert.ExceptionAlert</role>
<implementation>com.dianping.cat.report.task.exceptionAlert.ExceptionAlert</implementation>
......
......@@ -17,6 +17,7 @@ import com.dianping.cat.home.monitorrules.entity.Rule;
import com.dianping.cat.home.monitorrules.transform.DefaultSaxParser;
import com.dianping.cat.report.task.metric.AlertConfig;
import com.dianping.cat.report.task.metric.MetricType;
import com.dianping.cat.report.task.metric.SwitchAlertConfig;
public class RuleConfigTest {
......@@ -83,7 +84,7 @@ public class RuleConfigTest {
@Test
public void testMinute() {
AlertConfig alertConfig = new AlertConfig();
SwitchAlertConfig alertConfig = new SwitchAlertConfig();
MetricItemConfig config = new MetricItemConfig();
Map<String, List<com.dianping.cat.home.monitorrules.entity.Config>> configMap = buildConfigMap(buildMonitorRuleFromFile("/config/test-minute-monitor.xml"));
......
......@@ -154,7 +154,7 @@ public class AlertConfigTest {
@Test
public void testMinute() {
AlertConfig alertConfig = new AlertConfig();
SwitchAlertConfig alertConfig = new SwitchAlertConfig();
MetricItemConfig config = new MetricItemConfig();
Map<String, List<com.dianping.cat.home.monitorrules.entity.Config>> configMap = buildConfigMap(buildMonitorRuleFromFile("/config/test-minute-monitor.xml"));
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册