From 1a55f336ccc2b35177dc4238fda32dc0b1cb4f66 Mon Sep 17 00:00:00 2001 From: "leon.li" Date: Thu, 30 Oct 2014 10:51:47 +0800 Subject: [PATCH] add backend of high load sql module --- .../cat/consumer/state/StateDelegate.java | 5 +- .../main/java/com/dianping/cat/Constants.java | 2 + .../cat/build/TaskComponentConfigurator.java | 7 + .../task/highload/HighLoadSqlUpdater.java | 197 ++++++++++++++++++ .../report/task/highload/HighLoadUpdater.java | 31 +++ .../highload/TransactionHighLoadUpdater.java | 7 + .../task/overload/CapacityUpdateTask.java | 20 +- .../resources/META-INF/plexus/components.xml | 14 ++ 8 files changed, 270 insertions(+), 13 deletions(-) create mode 100644 cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadSqlUpdater.java create mode 100644 cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadUpdater.java create mode 100644 cat-home/src/main/java/com/dianping/cat/report/task/highload/TransactionHighLoadUpdater.java diff --git a/cat-consumer/src/main/java/com/dianping/cat/consumer/state/StateDelegate.java b/cat-consumer/src/main/java/com/dianping/cat/consumer/state/StateDelegate.java index e2af79e14..b3b3b6a59 100644 --- a/cat-consumer/src/main/java/com/dianping/cat/consumer/state/StateDelegate.java +++ b/cat-consumer/src/main/java/com/dianping/cat/consumer/state/StateDelegate.java @@ -44,7 +44,7 @@ public class StateDelegate implements ReportDelegate { String domain = report.getDomain(); m_taskManager.createTask(startTime, domain, StateAnalyzer.ID, TaskProlicy.ALL_EXCLUED_HOURLY); - + m_taskManager.createTask(startTime, domain, Constants.REPORT_SERVICE, TaskProlicy.ALL); m_taskManager.createTask(startTime, domain, Constants.REPORT_BUG, TaskProlicy.ALL); m_taskManager.createTask(startTime, domain, Constants.REPORT_HEAVY, TaskProlicy.ALL); @@ -52,11 +52,12 @@ public class StateDelegate implements ReportDelegate { m_taskManager.createTask(startTime, domain, Constants.REPORT_UTILIZATION, TaskProlicy.ALL); m_taskManager.createTask(startTime, domain, Constants.REPORT_NET_TOPOLOGY, TaskProlicy.HOULY); m_taskManager.createTask(startTime, domain, Constants.REPORT_DATABASE_CAPACITY, TaskProlicy.ALL); + m_taskManager.createTask(startTime, domain, Constants.HIGH_LOAD_SQL, TaskProlicy.DAILY); Calendar cal = Calendar.getInstance(); int hour = cal.get(Calendar.HOUR_OF_DAY); - // for daily report aggreation done + // for daily report aggreation done if (hour >= 4) { m_taskManager.createTask(startTime, domain, Constants.REPORT_NOTIFY, TaskProlicy.DAILY); m_taskManager.createTask(startTime, domain, Constants.REPORT_ROUTER, TaskProlicy.DAILY); diff --git a/cat-core/src/main/java/com/dianping/cat/Constants.java b/cat-core/src/main/java/com/dianping/cat/Constants.java index 5d4df7a34..99e7586e8 100644 --- a/cat-core/src/main/java/com/dianping/cat/Constants.java +++ b/cat-core/src/main/java/com/dianping/cat/Constants.java @@ -41,4 +41,6 @@ public class Constants { public static final String REPORT_NOTIFY = "dailyNotify"; + public static final String HIGH_LOAD_SQL = "highLoadSql"; + } diff --git a/cat-home/src/main/java/com/dianping/cat/build/TaskComponentConfigurator.java b/cat-home/src/main/java/com/dianping/cat/build/TaskComponentConfigurator.java index 998afa68a..eb0d7aa08 100644 --- a/cat-home/src/main/java/com/dianping/cat/build/TaskComponentConfigurator.java +++ b/cat-home/src/main/java/com/dianping/cat/build/TaskComponentConfigurator.java @@ -12,6 +12,7 @@ import com.dianping.cat.config.app.AppConfigManager; import com.dianping.cat.config.app.AppDataService; import com.dianping.cat.consumer.metric.MetricConfigManager; import com.dianping.cat.consumer.metric.ProductLineConfigManager; +import com.dianping.cat.consumer.transaction.TransactionAnalyzer; import com.dianping.cat.core.dal.DailyGraphDao; import com.dianping.cat.core.dal.DailyReportDao; import com.dianping.cat.core.dal.GraphDao; @@ -35,6 +36,7 @@ import com.dianping.cat.report.baseline.BaselineService; import com.dianping.cat.report.baseline.impl.DefaultBaselineCreator; import com.dianping.cat.report.baseline.impl.DefaultBaselineService; import com.dianping.cat.report.page.dependency.graph.TopologyGraphBuilder; +import com.dianping.cat.report.page.model.spi.ModelService; import com.dianping.cat.report.page.network.nettopology.NetGraphBuilder; import com.dianping.cat.report.page.transaction.TransactionMergeManager; import com.dianping.cat.report.service.ReportServiceManager; @@ -50,6 +52,7 @@ import com.dianping.cat.report.task.event.EventReportBuilder; import com.dianping.cat.report.task.heartbeat.HeartbeatGraphCreator; import com.dianping.cat.report.task.heartbeat.HeartbeatReportBuilder; import com.dianping.cat.report.task.heavy.HeavyReportBuilder; +import com.dianping.cat.report.task.highload.HighLoadSqlUpdater; import com.dianping.cat.report.task.matrix.MatrixReportBuilder; import com.dianping.cat.report.task.metric.MetricBaselineReportBuilder; import com.dianping.cat.report.task.metric.MetricPointParser; @@ -182,6 +185,10 @@ public class TaskComponentConfigurator extends AbstractResourceConfigurator { .req(CapacityUpdater.class, WeeklyCapacityUpdater.ID, "m_weeklyUpdater") .req(CapacityUpdater.class, MonthlyCapacityUpdater.ID, "m_monthlyUpdater")); + all.add(C(ReportTaskBuilder.class, HighLoadSqlUpdater.ID, HighLoadSqlUpdater.class)// + .req(ModelService.class, TransactionAnalyzer.ID)// + .req(ReportServiceManager.class)); + all.add(C(ReportRender.class, ReportRenderImpl.class)); all.add(C(AppDataComparisonRender.class)); diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadSqlUpdater.java b/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadSqlUpdater.java new file mode 100644 index 000000000..ab3ae8d69 --- /dev/null +++ b/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadSqlUpdater.java @@ -0,0 +1,197 @@ +package com.dianping.cat.report.task.highload; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.unidal.lookup.annotation.Inject; + +import com.dianping.cat.Cat; +import com.dianping.cat.Constants; +import com.dianping.cat.consumer.transaction.TransactionAnalyzer; +import com.dianping.cat.consumer.transaction.model.entity.TransactionName; +import com.dianping.cat.consumer.transaction.model.entity.TransactionReport; +import com.dianping.cat.helper.TimeHelper; +import com.dianping.cat.report.page.model.spi.ModelService; +import com.dianping.cat.report.page.transaction.DisplayNames; +import com.dianping.cat.report.page.transaction.DisplayNames.TransactionNameModel; +import com.dianping.cat.report.service.ReportServiceManager; +import com.dianping.cat.service.ModelRequest; +import com.dianping.cat.service.ModelResponse; + +public class HighLoadSqlUpdater extends TransactionHighLoadUpdater { + + @Inject(type = ModelService.class, value = TransactionAnalyzer.ID) + private ModelService m_transactionService; + + @Inject + private ReportServiceManager m_reportService; + + public static final String ID = Constants.HIGH_LOAD_SQL; + + @Override + public boolean buildDailyTask(String name, String domain, Date period) { + try { + List sqls = generateHighLoadSqls(); + return true; + } catch (Exception ex) { + Cat.logError(ex); + return false; + } + } + + private List generateHighLoadSqls() { + Set domains = queryDomains(); + Heap heap = new Heap(); + + for (String domain : domains) { + try { + TransactionReport report = m_reportService.queryTransactionReport(domain, TimeHelper.getYesterday(), + TimeHelper.getCurrentDay()); + DisplayNames displayNames = new DisplayNames(); + + displayNames.display("", getType(), "All", report, ""); + for (TransactionNameModel nameModel : displayNames.getResults()) { + HighLoadSQL sql = new HighLoadSQL(domain, nameModel.getDetail()); + heap.add(sql); + } + } catch (Exception e) { + Cat.logError(e); + } + } + + return heap.getSqls(); + } + + @Override + public String getID() { + return ID; + } + + @Override + public String getType() { + return "SQL"; + } + + private Set queryDomains() { + Set domains = new HashSet(); + ModelRequest request = new ModelRequest("cat", System.currentTimeMillis()); + + if (m_transactionService.isEligable(request)) { + ModelResponse response = m_transactionService.invoke(request); + domains.addAll(response.getModel().getDomainNames()); + } + return domains; + } + + public class Heap { + + private final int m_size = 100; + + private HighLoadSQL[] m_sqls = new HighLoadSQL[m_size]; + + public void add(HighLoadSQL sql) { + int nextIndex = findNextValidIndex(); + + if (nextIndex == m_size) { + if (isBigger(sql, m_sqls[0])) { + m_sqls[0] = sql; + heapAdjust(0, m_size - 1); + } + } else { + m_sqls[nextIndex] = sql; + if (nextIndex == m_size - 1) { + sort(); + } + } + } + + private int findNextValidIndex() { + int i; + + for (i = 0; i < m_size && m_sqls[i] != null; i++) { + } + return i; + } + + public List getSqls() { + return Arrays.asList(m_sqls); + } + + private void heapAdjust(int startIndex, int endIndex) { + int currentIndex = startIndex; + int tmpIndex = 2 * currentIndex + 1; + + while (tmpIndex <= endIndex) { + if (m_sqls[tmpIndex + 1] != null && isBigger(m_sqls[tmpIndex + 1], m_sqls[tmpIndex])) { + tmpIndex = tmpIndex + 1; + } + if (isBigger(m_sqls[currentIndex], m_sqls[tmpIndex])) { + swap(currentIndex, tmpIndex); + currentIndex = tmpIndex; + tmpIndex = 2 * currentIndex + 1; + } else { + break; + } + } + } + + private boolean isBigger(HighLoadSQL sql1, HighLoadSQL sql2) { + TransactionName name1 = sql1.getName(); + TransactionName name2 = sql2.getName(); + + return name1.getTotalCount() * name1.getAvg() - name2.getTotalCount() * name2.getAvg() > 0 ? true : false; + } + + private void sort() { + int currentIndex = (m_size - 1) / 2; + + for (; currentIndex >= 0; currentIndex--) { + heapAdjust(currentIndex, m_size); + } + + for (int i = m_size - 1; i > 0; i--) { + swap(0, i); + heapAdjust(0, i - 1); + } + + } + + private void swap(int index1, int index2) { + HighLoadSQL tmpNode = m_sqls[index1]; + m_sqls[index1] = m_sqls[index2]; + m_sqls[index2] = tmpNode; + } + } + + public class HighLoadSQL { + private String m_domain; + + private TransactionName m_name; + + public HighLoadSQL(String domain, TransactionName name) { + m_domain = domain; + m_name = name; + } + + public String getDomain() { + return m_domain; + } + + public TransactionName getName() { + return m_name; + } + + public void setDomain(String domain) { + m_domain = domain; + } + + public void setName(TransactionName name) { + m_name = name; + } + + } + +} \ No newline at end of file diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadUpdater.java b/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadUpdater.java new file mode 100644 index 000000000..c7433e18c --- /dev/null +++ b/cat-home/src/main/java/com/dianping/cat/report/task/highload/HighLoadUpdater.java @@ -0,0 +1,31 @@ +package com.dianping.cat.report.task.highload; + +import java.util.Date; + +import com.dianping.cat.report.task.spi.ReportTaskBuilder; + +public abstract class HighLoadUpdater implements ReportTaskBuilder { + + @Override + public boolean buildDailyTask(String name, String domain, Date period) { + throw new RuntimeException(getID() + " don't support daily update"); + } + + @Override + public boolean buildHourlyTask(String name, String domain, Date period) { + throw new RuntimeException(getID() + " don't support hourly update"); + } + + @Override + public boolean buildMonthlyTask(String name, String domain, Date period) { + throw new RuntimeException(getID() + " don't support monthly update"); + } + + @Override + public boolean buildWeeklyTask(String name, String domain, Date period) { + throw new RuntimeException(getID() + " don't support weekly update"); + } + + public abstract String getID(); + +} diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/highload/TransactionHighLoadUpdater.java b/cat-home/src/main/java/com/dianping/cat/report/task/highload/TransactionHighLoadUpdater.java new file mode 100644 index 000000000..8a32ff30b --- /dev/null +++ b/cat-home/src/main/java/com/dianping/cat/report/task/highload/TransactionHighLoadUpdater.java @@ -0,0 +1,7 @@ +package com.dianping.cat.report.task.highload; + +public abstract class TransactionHighLoadUpdater extends HighLoadUpdater { + + public abstract String getType(); + +} diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/overload/CapacityUpdateTask.java b/cat-home/src/main/java/com/dianping/cat/report/task/overload/CapacityUpdateTask.java index a88bbdbf2..95b848e33 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/task/overload/CapacityUpdateTask.java +++ b/cat-home/src/main/java/com/dianping/cat/report/task/overload/CapacityUpdateTask.java @@ -13,8 +13,6 @@ import com.dianping.cat.report.task.spi.ReportTaskBuilder; public class CapacityUpdateTask implements ReportTaskBuilder, LogEnabled { - - public static final String ID = Constants.REPORT_DATABASE_CAPACITY; @Inject(type = CapacityUpdater.class, value = HourlyCapacityUpdater.ID) @@ -81,19 +79,19 @@ public class CapacityUpdateTask implements ReportTaskBuilder, LogEnabled { } public void setHourlyUpdater(CapacityUpdater hourlyUpdater) { - m_hourlyUpdater = hourlyUpdater; - } + m_hourlyUpdater = hourlyUpdater; + } public void setDailyUpdater(CapacityUpdater dailyUpdater) { - m_dailyUpdater = dailyUpdater; - } + m_dailyUpdater = dailyUpdater; + } public void setWeeklyUpdater(CapacityUpdater weeklyUpdater) { - m_weeklyUpdater = weeklyUpdater; - } + m_weeklyUpdater = weeklyUpdater; + } public void setMonthlyUpdater(CapacityUpdater monthlyUpdater) { - m_monthlyUpdater = monthlyUpdater; - } - + m_monthlyUpdater = monthlyUpdater; + } + } diff --git a/cat-home/src/main/resources/META-INF/plexus/components.xml b/cat-home/src/main/resources/META-INF/plexus/components.xml index 4c272f877..02260cade 100755 --- a/cat-home/src/main/resources/META-INF/plexus/components.xml +++ b/cat-home/src/main/resources/META-INF/plexus/components.xml @@ -1439,6 +1439,20 @@ + + com.dianping.cat.report.task.spi.ReportTaskBuilder + highLoadSql + com.dianping.cat.report.task.highload.HighLoadSqlUpdater + + + com.dianping.cat.report.page.model.spi.ModelService + transaction + + + com.dianping.cat.report.service.ReportServiceManager + + + com.dianping.cat.report.task.notify.ReportRender com.dianping.cat.report.task.notify.ReportRenderImpl -- GitLab