diff --git a/cat-home/src/main/java/com/dianping/cat/report/ReportPage.java b/cat-home/src/main/java/com/dianping/cat/report/ReportPage.java index 75ca174db828ac112f7893c3fd2413adfcb33580..a756aeb0c2f8412a556b35b44e53ab70c5c36f97 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/ReportPage.java +++ b/cat-home/src/main/java/com/dianping/cat/report/ReportPage.java @@ -9,25 +9,15 @@ public enum ReportPage implements Page { PROBLEM("problem", "p", "Problem", "Problem discovered", true), -<<<<<<< HEAD LOGVIEW("logview", "m", "Logview", "CAT log details view for a given message.", false), -======= + TRANSACTION("transaction", "t", "Transaction", "Transaction summary report", true), - LOGVIEW("logview", "m", "Logview", "Log view details", false), ->>>>>>> 1ce8aecf691c885c4773c6c52388dad926ad9557 - IP("ip", "ip", "Top IP List", "Top visited IP addresses", true), -<<<<<<< HEAD MODEL("model", "model", "Model", "Model Service", false), - - PROBLEM("problem", "p", "Problem", "Problem", true); -======= - MODEL("model", "model", "Model", "Service model", false), - + SQL("sql", "sql", "Sql", "Sql", true); ->>>>>>> 1ce8aecf691c885c4773c6c52388dad926ad9557 private String m_name; diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Action.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Action.java index 684b016fadf9ed915c191b885111ecc72dce838b..9d45f6732b0c5ad4abb4de32c0ddcf7c789719c5 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Action.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Action.java @@ -1,7 +1,9 @@ package com.dianping.cat.report.page.sql; public enum Action implements com.site.web.mvc.Action { - VIEW("view"); + VIEW("view"), + + GRAPHS("graphs"); private String m_name; diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Handler.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Handler.java index f4209b8e302004d1ac89675c90a7750a092bccb4..d00767bcfca75f037c26e48698c0fe0bddca0b1d 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Handler.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Handler.java @@ -11,6 +11,7 @@ import com.dianping.cat.job.sql.dal.SqlReportRecord; import com.dianping.cat.job.sql.dal.SqlReportRecordDao; import com.dianping.cat.job.sql.dal.SqlReportRecordEntity; import com.dianping.cat.report.ReportPage; +import com.dianping.cat.report.graph.GraphBuilder; import com.site.dal.jdbc.DalException; import com.site.dal.jdbc.Readset; import com.site.lookup.annotation.Inject; @@ -25,6 +26,9 @@ public class Handler implements PageHandler { @Inject private SqlReportRecordDao m_dao; + + @Inject + private GraphBuilder m_builder; @Override @PayloadMeta(Payload.class) @@ -37,11 +41,49 @@ public class Handler implements PageHandler { @OutboundActionMeta(name = "sql") public void handleOutbound(Context ctx) throws ServletException, IOException { Model model = new Model(ctx); - model.setAction(Action.VIEW); model.setPage(ReportPage.SQL); - Payload payload = ctx.getPayload(); - SqlReport report = new SqlReport(); + + Action action = payload.getAction(); + if(action==null||action==Action.VIEW){ + model.setAction(Action.VIEW); + showReport(model, payload); + }else{ + model.setAction(Action.GRAPHS); + showGraphs(model, payload); + } + m_jspViewer.view(ctx, model); + } + + public void showGraphs(Model model,Payload payload){ + int id = payload.getId(); + try { + SqlReportRecord record= m_dao.findByPK(id, SqlReportRecordEntity.READSET_FULL); + String statement = record.getStatement(); + String durationDistribution =record.getDurationdistribution(); + String durationOvertime =record.getDurationovertime(); + String hitsovOvrtime =record.getHitsovertime(); + String failureOvertime =record.getFailureovertime(); + + String graph1 = m_builder.build(new SqlGraphPayload(0,"SQL Exeture Time Distribution", "Duration (ms)", "Count", durationDistribution)); + String graph2 = m_builder.build(new SqlGraphPayload(1,"SQL Hits Over One Hour", "Time (min)", "Count", hitsovOvrtime)); + String graph3 = m_builder.build(new SqlGraphPayload(2,"SQL Exeture Average Time Over One Hour", "Time (min)", + "Average Duration (ms)", durationOvertime)); + String graph4 = m_builder.build(new SqlGraphPayload(3,"SQL Failures Over One Hour", "Time (min)", "Count", failureOvertime)); + + model.setGraph1(graph1); + model.setGraph2(graph2); + model.setGraph3(graph3); + model.setGraph4(graph4); + model.setStatement(statement); + } catch (DalException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public void showReport(Model model, Payload payload) { + SqlReport report = new SqlReport(); String domain = payload.getDomain(); long startDate = payload.getDate(); model.setDate(startDate); @@ -82,7 +124,5 @@ public class Handler implements PageHandler { } catch (DalException e) { e.printStackTrace(); } - - m_jspViewer.view(ctx, model); - } + } } diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspFile.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspFile.java index c72856d7387177972f70dbe6bbd7e623e9b4599e..817381be156e128e0cdc5e7ac3ab9d9160e72c45 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspFile.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspFile.java @@ -3,6 +3,7 @@ package com.dianping.cat.report.page.sql; public enum JspFile { VIEW("/jsp/report/sql.jsp"), + GRAPHS("/jsp/report/sql_graphs.jsp"), ; private String m_path; diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspViewer.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspViewer.java index 2a0b2df698231f3b1b15efc20341e4fc56d28827..fd114d04854456391cd039ba58785977574495c6 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspViewer.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/JspViewer.java @@ -11,6 +11,8 @@ public class JspViewer extends BaseJspViewer switch (action) { case VIEW: return JspFile.VIEW.getPath(); + case GRAPHS: + return JspFile.GRAPHS.getPath(); } throw new RuntimeException("Unknown action: " + action); diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Model.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Model.java index 6e4284e00e0d169e746ba0ec45090b8e8d326ecd..1b534189eb86242805f1bbec578b833e6ee971de 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Model.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Model.java @@ -7,6 +7,16 @@ import com.dianping.cat.report.page.AbstractReportModel; public class Model extends AbstractReportModel { private SqlReport m_report; + + private String m_graph1; + + private String m_graph2; + + private String m_graph3; + + private String m_graph4; + + private String m_statement; public Model(Context ctx) { super(ctx); @@ -41,5 +51,45 @@ public class Model extends AbstractReportModel { } else { return m_report.getDomains(); } + } + + public String getGraph1() { + return m_graph1; + } + + public void setGraph1(String graph1) { + m_graph1 = graph1; + } + + public String getGraph2() { + return m_graph2; + } + + public void setGraph2(String graph2) { + m_graph2 = graph2; + } + + public String getGraph3() { + return m_graph3; + } + + public void setGraph3(String graph3) { + m_graph3 = graph3; + } + + public String getGraph4() { + return m_graph4; + } + + public void setGraph4(String graph4) { + m_graph4 = graph4; + } + + public String getStatement() { + return m_statement; + } + + public void setStatement(String statement) { + m_statement = statement; } } diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Payload.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Payload.java index f662d5f8a1b41ca7424f96f65333b4d339087aca..593cbdc751df91f55f02d20c1ae9daeeadaa4a96 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/Payload.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/Payload.java @@ -2,38 +2,36 @@ package com.dianping.cat.report.page.sql; import com.dianping.cat.report.ReportPage; import com.dianping.cat.report.page.AbstractReportPayload; -import com.dianping.cat.report.page.transaction.Action; import com.site.web.mvc.ActionContext; import com.site.web.mvc.payload.annotation.FieldMeta; public class Payload extends AbstractReportPayload { - private ReportPage m_page; - @FieldMeta("op") private Action m_action; + @FieldMeta("id") + private int id; + public Payload() { super(ReportPage.SQL); } - public void setAction(Action action) { - m_action = action; + public int getId() { + return id; } - @Override - public Action getAction() { - return m_action; + public void setId(int id) { + this.id = id; } - @Override - public ReportPage getPage() { - return m_page; + public void setAction(String action) { + m_action = Action.getByName(action, Action.VIEW); } @Override - public void setPage(String page) { - m_page = ReportPage.getByName(page, ReportPage.SQL); + public Action getAction() { + return m_action; } @Override diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlGraphPayload.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlGraphPayload.java new file mode 100644 index 0000000000000000000000000000000000000000..d48ecef43447c6d26ffca0e89bc00114fb5b5073 --- /dev/null +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlGraphPayload.java @@ -0,0 +1,95 @@ +package com.dianping.cat.report.page.sql; + +import java.util.List; + +import org.apache.commons.lang.math.NumberUtils; + +import com.dianping.cat.report.graph.AbstractGraphPayload; +import com.site.helper.Splitters; + +public class SqlGraphPayload extends AbstractGraphPayload { + private String[] m_labels; + + private double[] m_values; + + private int m_size; + + private int m_index; + + public SqlGraphPayload(int index, String title, String axisXLabel, String axisYLabel, String metaData) { + super(title, axisXLabel, axisYLabel); + + m_index = index; + + List data = Splitters.by(",").noEmptyItem().split(metaData); + if (data != null) { + m_size = data.size(); + m_labels = new String[m_size]; + m_values = new double[m_size - 1]; + + for (int i = 0; i < m_size; i++) { + String temp = data.get(i); + String[] s = temp.split(":"); + if (i == m_size - 1) { + m_labels[i] = s[0]; + } else { + m_labels[i] = s[0]; + m_values[i] = NumberUtils.toDouble(s[1], 0d); + } + } + + } else { + m_labels = new String[1]; + m_values = new double[1]; + } + } + + @Override + public String getAxisXLabel(int index) { + return m_labels[index]; + } + + @Override + public int getOffsetX() { + if (m_index % 2 == 1) { + return getDisplayWidth(); + } else { + return 0; + } + } + + @Override + public int getOffsetY() { + if (m_index / 2 == 1) { + return getDisplayHeight() + 20; + } else { + return 0; + } + } + + @Override + protected double[] loadValues() { + return m_values; + } + + @Override + public int getDisplayHeight() { + return (int) (super.getDisplayHeight() * 0.7); + } + + @Override + public int getDisplayWidth() { + return (int) (super.getDisplayWidth() * 0.7); + } + + @Override + public int getWidth() { + return super.getWidth() + 120; + } + + @Override + public boolean isStandalone() { + return false; + } + +} diff --git a/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlReportModel.java b/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlReportModel.java index 792be5df2ba65d2766ec8545ed8b7b5141f06c8c..3168e7bef24098d3af926935f7040747a25f3bf0 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlReportModel.java +++ b/cat-home/src/main/java/com/dianping/cat/report/page/sql/SqlReportModel.java @@ -1,41 +1,32 @@ package com.dianping.cat.report.page.sql; -import java.text.DecimalFormat; - import com.dianping.cat.job.sql.dal.SqlReportRecord; public class SqlReportModel { private SqlReportRecord m_record; - private DecimalFormat percent = new DecimalFormat("#.##%"); - - private DecimalFormat number = new DecimalFormat("#.##"); public SqlReportModel(SqlReportRecord record) { m_record = record; } - public String getFailurePercent() { - double value = m_record.getFailures() / m_record.getTotalcount(); - return percent.format(value); + public double getFailurePercent() { + return (double)m_record.getFailures() / (double)m_record.getTotalcount(); } - public String getLongPercent() { - double value = m_record.getLongsqls() / m_record.getTotalcount(); - return percent.format(value); + public double getLongPercent() { + return (double)m_record.getLongsqls() / (double)m_record.getTotalcount(); } - public String getAvg() { - double value = m_record.getSumvalue() / m_record.getTotalcount(); - return number.format(value); + public double getAvg() { + return (double)m_record.getSumvalue() / (double)m_record.getTotalcount(); } - public String getStd() { + public double getStd() { double sum2 = m_record.getSum2value(); int count = m_record.getTotalcount(); double avg = m_record.getSumvalue() / m_record.getTotalcount(); - double std = Math.sqrt(sum2 / count - avg * avg); - return number.format(std); + return Math.sqrt(sum2 / count - avg * avg); } public SqlReportRecord getRecord() { diff --git a/cat-home/src/main/java/com/dianping/cat/report/view/NavigationBar.java b/cat-home/src/main/java/com/dianping/cat/report/view/NavigationBar.java index 415d489a481146f9e97738d6422470bda75caffc..045ba9ed2b57d35155cdf6679239a6aa8f61b37a 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/view/NavigationBar.java +++ b/cat-home/src/main/java/com/dianping/cat/report/view/NavigationBar.java @@ -9,22 +9,14 @@ public class NavigationBar { ReportPage.HOME, -<<<<<<< HEAD -======= - //ReportPage.FAILURE, - ->>>>>>> 1ce8aecf691c885c4773c6c52388dad926ad9557 ReportPage.TRANSACTION, ReportPage.PROBLEM, ReportPage.IP, -<<<<<<< HEAD -======= ReportPage.SQL, //ReportPage.SERVICE, ->>>>>>> 1ce8aecf691c885c4773c6c52388dad926ad9557 ReportPage.LOGVIEW diff --git a/cat-home/src/main/resources/META-INF/plexus/components.xml b/cat-home/src/main/resources/META-INF/plexus/components.xml index a6e5d544ab3c45422f587637ba336c409efae475..dd9f43755b8262c42ae6e497a95c94fd0b52203f 100644 --- a/cat-home/src/main/resources/META-INF/plexus/components.xml +++ b/cat-home/src/main/resources/META-INF/plexus/components.xml @@ -214,8 +214,6 @@ com.dianping.cat.report.page.problem.JspViewer -<<<<<<< HEAD -======= com.dianping.cat.report.page.transaction.Handler com.dianping.cat.report.page.transaction.Handler @@ -237,7 +235,6 @@ com.dianping.cat.report.page.transaction.JspViewer ->>>>>>> 1ce8aecf691c885c4773c6c52388dad926ad9557 com.dianping.cat.report.page.logview.Handler com.dianping.cat.report.page.logview.Handler @@ -352,6 +349,7 @@ com.dianping.cat.message.spi.MessageCodec + html @@ -365,6 +363,9 @@ com.dianping.cat.job.sql.dal.SqlReportRecordDao + + com.dianping.cat.report.graph.GraphBuilder + diff --git a/cat-home/src/main/webapp/WEB-INF/tags/report.tag b/cat-home/src/main/webapp/WEB-INF/tags/report.tag index 08b7ca2d2ca99de6383a146f40342aeaf30506ae..dfe93792eb04780147610eedd0d6caebf1319015 100644 --- a/cat-home/src/main/webapp/WEB-INF/tags/report.tag +++ b/cat-home/src/main/webapp/WEB-INF/tags/report.tag @@ -26,10 +26,10 @@   - [ ${domain} ] + [ ${domain} ] - [ ${domain} ] + [ ${domain} ]   diff --git a/cat-home/src/main/webapp/js/sql.js b/cat-home/src/main/webapp/js/sql.js new file mode 100644 index 0000000000000000000000000000000000000000..52aa406327b0e6895eae999604f21da760b71222 --- /dev/null +++ b/cat-home/src/main/webapp/js/sql.js @@ -0,0 +1,39 @@ +var isCtrl = false; + +function showGraphs(anchor,target, id) { + if (isCtrl) return true; + + var cell = document.getElementById(target); + var text = anchor.innerHTML; + + if (text == '[:: show ::]') { + anchor.innerHTML = '[:: hide ::]'; + + if (cell.nodeName == 'IMG') { // + cell.src = "?op=graphs&id="+id; + } else { //
...
+ $.ajax({ + type: "get", + url: "?op=graphs&id="+id, + success : function(data, textStatus) { + cell.innerHTML = data; + } + }); + } + + cell.style.display = 'block'; + cell.parentNode.style.display = 'block'; + } else { + anchor.innerHTML = '[:: show ::]'; + cell.style.display = 'none'; + cell.parentNode.style.display = 'none'; + } + + return false; +} + +$(document).keydown(function(e) { + if(e.ctrlKey || e.metaKey) isCtrl = true; +}).keyup(function(e) { + isCtrl = false; +}); \ No newline at end of file diff --git a/cat-home/src/main/webapp/jsp/report/problem.jsp b/cat-home/src/main/webapp/jsp/report/problem.jsp index c545e96228901db39a8dfc8d15dee50ecb21a7ce..5084f089b0028894c54216b93cda65fad0a77587 100644 --- a/cat-home/src/main/webapp/jsp/report/problem.jsp +++ b/cat-home/src/main/webapp/jsp/report/problem.jsp @@ -9,16 +9,15 @@ - - - - From ${w:format(report.startTime,'yyyy-MM-dd HH:mm:ss')} to ${w:format(report.endTime,'yyyy-MM-dd HH:mm:ss')} + + + diff --git a/cat-home/src/main/webapp/jsp/report/sql.jsp b/cat-home/src/main/webapp/jsp/report/sql.jsp index b05541f39d98b50aacf7acdbb2feb060b7e28086..e166d529f0924dda323d04473355c3f2bcad6f31 100644 --- a/cat-home/src/main/webapp/jsp/report/sql.jsp +++ b/cat-home/src/main/webapp/jsp/report/sql.jsp @@ -24,29 +24,34 @@
- + + + + - - - + - - - - - - - - + + + + + + + + + +
SQLTotalExec Counts Failure Failure%Min / Max / Avg / Std(ms)95% LineDB Time LongSQL Long%Min/Max/Avg/Std(ms)95% AvgSample LinkSample
${reportRecord.record.name}${reportRecord.record.totalcount}${reportRecord.record.failures}${reportRecord.failurePercent}${reportRecord.record.longsqls}${reportRecord.longPercent}${reportRecord.record.minvalue}/${reportRecord.record.maxvalue}/${reportRecord.avg}/${reportRecord.std}${reportRecord.avg}[:: show ::] ${reportRecord.record.name}${w:format(reportRecord.record.totalcount,'0.00')}${w:format(reportRecord.record.failures,'0.00')}${w:format(reportRecord.failurePercent,'0.00%')}${w:format(reportRecord.record.minvalue,'0.00')} / ${w:format(reportRecord.record.maxvalue,'0.00')} / ${w:format(reportRecord.avg,'0.00')} / ${w:format(reportRecord.std,'0.00')}${w:format(reportRecord.record.avg2value,'0.00')}${w:format(reportRecord.record.sumvalue,'0.00')}${w:format(reportRecord.record.longsqls,'0.00')}${w:format(reportRecord.longPercent,'0.00%')} Link

+ + diff --git a/cat-home/src/main/webapp/jsp/report/sql_graph.jsp b/cat-home/src/main/webapp/jsp/report/sql_graph.jsp new file mode 100644 index 0000000000000000000000000000000000000000..e59882026313d0956f5fbf6e96ef9d58565418fb --- /dev/null +++ b/cat-home/src/main/webapp/jsp/report/sql_graph.jsp @@ -0,0 +1,46 @@ +<%@ page contentType="image/svg+xml; charset=utf-8"%> + + + Duration + + + + + + + + + 200 + 160 + 120 + 80 + 40 + 0 + + + 0 + 2 + 4 + 6 + 8 + 10 + 12 + 14 + 16 + 18 + 20 + 22 + + + Bottom + Title + + + + 60 + + 30 + + 50 + + \ No newline at end of file diff --git a/cat-home/src/main/webapp/jsp/report/sql_graphs.jsp b/cat-home/src/main/webapp/jsp/report/sql_graphs.jsp new file mode 100644 index 0000000000000000000000000000000000000000..01083ab0bb44a9afbc4004e1057b2c4f2d324332 --- /dev/null +++ b/cat-home/src/main/webapp/jsp/report/sql_graphs.jsp @@ -0,0 +1,10 @@ +<%@ page contentType="text/html; charset=utf-8"%> + + +SQL Statement: ${model.statement} + + ${model.graph1} + ${model.graph2} + ${model.graph3} + ${model.graph4} + diff --git a/cat-job/src/main/java/com/dianping/cat/job/job/BrowserAnalyzer.java b/cat-job/src/main/java/com/dianping/cat/job/job/BrowserAnalyzer.java index 69f3aa9344fc017a28a7eb1c33cf8b99337bdf28..50a39dc577f127b55f3f13e715fa420073f623cb 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/job/BrowserAnalyzer.java +++ b/cat-job/src/main/java/com/dianping/cat/job/job/BrowserAnalyzer.java @@ -40,7 +40,7 @@ public class BrowserAnalyzer extends Configured implements Tool { job.setInputFormatClass(MessageTreeInputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); - FileInputFormat.addInputPath(job, new Path("target/hdfs/20120215/17/null")); + FileInputFormat.addInputPath(job, new Path("target/hdfs/20120312/22/null/")); FileOutputFormat.setOutputPath(job, new Path("target/browser")); Files.forDir().delete(new File("target/browser"), true); diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMain.java b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMain.java index 4ca93a2f4a66b9debc9483a00f7772bdea92615a..0f98a64dddfe5465ea3bffbb8a85372bbe78acfd 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMain.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMain.java @@ -52,17 +52,18 @@ public class SqlJobMain extends Configured implements Tool { @Override public int run(String[] args) throws Exception { Configuration conf = getConf(); - Job job = new Job(conf, "Sql Analyzer"); + job.setJarByClass(SqlJobMain.class); job.setMapperClass(SqlJobMapper.class); job.setReducerClass(SqlJobReducer.class); job.setInputFormatClass(MessageTreeInputFormat.class); job.setOutputKeyClass(SqlStatementKey.class); job.setOutputValueClass(SqlJobResult.class); - job.setPartitionerClass(SqlJobPatitioner.class); job.setMapOutputKeyClass(SqlStatementKey.class); job.setMapOutputValueClass(SqlStatementValue.class); + + job.setPartitionerClass(SqlJobPatitioner.class); job.setNumReduceTasks(DEFAULT_REDUCE_NUMBER); if (args.length > 0) { @@ -103,6 +104,7 @@ public class SqlJobMain extends Configured implements Tool { * insert the result to mysql */ private int runSqlRecordJob(String currentHour) throws Exception { + System.out.println("Insert database job start!"); Configuration conf = getConf(); conf.set("JobHour", currentHour); Job job = new Job(conf, "Sql Record"); diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMapper.java b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMapper.java index 28b8469198ea199111c26f0454d6961768ad6c13..f96d07b2d3d5b51b32bf24cc7f15bcf1ac864e5a 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMapper.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobMapper.java @@ -49,7 +49,10 @@ public class SqlJobMapper extends Mapper values, Context context) throws IOException, InterruptedException { SqlJobResult result = new SqlJobResult(); - for (SqlStatementValue val : values) { - result.add(val.getValue(), val.getFlag(), val.getSampleUrl()); + result.add(val.getValue(), val.getFlag(), val.getSampleUrl().toString() ,val.getMinute()); } context.write(key, result); } diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobResult.java b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobResult.java index cc13a6a054039e582b180f5d0ab502fd167f889b..ece679a87039f812ed37a870b2b9dc5bbecbe859 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobResult.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlJobResult.java @@ -6,7 +6,10 @@ import java.io.IOException; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import org.apache.hadoop.io.Writable; @@ -32,24 +35,47 @@ public class SqlJobResult implements Writable { private List m_urls = new ArrayList(); + private Map m_durationDistribution = new LinkedHashMap(); + + private Map m_hitsOverTime = new LinkedHashMap(); + + private Map m_durationOverTime = new LinkedHashMap(); + + private Map m_durationOverTimeSum = new LinkedHashMap(); + + private Map m_failureOverTime = new LinkedHashMap(); + private DecimalFormat df = new DecimalFormat("#.##"); - public void add(double value, int flag, String url ) { - m_sum += value; - m_sum2 += value * value; + public SqlJobResult() { + for (int i = 0; i <= 60; i = i + 5) { + m_hitsOverTime.put(i, 0); + m_durationOverTime.put(i, 0.0); + m_failureOverTime.put(i, 0); + } + + m_durationDistribution.put(0, 0); + for (int i = 1; i <= 65536; i = i * 2) { + m_durationDistribution.put(i, 0); + } + } + + public void add(double duration, int flag, String url, int minute) { + m_sum += duration; + m_sum2 += duration * duration; if (flag == 1) { m_failureCount++; } - if (value > LONG_TIME) { + if (duration > LONG_TIME) { m_longTimeCount++; } - if (value < m_min) { - m_min = value; + if (duration < m_min) { + m_min = duration; } - if (value > m_max) { - m_max = value; + if (duration > m_max) { + m_max = duration; } - m_durations.add(value); + m_durations.add(duration); int size = m_urls.size(); if (size == 0) { @@ -57,6 +83,24 @@ public class SqlJobResult implements Writable { } else if (size == 1 && flag == 1) { m_urls.add(url); } + int minuteKey = minute - minute % 5; + m_hitsOverTime.put(minuteKey, m_hitsOverTime.get(minuteKey) + 1); + + m_durationOverTime.put(minuteKey, m_durationOverTime.get(minuteKey) + 1); + + if (flag == 1) { + m_failureOverTime.put(minuteKey, m_failureOverTime.get(minuteKey) + 1); + } + int durationKey = getDuration(duration); + m_durationDistribution.put(durationKey, m_durationDistribution.get(durationKey) + 1); + } + + public int getDuration(double duration) { + int min = 0; + while (duration > Math.pow(2, min + 1)) { + min++; + } + return (int) Math.pow(2, min); } public double getAvg() { @@ -84,10 +128,46 @@ public class SqlJobResult implements Writable { .append(SPIT); sb.append(df.format(m_min)).append(SPIT).append(df.format(m_max)).append(SPIT).append(df.format(m_sum)) .append(SPIT).append(df.format(m_sum2)).append(SPIT).append(df.format(getAvg())).append(SPIT); - + int size = m_urls.size(); - - sb.append(m_urls.get(size-1)); + + sb.append(m_urls.get(size - 1)).append(SPIT); + sb.append(mapToString(m_durationDistribution)).append(SPIT); + sb.append(mapToString(m_hitsOverTime)).append(SPIT); + for (Entry entry : m_durationOverTimeSum.entrySet()) { + Integer key = entry.getKey(); + double value = 0; + int count = m_hitsOverTime.get(key); + if (count > 0) { + value = m_durationOverTimeSum.get(key) / count; + } + m_durationOverTime.put(key, value); + } + + sb.append(map2String(m_durationOverTime)).append(SPIT); + sb.append(mapToString(m_failureOverTime)).append(SPIT); + return sb.toString(); + } + + private String map2String(Map map) { + StringBuilder sb = new StringBuilder(); + if (map != null) { + for (Entry entry : map.entrySet()) { + sb.append(entry.getKey().toString()).append(":").append(entry.getValue().toString()).append(","); + } + sb.deleteCharAt(sb.length() - 1); + } + return sb.toString(); + } + + private String mapToString(Map map) { + StringBuilder sb = new StringBuilder(); + if (map != null) { + for (Entry entry : map.entrySet()) { + sb.append(entry.getKey().toString()).append(":").append(entry.getValue().toString()).append(","); + } + sb.deleteCharAt(sb.length() - 1); + } return sb.toString(); } diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlStatementValue.java b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlStatementValue.java index 8c43c3d8b3e5b5702c044f26de6e1a0a5d70b9d0..4a3cec64d92e68f658da182a526c9b8ef99557cb 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/SqlStatementValue.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/SqlStatementValue.java @@ -13,6 +13,8 @@ public class SqlStatementValue implements Writable { public double m_value; + public int m_minute; + public Text m_sampleUrl; public SqlStatementValue(){ @@ -26,14 +28,19 @@ public class SqlStatementValue implements Writable { public double getValue() { return m_value; } - - public String getSampleUrl(){ - return m_sampleUrl.toString(); - } - public SqlStatementValue(int flag, double value ,String url) { + public int getMinute() { + return m_minute; + } + + public Text getSampleUrl() { + return m_sampleUrl; + } + + public SqlStatementValue(int flag, double value ,String url ,int minute) { m_flag = flag; m_value = value; + m_minute = minute; m_sampleUrl=new Text(url); } @@ -42,6 +49,7 @@ public class SqlStatementValue implements Writable { m_flag = input.readInt(); m_value = input.readDouble(); m_sampleUrl.readFields(input); + m_minute = input.readInt(); } @Override @@ -49,5 +57,6 @@ public class SqlStatementValue implements Writable { output.writeInt(m_flag); output.writeDouble(m_value); m_sampleUrl.write(output); + output.writeInt(m_minute); } } diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlRecordJobReducer.java b/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlRecordJobReducer.java index c6623af86a3083e3b9b624793e759cf5b620b79c..451e7843797e9a00d4d607bdf17dec10dfb67740 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlRecordJobReducer.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlRecordJobReducer.java @@ -11,34 +11,37 @@ import com.dianping.cat.job.sql.dal.SqlReportRecord; import com.dianping.cat.job.sql.dal.SqlReportRecordDao; import com.site.dal.jdbc.DalException; -public class SqlRecordJobReducer extends Reducer{ - public void reduce(Text key, Iterable values, Context context) throws IOException, - InterruptedException { - Text currentHour = values.iterator().next(); - SqlReportJobRecord sql = new SqlReportJobRecord(currentHour.toString(),key.toString()); - +public class SqlRecordJobReducer extends Reducer { + public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { + Text currentHour = values.iterator().next(); + SqlReportJobRecord sql = new SqlReportJobRecord(currentHour.toString(), key.toString()); + try { - SqlReportRecordDao dao = ContainerBootstrap.INSTANCE.lookup(SqlReportRecordDao.class); - SqlReportRecord row = dao.createLocal(); - row.setDomain(sql.getDomain()); - row.setTotalcount(sql.getTotalCount()); - row.setFailures(sql.getFailureCount()); - row.setLongsqls(sql.getLongCount()); - row.setAvg2value(sql.getAvg2()); - row.setSumvalue(sql.getSum()); - row.setSum2value(sql.getSum2()); - row.setMaxvalue(sql.getMax()); - row.setMinvalue(sql.getMin()); - row.setStatement(sql.getStatement()); - row.setName(sql.getName()); - row.setSamplelink(sql.getSampleLink()); - row.setTransactiondate(sql.getDate()); - row.setCreationdate(new Date()); - dao.insert(row); - } catch (ComponentLookupException e) { - e.printStackTrace(); - } catch (DalException e) { - e.printStackTrace(); - } + SqlReportRecordDao dao = ContainerBootstrap.INSTANCE.lookup(SqlReportRecordDao.class); + SqlReportRecord row = dao.createLocal(); + row.setDomain(sql.getDomain()); + row.setTotalcount(sql.getTotalCount()); + row.setFailures(sql.getFailureCount()); + row.setLongsqls(sql.getLongCount()); + row.setAvg2value(sql.getAvg2()); + row.setSumvalue(sql.getSum()); + row.setSum2value(sql.getSum2()); + row.setMaxvalue(sql.getMax()); + row.setMinvalue(sql.getMin()); + row.setStatement(sql.getStatement()); + row.setName(sql.getName()); + row.setSamplelink(sql.getSampleLink()); + row.setTransactiondate(sql.getDate()); + row.setCreationdate(new Date()); + row.setDurationdistribution(sql.getDurationDistribution()); + row.setHitsovertime(sql.getHitsOverTime()); + row.setDurationovertime(sql.getDurationOverTime()); + row.setFailureovertime(sql.getFailureOverTime()); + dao.insert(row); + } catch (ComponentLookupException e) { + e.printStackTrace(); + } catch (DalException e) { + e.printStackTrace(); + } } -} +} diff --git a/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlReportJobRecord.java b/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlReportJobRecord.java index ed9625c972c51c2eca0edeb3e9d03b7286baf8a9..bb3ad64cbea468036e6dfb13eb90673273b6a82c 100644 --- a/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlReportJobRecord.java +++ b/cat-job/src/main/java/com/dianping/cat/job/sql/database/SqlReportJobRecord.java @@ -7,7 +7,7 @@ import java.util.Date; public class SqlReportJobRecord { private String m_domain; - + private String m_name; private String m_statement; @@ -37,16 +37,24 @@ public class SqlReportJobRecord { private Date m_creationDate; + private String m_durationDistribution; + + private String m_hitsOverTime; + + private String m_durationOverTime; + + private String m_failureOverTime; + private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - + private SimpleDateFormat hourFormat = new SimpleDateFormat("yyyyMMdd/HH"); private static final String SPIT = "\t"; - public SqlReportJobRecord(){ - + public SqlReportJobRecord() { + } - + public String toString() { StringBuilder sb = new StringBuilder(); @@ -68,15 +76,15 @@ public class SqlReportJobRecord { } // domain1 SQLStatement Internal9 500 500 500 100 199 74750 11591750 147 - public SqlReportJobRecord(String currentHour,String text) { - + public SqlReportJobRecord(String currentHour, String text) { + try { - m_transactionDate = hourFormat.parse(currentHour); - } catch (ParseException e) { - Date error = new Date(); - error.setTime(0); - m_transactionDate =error ; - } + m_transactionDate = hourFormat.parse(currentHour); + } catch (ParseException e) { + Date error = new Date(); + error.setTime(0); + m_transactionDate = error; + } m_creationDate = new Date(); String[] params = text.split("\t"); @@ -92,16 +100,19 @@ public class SqlReportJobRecord { m_sum2 = Double.parseDouble(params[9]); m_avg2 = Double.parseDouble(params[10]); m_sampleLink = params[11]; + m_durationDistribution = params[12]; + m_hitsOverTime = params[13]; + m_durationOverTime = params[14]; + m_failureOverTime = params[15]; } - public String getName() { - return m_name; - } + return m_name; + } public void setName(String name) { - m_name = name; - } + m_name = name; + } public String getDomain() { return m_domain; @@ -207,4 +218,36 @@ public class SqlReportJobRecord { m_creationDate = creatTime; } + public Date getTransactionDate() { + return m_transactionDate; + } + + public Date getCreationDate() { + return m_creationDate; + } + + public String getDurationDistribution() { + return m_durationDistribution; + } + + public String getHitsOverTime() { + return m_hitsOverTime; + } + + public String getDurationOverTime() { + return m_durationOverTime; + } + + public String getFailureOverTime() { + return m_failureOverTime; + } + + public SimpleDateFormat getSdf() { + return sdf; + } + + public SimpleDateFormat getHourFormat() { + return hourFormat; + } + } diff --git a/cat-job/src/main/resources/META-INF/dal/jdbc/codegen.xml b/cat-job/src/main/resources/META-INF/dal/jdbc/codegen.xml index b28ad917484b9e36538350d4c81165719825a698..92aadc62204e31cdac837e6fea2fa48208c43571 100644 --- a/cat-job/src/main/resources/META-INF/dal/jdbc/codegen.xml +++ b/cat-job/src/main/resources/META-INF/dal/jdbc/codegen.xml @@ -1,6 +1,6 @@ - + @@ -16,6 +16,10 @@ + + + + diff --git a/cat-job/src/main/resources/META-INF/dal/jdbc/dal.xml b/cat-job/src/main/resources/META-INF/dal/jdbc/dal.xml index 95887707d673363206e25b481caa05b9d3902fce..8d9132c9a767a523baff9cd053926330e0080f3e 100644 --- a/cat-job/src/main/resources/META-INF/dal/jdbc/dal.xml +++ b/cat-job/src/main/resources/META-INF/dal/jdbc/dal.xml @@ -2,22 +2,18 @@ - - - - - + + + + + + diff --git a/cat-job/src/test/java/com/dianping/cat/job/sql/SqlJobDataProduceTest.java b/cat-job/src/test/java/com/dianping/cat/job/sql/SqlJobDataProduceTest.java index a000e4ccef2f938a324a1c31503e7f3b80efbef2..4df9511a346d6280c5f85a181ad49088080b5243 100644 --- a/cat-job/src/test/java/com/dianping/cat/job/sql/SqlJobDataProduceTest.java +++ b/cat-job/src/test/java/com/dianping/cat/job/sql/SqlJobDataProduceTest.java @@ -22,8 +22,10 @@ public class SqlJobDataProduceTest extends CatTestCase { MessageStorage storage = lookup(MessageStorage.class, "hdfs"); MessageProducer producer = lookup(MessageProducer.class); InMemoryQueue queue = lookup(InMemoryQueue.class); + + long currentHour = System.currentTimeMillis() - System.currentTimeMillis() / (60 * 60 * 1000); for (int i = 0; i < 3; i++) { - for (int j = 0; j < 10000; j++) { + for (int j = 0; j < 1200; j++) { Transaction t = producer.newTransaction("URL", "MyPage" + (int) (j / 500)); try { @@ -39,7 +41,7 @@ public class SqlJobDataProduceTest extends CatTestCase { producer.logEvent("URL", "Payload", Message.SUCCESS, "host=my-host&ip=127.0.0.1&agent=..."); producer.logEvent("URL", "Payload", Message.SUCCESS, "host=my-host&ip=127.0.0.1&agent=..."); - String sqlName = "SQLStatement" + j / 500; + String sqlName = "Project.insert" + j / 500; String sqlParaMeter = "SQLParaMeter" + j / 500; String sqlStatement = "select * from table where id=\"1\"\n order by id desc"; Transaction sqlTran = producer.newTransaction("SQL", sqlName); @@ -48,34 +50,20 @@ public class SqlJobDataProduceTest extends CatTestCase { Stringizers.forJson().compact().from(sqlParaMeter)); sqlTran.addData(sqlStatement); - String sqlInternalName = "SQLStatement Internal" + j / 500; - String sqlParaInternal = "SQLParaMeter Internal" + j / 500; - String sqlInternal = "select * from intenal table where id=\"1\"\n order by id desc"; - Transaction internal = producer.newTransaction("SQL", sqlInternalName); - - producer.logEvent("SQL.PARAM", sqlParaInternal, Transaction.SUCCESS, Stringizers.forJson().compact() - .from(sqlParaInternal)); - internal.addData(sqlInternal); - internal.complete(); - - if (j % 2 == 1) { - internal.setStatus(Message.SUCCESS); - } else { - internal.setStatus("Error"); - } - sqlTran.complete(); - DefaultTransaction sqlInternalTran = (DefaultTransaction) internal; - sqlInternalTran.setDuration(j % 100 + 100); + DefaultTransaction sqlInternalTran = (DefaultTransaction) sqlTran; + sqlInternalTran.setDuration((long)Math.pow(2, j % 12)); if (j % 2 == 1) { sqlTran.setStatus(Message.SUCCESS); } else { sqlTran.setStatus("Error"); } + sqlInternalTran.setTimestamp(currentHour + (j % 60) * 1000 * 60); DefaultTransaction def = (DefaultTransaction) sqlTran; def.setDuration(j % 100 + 50); + def.setTimestamp(currentHour + (j % 60) * 1000 * 60); t.setStatus(Message.SUCCESS); } catch (Exception e) { t.setStatus(e);