提交 3690430d 编写于 作者: Y youyong205

refator broswer report

上级 4eb7233e
......@@ -75,7 +75,7 @@ public class HttpABTestEntityRepository extends ContainerHolder implements ABTes
public void initialize() throws InitializationException {
m_domain = m_configManager.getDomain().getId();
ScriptEngineManager mgr = new ScriptEngineManager();
m_engine = mgr.getEngineByExtension("java");
}
......@@ -114,6 +114,12 @@ public class HttpABTestEntityRepository extends ContainerHolder implements ABTes
public void run() {
while (true) {
long start = System.currentTimeMillis();
try {
// waiting for server start
Thread.sleep(20000);
} catch (InterruptedException e) {
break;
}
try {
refresh();
......
......@@ -61,7 +61,7 @@
${basedir}/src/main/resources/META-INF/dal/model/metric-report-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/dependency-report-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/metric-config-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/browser-report-manifest.xml,</manifest>
${basedir}/src/main/resources/META-INF/dal/model/browser-meta-report-manifest.xml,</manifest>
</configuration>
</execution>
<execution>
......
......@@ -5,42 +5,22 @@ import java.util.List;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.analysis.AbstractMessageAnalyzer;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browser.model.entity.UserAgent;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.message.Message;
import com.dianping.cat.message.Transaction;
import com.dianping.cat.message.spi.MessageTree;
import com.dianping.cat.service.DefaultReportManager.StoragePolicy;
import com.dianping.cat.service.ReportManager;
public class BrowserAnalyzer extends AbstractMessageAnalyzer<BrowserReport> {
public static final String ID = "browser";
public class BrowserMetaAnalyzer extends AbstractMessageAnalyzer<BrowserMetaReport> {
public static final String ID = "browserMeta";
@Inject(ID)
private ReportManager<BrowserReport> m_reportManager;
@Inject
private UserAgentManager m_userAgentManager;
// public void removeMapItem(Map<String, Integer> map) {
//
// m_userAgents.put("domain", value );
//
// Map.Entry<String, Integer> itemRemoved = null;
//
// for (Map.Entry<String, Integer> item : map.entrySet()) {
// if (itemRemoved == null) {
// itemRemoved = item;
// } else if (item.getValue() < itemRemoved.getValue()) {
// itemRemoved = item;
// }
// }
// map.remove(itemRemoved.getKey());
// }
protected ReportManager<BrowserMetaReport> m_reportManager;
@Override
public void doCheckpoint(boolean atEnd) {
// m_userAgentManager.storeResult();
if (atEnd && !isLocalMode()) {
m_reportManager.storeHourlyReports(getStartTime(), StoragePolicy.FILE_AND_DB);
} else {
......@@ -49,14 +29,14 @@ public class BrowserAnalyzer extends AbstractMessageAnalyzer<BrowserReport> {
}
@Override
public BrowserReport getReport(String domain) {
BrowserReport report = m_reportManager.getHourlyReport(getStartTime(), domain, false);
public BrowserMetaReport getReport(String domain) {
BrowserMetaReport report = m_reportManager.getHourlyReport(getStartTime(), domain, false);
report.getDomainNames().addAll(m_reportManager.getDomains(getStartTime()));
return report;
}
protected String parseValue(final String key, final String data) {
private String parseValue(final String key, final String data) {
int len = data == null ? 0 : data.length();
int keyLen = key.length();
StringBuilder name = new StringBuilder();
......@@ -111,54 +91,19 @@ public class BrowserAnalyzer extends AbstractMessageAnalyzer<BrowserReport> {
for (Message child : children) {
String childType = child.getType();
String childName = child.getName();
BrowserReport report = m_reportManager.getHourlyReport(getStartTime(), "Cat", true);
if ("URL".equals(childType) && ("URL.Server".equals(childName) || "ClientInfo".equals(childName))) {
String data = (String) child.getData();
String agent = parseValue("agent", data);
String domain = tree.getDomain();
updateBrowserReport(report, domain, data);
BrowserMetaReport report = m_reportManager.getHourlyReport(getStartTime(), domain, true);
report.findOrCreateUserAgent(agent).incCount();
return;
}
}
}
}
}
private void updateBrowserReport(BrowserReport report, String domain, String data) {
UserAgent userAgent = report.findOrCreateDomainDetail(domain).findOrCreateUserAgent(data);
int count = 0;
if(userAgent.getCount()!=null){
count = userAgent.getCount();
}
userAgent.setCount(count + 1);
}
// private void updateBrowserReport(BrowserReport report, String domain,
// String data) {
// String agent = parseValue("Agent", data);
//
// if (agent == null || agent.isEmpty()) {
// m_logger.error("Can not get agent from url when browser analyze: "
// + data);
// }
//
// UserAgent userAgent = m_userAgentManager.parse(agent);
// String browserName = userAgent.getBrowser();
// String osName = userAgent.getOs();
// String versionName = userAgent.getVersion();
// DomainDetail detail = report.findOrCreateDomainDetail(domain);
//
// Browser browser = detail.findOrCreateBrowser(browserName);
// BrowserVersion version = browser
// .findOrCreateBrowserVersion(versionName);
// Os os = detail.findOrCreateOs(osName);
//
// browser.setCount(browser.getCount() + 1);
// os.setCount(os.getCount() + 1);
// version.setCount(version.getCount() + 1);
// }
}
......@@ -2,56 +2,49 @@ package com.dianping.cat.consumer.browser;
import java.util.Date;
import java.util.Map;
import java.util.Set;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browser.model.transform.DefaultNativeBuilder;
import com.dianping.cat.consumer.browser.model.transform.DefaultNativeParser;
import com.dianping.cat.consumer.browser.model.transform.DefaultSaxParser;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.consumer.browsermeta.model.transform.DefaultNativeBuilder;
import com.dianping.cat.consumer.browsermeta.model.transform.DefaultNativeParser;
import com.dianping.cat.consumer.browsermeta.model.transform.DefaultSaxParser;
import com.dianping.cat.service.ReportDelegate;
import com.dianping.cat.task.TaskManager;
import com.dianping.cat.task.TaskManager.TaskProlicy;
public class BrowserDelegate implements ReportDelegate<BrowserReport> {
public class BrowserMetaDelegate implements ReportDelegate<BrowserMetaReport> {
@Inject
private TaskManager m_taskManager;
@Override
public void afterLoad(Map<String, BrowserReport> reports) {
public void afterLoad(Map<String, BrowserMetaReport> reports) {
}
@Override
public void beforeSave(Map<String, BrowserReport> reports) {
for (BrowserReport report : reports.values()) {
Set<String> domainNames = report.getDomainNames();
domainNames.clear();
domainNames.addAll(reports.keySet());
}
public void beforeSave(Map<String, BrowserMetaReport> reports) {
}
@Override
public String buildXml(BrowserReport report) {
public String buildXml(BrowserMetaReport report) {
return report.toString();
}
@Override
public boolean createHourlyTask(BrowserReport report) {
return m_taskManager.createTask(report.getStartTime(), report.getDomain(), BrowserAnalyzer.ID,
TaskProlicy.ALL_EXCLUED_HOURLY);
public boolean createHourlyTask(BrowserMetaReport report) {
return m_taskManager.createTask(report.getStartTime(), report.getDomain(), BrowserMetaAnalyzer.ID,
TaskProlicy.HOULY);
}
@Override
public String getDomain(BrowserReport report) {
public String getDomain(BrowserMetaReport report) {
return report.getDomain();
}
@Override
public BrowserReport makeReport(String domain, long startTime, long duration) {
BrowserReport report = new BrowserReport(domain);
public BrowserMetaReport makeReport(String domain, long startTime, long duration) {
BrowserMetaReport report = new BrowserMetaReport(domain);
report.setStartTime(new Date(startTime));
report.setEndTime(new Date(startTime + duration - 1));
......@@ -60,27 +53,27 @@ public class BrowserDelegate implements ReportDelegate<BrowserReport> {
}
@Override
public BrowserReport mergeReport(BrowserReport old, BrowserReport other) {
BrowserReportMerger merger = new BrowserReportMerger(old);
public BrowserMetaReport mergeReport(BrowserMetaReport old, BrowserMetaReport other) {
BrowserMetaReportMerger merger = new BrowserMetaReportMerger(old);
other.accept(merger);
return old;
}
@Override
public BrowserReport parseXml(String xml) throws Exception {
BrowserReport report = DefaultSaxParser.parse(xml);
public BrowserMetaReport parseXml(String xml) throws Exception {
BrowserMetaReport report = DefaultSaxParser.parse(xml);
return report;
}
@Override
public byte[] buildBinary(BrowserReport report) {
public byte[] buildBinary(BrowserMetaReport report) {
return DefaultNativeBuilder.build(report);
}
@Override
public BrowserReport parseBinary(byte[] bytes) {
public BrowserMetaReport parseBinary(byte[] bytes) {
return DefaultNativeParser.parse(bytes);
}
}
package com.dianping.cat.consumer.browser;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.consumer.browsermeta.model.entity.UserAgent;
import com.dianping.cat.consumer.browsermeta.model.transform.DefaultMerger;
public class BrowserMetaReportMerger extends DefaultMerger {
public BrowserMetaReportMerger(BrowserMetaReport metaReport) {
super(metaReport);
}
@Override
protected void mergeBrowserMetaReport(BrowserMetaReport to, BrowserMetaReport from) {
super.mergeBrowserMetaReport(to, from);
}
@Override
protected void mergeUserAgent(UserAgent to, UserAgent from) {
to.setCount(to.getCount()+from.getCount());
}
}
package com.dianping.cat.consumer.browser;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.codehaus.plexus.logging.LogEnabled;
import org.codehaus.plexus.logging.Logger;
import org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException;
import org.unidal.dal.jdbc.DalException;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.Cat;
import com.dianping.cat.consumer.advanced.dal.UserAgent;
import com.dianping.cat.consumer.advanced.dal.UserAgentDao;
import com.dianping.cat.consumer.advanced.dal.UserAgentEntity;
public class DefaultUserAgentManager implements UserAgentManager, LogEnabled {
private Map<String, UserAgent> m_userAgents = new LinkedHashMap<String, UserAgent>(){
private static final long serialVersionUID = 1L;
private int MAX_SIZE = 100000;
@Override
protected boolean removeEldestEntry(Entry<String, UserAgent> eldest) {
return size() > MAX_SIZE;
}
};
@Inject
private UserAgentDao m_userAgentDao;
private Logger m_logger;
@Override
public void initialize() throws InitializationException {
try {
List<UserAgent> userAgents = m_userAgentDao.findAll(UserAgentEntity.READSET_FULL);
for (UserAgent userAgent : userAgents) {
m_userAgents.put(userAgent.getBrowser(), userAgent);
}
} catch (DalException e) {
Cat.logError(e);
}
}
@Override
public UserAgent parse(String userAgentString) {
if (m_userAgents.containsKey(userAgentString)) {
return m_userAgents.get(userAgentString);
} else {
UserAgentParser uap = null;
try {
uap = new UserAgentParser(userAgentString);
} catch (UserAgentParseException e) {
m_logger.warn("Can not parse user agent: " + userAgentString, e);
}
UserAgent result = m_userAgentDao.createLocal();
if (uap != null) {
result.setId(0).setBrowser(uap.getBrowserName()).setVersion(uap.getBrowserVersion())
.setOs(uap.getBrowserOperatingSystem()).setUserAgent(uap.getUserAgentString());
}
m_userAgents.put(userAgentString, result);
return result;
}
}
@Override
public void storeResult() {
for (UserAgent agent : m_userAgents.values()) {
if (agent.getId() == 0) {
try {
m_userAgentDao.insert(agent);
} catch (DalException e) {
m_logger.warn(e.getMessage(), e);
}
}
}
}
@Override
public void enableLogging(Logger logger) {
m_logger = logger;
}
}
......@@ -15,11 +15,8 @@ import com.dianping.cat.analysis.MessageAnalyzer;
import com.dianping.cat.consumer.CatConsumerAdvancedModule;
import com.dianping.cat.consumer.advanced.dal.BusinessReportDao;
import com.dianping.cat.consumer.advanced.dal.SqltableDao;
import com.dianping.cat.consumer.advanced.dal.UserAgentDao;
import com.dianping.cat.consumer.browser.BrowserAnalyzer;
import com.dianping.cat.consumer.browser.BrowserDelegate;
import com.dianping.cat.consumer.browser.DefaultUserAgentManager;
import com.dianping.cat.consumer.browser.UserAgentManager;
import com.dianping.cat.consumer.browser.BrowserMetaAnalyzer;
import com.dianping.cat.consumer.browser.BrowserMetaDelegate;
import com.dianping.cat.consumer.cross.CrossAnalyzer;
import com.dianping.cat.consumer.cross.CrossDelegate;
import com.dianping.cat.consumer.cross.IpConvertManager;
......@@ -122,17 +119,15 @@ public class ComponentsConfigurator extends AbstractResourceConfigurator {
private Collection<Component> defineBrowserComponents() {
final List<Component> all = new ArrayList<Component>();
final String ID = BrowserAnalyzer.ID;
final String ID = BrowserMetaAnalyzer.ID;
all.add(C(MessageAnalyzer.class, ID, BrowserAnalyzer.class).is(PER_LOOKUP) //
.req(ReportManager.class, ID).req(UserAgentManager.class));
all.add(C(UserAgentManager.class,DefaultUserAgentManager.class).req(UserAgentDao.class));
all.add(C(MessageAnalyzer.class, ID, BrowserMetaAnalyzer.class).is(PER_LOOKUP) //
.req(ReportManager.class, ID));
all.add(C(ReportManager.class, ID, DefaultReportManager.class) //
.req(ReportDelegate.class, ID) //
.req(BucketManager.class, HourlyReportDao.class, HourlyReportContentDao.class) //
.config(E("name").value(ID)));
all.add(C(ReportDelegate.class, ID, BrowserDelegate.class).req(TaskManager.class));
all.add(C(ReportDelegate.class, ID, BrowserMetaDelegate.class).req(TaskManager.class));
return all;
}
......
<?xml version="1.0" encoding="UTF-8"?>
<model>
<entity name="browser-meta-report" root="true">
<attribute name="domain" value-type="String" />
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<entity-ref name="user-agent" type="list" names="user-agents" />
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" />
<attribute name="count" value-type="int" />
</entity>
</model>
<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<file path="browser-meta-report-codegen.xml" />
<file path="browser-meta-report-model.xml" />
</manifest>
<?xml version="1.0" encoding="UTF-8"?>
<model model-package="com.dianping.cat.consumer.browsermeta.model" enable-merger="true" enable-sax-parser="true"
enable-native-parser="true" enable-native-builder="true">
<entity name="browser-meta-report" root="true">
<attribute name="domain" value-type="String" key="true"/>
<element name="domain" value-type="String" type="set" names="domain-names" />
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<entity-ref name="user-agent" type="map" names="user-agents" method-find-or-create="true" />
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" key="true"/>
<attribute name="count" method-inc="true" primitive="true" />
</entity>
</model>
......@@ -9,7 +9,7 @@
<model package="com.dianping.cat.advanced.metric-config" name="metric-config">
<sample-model>src/test/resources/com/dianping/cat/consumer/model/metric-config.xml</sample-model>
</model>
<model package="com.dianping.cat.advanced.browser-report" name="browser-report">
<sample-model>src/test/resources/com/dianping/cat/consumer/model/browser.xml</sample-model>
<model package="com.dianping.cat.advanced.browser-meta-report" name="browser-meta-report">
<sample-model>src/test/resources/com/dianping/cat/consumer/model/browser-meta.xml</sample-model>
</model>
</wizard>
package com.dianping.cat.consumer.browser;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.unidal.helper.Files;
import org.unidal.lookup.ComponentTestCase;
import com.dianping.cat.Constants;
import com.dianping.cat.analysis.MessageAnalyzer;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.message.Message;
import com.dianping.cat.message.internal.DefaultEvent;
import com.dianping.cat.message.internal.DefaultTransaction;
import com.dianping.cat.message.spi.MessageTree;
import com.dianping.cat.message.spi.internal.DefaultMessageTree;
public class BrowserAnalyzerTest extends ComponentTestCase {
private long m_timestamp;
private BrowserAnalyzer m_analyzer;
private String m_domain = "group";
@Test
public void testParseValue() {
BrowserAnalyzer browserAnalyzer = new BrowserAnalyzer();
String data = "RemoteIP=10.1.1.109&VirtualIP=10.1.1.109&Server=cat.dianpingoa.com&Referer=http://cat.dianpingoa.com/cat/r/p?date=2013111217&ip=All&step=-1&op=view&domain=MBookingWebShop&ip=All&urlThreshold=1000&sqlThreshold=100&serviceThreshold=50&cacheThreshold=10&callThreshold=50&Agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36";
String agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36";
Assert.assertEquals(agent, browserAnalyzer.parseValue("Agent", data));
Assert.assertNull(browserAnalyzer.parseValue("UnknownKey", data));
}
@Before
public void setUp() throws Exception {
super.setUp();
TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai"));
long currentTimeMillis = System.currentTimeMillis();
m_timestamp = currentTimeMillis - currentTimeMillis % (3600 * 1000);
m_analyzer = (BrowserAnalyzer) lookup(MessageAnalyzer.class, BrowserAnalyzer.ID);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HH:mm");
Date date = sdf.parse("20120101 00:00");
m_analyzer.initialize(date.getTime(), Constants.HOUR, Constants.MINUTE * 5);
}
@Test
public void testProcess() throws Exception {
for (int i = 1; i <= 100; i++) {
MessageTree tree = generateMessageTree(i);
m_analyzer.process(tree);
}
BrowserReport report = m_analyzer.getReport("Cat");
String expected = Files.forIO().readFrom(getClass().getResourceAsStream("browser_analyzer.xml"), "utf-8");
Assert.assertEquals(expected.replaceAll("\r", ""), report.toString().replaceAll("\r", ""));
}
protected MessageTree generateMessageTree(int i) {
MessageTree tree = new DefaultMessageTree();
tree.setMessageId("" + i);
tree.setDomain(m_domain);
tree.setHostName("group001");
tree.setIpAddress("192.168.1.1");
DefaultTransaction t;
if (i % 2 == 0) {
t = new DefaultTransaction("URL", "Cat-Test-Call", null);
DefaultEvent event = new DefaultEvent("URL", "URL.Server");
event.addData("RemoteIP=221.226.186.58&VirtualIP=127.0.0.1&Server=t.dianping.com&Referer=http://youku.com.gambol.pw/abc.flv&Agent=Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36");
event.setTimestamp(m_timestamp + 5 * 60 * 1000);
event.setStatus(Message.SUCCESS);
t.addChild(event);
} else {
t = new DefaultTransaction("URL", "Cat-Test-Service", null);
DefaultEvent event = new DefaultEvent("URL", "ClientInfo");
event.addData("RemoteIP=111.172.157.237&VirtualIP=127.0.0.1&Server=t.dianping.com&Referer=http://t.dianping.com/wuhan?utm_source=sogou_tglogo&Agent=Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.802.30 Safari/535.1 SE 2.X MetaSr 1.0");
event.setTimestamp(m_timestamp + 5 * 60 * 1000);
event.setStatus(Message.SUCCESS);
t.addChild(event);
}
t.complete();
t.setDurationInMillis(i * 2);
t.setTimestamp(m_timestamp + 1000);
tree.setMessage(t);
return tree;
}
//
// @Test
// public void testRemoveMapItem() {
// Map<String, Integer> map = new HashMap<String, Integer>();
// map.put("test1", 1);
// map.put("test2", 2);
// m_analyzer.removeMapItem(map);
// }
}
package com.dianping.cat.consumer.browser;
import java.util.ArrayList;
import java.util.List;
import org.unidal.lookup.annotation.Inject;
import org.unidal.lookup.configuration.AbstractResourceConfigurator;
import org.unidal.lookup.configuration.Component;
import com.dianping.cat.Constants;
import com.dianping.cat.consumer.MockReportManager;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.service.ReportDelegate;
import com.dianping.cat.service.ReportManager;
public class Configurator extends AbstractResourceConfigurator {
public static void main(String[] args) {
generatePlexusComponentsXmlFile(new Configurator());
}
protected Class<?> getTestClass() {
return BrowserAnalyzerTest.class;
}
@Override
public List<Component> defineComponents() {
List<Component> all = new ArrayList<Component>();
final String ID = BrowserAnalyzer.ID;
all.add(C(ReportManager.class, ID, MockBrowserReportManager.class)//
.req(ReportDelegate.class, ID));
all.add(C(ReportDelegate.class, ID, ExtendedBrowserDelegate.class));
return all;
}
public static class ExtendedBrowserDelegate extends BrowserDelegate {
}
public static class MockBrowserReportManager extends MockReportManager<BrowserReport> {
private BrowserReport m_report;
@Inject
private ReportDelegate<BrowserReport> m_delegate;
@Override
public BrowserReport getHourlyReport(long startTime, String domain, boolean createIfNotExist) {
if (m_report == null) {
m_report = (BrowserReport) m_delegate.makeReport(domain, startTime, Constants.HOUR);
}
return m_report;
}
}
}
package com.dianping.cat.consumer.performance;
import static org.junit.Assert.assertEquals;
import java.util.Date;
import org.junit.Test;
import org.unidal.lookup.ComponentTestCase;
import com.dianping.cat.Constants;
import com.dianping.cat.analysis.MessageAnalyzer;
import com.dianping.cat.consumer.browser.BrowserAnalyzer;
import com.dianping.cat.consumer.browser.UserAgentParser;
import com.dianping.cat.message.Message;
import com.dianping.cat.message.internal.MockMessageBuilder;
import com.dianping.cat.message.spi.MessageTree;
import com.dianping.cat.message.spi.internal.DefaultMessageTree;
public class BrowserPerformanceTest extends ComponentTestCase {
public void testStandardIE6() {
String header = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; .NET CLR 1.1.4322)";
UserAgentParser uap = new UserAgentParser(header);
assertEquals("Windows NT 5.0", uap.getBrowserOperatingSystem());
assertEquals("MSIE", uap.getBrowserName());
assertEquals("6.0", uap.getBrowserVersion());
}
// @Test
// public void performanceTest(){
// long current = System.currentTimeMillis();
//
// int size = 10000000;
// for (int i = 0; i < size; i++) {
// testStandardIE6();
// }
// System.out.println("Cost " + (System.currentTimeMillis() - current) / 1000);
// }
// //Cost 74
@Test
public void test() throws Exception {
BrowserAnalyzer analyzer = (BrowserAnalyzer) lookup(MessageAnalyzer.class, BrowserAnalyzer.ID);
MessageTree tree = buildMessage();
long current = System.currentTimeMillis();
analyzer.initialize(new Date().getTime(), Constants.HOUR, Constants.MINUTE * 5);
int size = 100000;
for (int i = 0; i < size; i++) {
analyzer.process(tree);
}
System.out.println(analyzer.getReport("Cat"));
System.out.println("Cost " + (System.currentTimeMillis() - current) / 1000);
// cost 26
}
public MessageTree buildMessage() {
Message message = new MockMessageBuilder() {
@Override
public MessageHolder define() {
TransactionHolder t = t("URL", "/redirect", 112819)
.child(
e("URL",
"URL.Server",
"RemoteIP=10.1.1.109&VirtualIP=10.1.1.109&Server=cat.dianpingoa.com&Referer=http://cat.dianpingoa.com/cat/r/p?date=2013111217&ip=All&step=-1&op=view&domain=MBookingWebShop&ip=All&urlThreshold=1000&sqlThreshold=100&serviceThreshold=50&cacheThreshold=10&callThreshold=50&Agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"))
.child(
t("PigeonCall",
"groupService:groupNoteService_1.0.1:updateNoteDraft1(Integer,Integer,String,String)",
"", 100).child(e("PigeonCall.server", "10.1.2.199:2011", "Execute[34796272]")))
.child(
t("PigeonCall",
"groupService:groupNoteService_1.0.1:updateNoteDraft2(Integer,Integer,String,String)",
"", 100).child(e("PigeonCall.server", "10.1.2.199:2011", "Execute[34796272]")))
.child(
t("PigeonCall",
"groupService:groupNoteService_1.0.1:updateNoteDraft3(Integer,Integer,String,String)",
"", 100).child(e("PigeonCall.server", "10.1.2.199:2011", "Execute[34796272]")))
.child(
t("PigeonCall",
"groupService:groupNoteService_1.0.1:updateNoteDraft4(Integer,Integer,String,String)",
"", 100).child(e("PigeonCall.server", "10.1.2.199:2011", "Execute[34796272]")))
.child(
t("PigeonService",
"groupService:groupNoteService_1.0.1:updateNoteDraft5(Integer,Integer,String,String)",
"", 100).child(e("PigeonService.client", "10.1.7.127:37897", "Execute[34796272]")))
.child(
t("PigeonService",
"groupService:groupNoteService_1.0.1:updateNoteDraft7(Integer,Integer,String,String)",
"", 100).child(e("PigeonService.client", "10.1.7.127:37897", "Execute[34796272]")))
.child(
t("PigeonService",
"groupService:groupNoteService_1.0.1:updateNoteD1aft6(Integer,Integer,String,String)",
"", 100).child(e("PigeonService1.client", "10.1.7.128:37897", "Execute[34796272]")));
return t;
}
}.build();
MessageTree tree = new DefaultMessageTree();
tree.setDomain("Cat");
tree.setHostName("test");
tree.setIpAddress("test");
tree.setThreadGroupName("test");
tree.setThreadId("test");
tree.setThreadName("test");
tree.setMessage(message);
tree.setMessageId("MobileApi-0a01077f-379304-1362256");
return tree;
}
}
<?xml version="1.0" encoding="utf-8"?>
<browser-report domain="Cat" startTime="2012-01-01 00:00:00" endTime="2012-01-01 00:59:59">
<domain-detail id="group">
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" count="100"/>
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1650.63 Safari/537.36" count="99"/>
</domain-detail>
</browser-report>
<browser-meta-report domain="Cat" startTime="2012-01-01 00:00:00" endTime="2012-01-01 00:59:59">
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" count="100"/>
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1650.63 Safari/537.36" count="99"/>
</browser-meta-report>
<browser-meta-report domain="Cat" startTime="2012-01-01 00:00:00" endTime="2012-01-01 00:59:59">
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" count="100"/>
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1650.63 Safari/537.36" count="99"/>
</browser-meta-report>
<browser-report domain="Cat" startTime="2012-01-01 00:00:00" endTime="2012-01-01 00:59:59">
<domain>group</domain>
<domain-detail id="group">
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" count="100"/>
<user-agent id="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1650.63 Safari/537.36" count="99"/>
</domain-detail>
</browser-report>
......@@ -23,7 +23,7 @@
<query-defs>
<query name="insert" type="INSERT">
<statement><![CDATA[
INSERT IGNORE INTO <TABLE/>
INSERT INTO <TABLE/>
(<FIELDS/>)
VALUES
(<VALUES/>)
......@@ -295,7 +295,7 @@
<query-defs>
<query name="insert" type="INSERT">
<statement><![CDATA[
INSERT IGNORE INTO <TABLE/>
INSERT INTO <TABLE/>
(<FIELDS/>)
VALUES
(<VALUES/>)
......@@ -329,7 +329,7 @@
<query-defs>
<query name="insert" type="INSERT">
<statement><![CDATA[
INSERT IGNORE INTO <TABLE/>
INSERT INTO <TABLE/>
(<FIELDS/>)
VALUES
(<VALUES/>)
......@@ -439,7 +439,7 @@
<query-defs>
<query name="insert" type="INSERT">
<statement><![CDATA[
INSERT IGNORE INTO <TABLE/>
INSERT INTO <TABLE/>
(<FIELDS/>)
VALUES
(<VALUES/>)
......
......@@ -159,7 +159,8 @@
${basedir}/src/main/resources/META-INF/dal/model/heavy-report-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/utilization-report-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/utilization-config-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/abtest-report-manifest.xml,</manifest>
${basedir}/src/main/resources/META-INF/dal/model/abtest-report-manifest.xml,
${basedir}/src/main/resources/META-INF/dal/model/browser-report-manifest.xml,</manifest>
</configuration>
</execution>
<execution>
......
......@@ -16,6 +16,7 @@ import com.dianping.cat.home.dal.report.DailyReportContentDao;
import com.dianping.cat.home.dal.report.MonthlyReportContentDao;
import com.dianping.cat.home.dal.report.WeeklyReportContentDao;
import com.dianping.cat.report.service.ReportService;
import com.dianping.cat.report.service.impl.BrowserMetaReportService;
import com.dianping.cat.report.service.impl.BrowserReportService;
import com.dianping.cat.report.service.impl.BugReportService;
import com.dianping.cat.report.service.impl.CrossReportService;
......@@ -77,6 +78,7 @@ public class ReportServiceComponentConfigurator extends AbstractResourceConfigur
MonthlyReportDao.class, HourlyReportContentDao.class, DailyReportContentDao.class,
WeeklyReportContentDao.class, MonthlyReportContentDao.class));
all.add(C(BrowserMetaReportService.class).req(HourlyReportDao.class, HourlyReportContentDao.class));
all.add(C(TopReportService.class).req(HourlyReportDao.class, HourlyReportContentDao.class));
all.add(C(DependencyReportService.class).req(HourlyReportDao.class, HourlyReportContentDao.class));
all.add(C(HeartbeatReportService.class).req(HourlyReportDao.class, HourlyReportContentDao.class));
......@@ -91,7 +93,7 @@ public class ReportServiceComponentConfigurator extends AbstractResourceConfigur
.req(TopReportService.class, StateReportService.class, CrossReportService.class)
.req(HeartbeatReportService.class, MetricReportService.class, BugReportService.class)
.req(HeavyReportService.class, ServiceReportService.class, UtilizationReportService.class)
.req(BrowserReportService.class));
.req(BrowserReportService.class, BrowserMetaReportService.class));
return all;
}
......
......@@ -28,7 +28,7 @@ import com.dianping.cat.report.page.transaction.TransactionMergeManager;
import com.dianping.cat.report.service.ReportService;
import com.dianping.cat.report.task.DefaultTaskConsumer;
import com.dianping.cat.report.task.abtest.ABTestReportBuilder;
import com.dianping.cat.report.task.browser.BrowserReportBuilder;
import com.dianping.cat.report.task.browser.BrowserMetaReportBuilder;
import com.dianping.cat.report.task.bug.BugReportBuilder;
import com.dianping.cat.report.task.cross.CrossReportBuilder;
import com.dianping.cat.report.task.dependency.DependencyReportBuilder;
......@@ -120,7 +120,7 @@ public class TaskComponentConfigurator extends AbstractResourceConfigurator {
all.add(C(UtilizationReportBuilder.class).req(ReportService.class, TransactionMergeManager.class));
all.add(C(BrowserReportBuilder.class).req(ReportService.class));
all.add(C(BrowserMetaReportBuilder.class).req(ReportService.class));
all.add(C(DependencyReportBuilder.class).req(ReportService.class, TopologyGraphBuilder.class,
TopologyGraphDao.class));
......@@ -134,7 +134,7 @@ public class TaskComponentConfigurator extends AbstractResourceConfigurator {
SqlReportBuilder.class, StateReportBuilder.class, DependencyReportBuilder.class,
BugReportBuilder.class, ServiceReportBuilder.class, MetricBaselineReportBuilder.class,
HeavyReportBuilder.class, UtilizationReportBuilder.class, ABTestReportBuilder.class,
BrowserReportBuilder.class));
BrowserMetaReportBuilder.class));
return all;
}
......
......@@ -24,11 +24,13 @@ import org.unidal.web.mvc.annotation.PayloadMeta;
import com.dianping.cat.Cat;
import com.dianping.cat.Constants;
import com.dianping.cat.consumer.browser.BrowserMetaAnalyzer;
import com.dianping.cat.core.dal.Project;
import com.dianping.cat.core.dal.ProjectDao;
import com.dianping.cat.core.dal.ProjectEntity;
import com.dianping.cat.helper.MapUtils;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.bug.entity.BugReport;
import com.dianping.cat.home.bug.entity.Domain;
import com.dianping.cat.home.bug.entity.ExceptionItem;
......@@ -50,95 +52,6 @@ import com.dianping.cat.system.config.BugConfigManager;
import com.dianping.cat.system.config.UtilizationConfigManager;
public class Handler implements PageHandler<Context> {
public class BugReportVisitor extends BaseVisitor {
private Domain m_currentDomain;
private Map<String, ErrorStatis> m_errors = new HashMap<String, ErrorStatis>();
public ErrorStatis findOrCreateErrorStatis(String productLine) {
ErrorStatis statis = m_errors.get(productLine);
if (statis == null) {
statis = new ErrorStatis();
m_errors.put(productLine, statis);
}
return statis;
}
public Map<String, ErrorStatis> getErrors() {
return m_errors;
}
@Override
public void visitDomain(Domain domain) {
m_currentDomain = domain;
super.visitDomain(domain);
}
@Override
public void visitExceptionItem(ExceptionItem exceptionItem) {
String exception = exceptionItem.getId();
int count = exceptionItem.getCount();
Project project = findProjectByDomain(m_currentDomain.getId());
if (project != null) {
String productLine = project.getProjectLine();
String department = project.getDepartment();
ErrorStatis statis = findOrCreateErrorStatis(productLine);
statis.setDepartment(department);
statis.setProductLine(productLine);
m_currentDomain.setDepartment(department);
m_currentDomain.setProductLine(productLine);
Map<String, ExceptionItem> items = null;
if (isBug(m_currentDomain.getId(), exception)) {
items = statis.getBugs();
} else {
items = statis.getExceptions();
}
ExceptionItem item = items.get(exception);
if (item == null) {
item = new ExceptionItem(exception);
item.setCount(count);
item.getMessages().addAll(exceptionItem.getMessages());
items.put(exception, item);
} else {
List<String> messages = item.getMessages();
item.setCount(item.getCount() + count);
messages.addAll(exceptionItem.getMessages());
if (messages.size() > 10) {
messages = messages.subList(0, 10);
}
}
}
}
}
public class ClearBugReport extends BaseVisitor {
@Override
public void visitDomain(Domain domain) {
String domainName = domain.getId();
Set<String> removes = new HashSet<String>();
Map<String, ExceptionItem> items = domain.getExceptionItems();
for (ExceptionItem item : items.values()) {
if (!isBug(domainName, item.getId())) {
removes.add(item.getId());
}
}
for (String remove : removes) {
items.remove(remove);
}
}
}
@Inject
private JspViewer m_jspViewer;
......@@ -157,35 +70,6 @@ public class Handler implements PageHandler<Context> {
@Inject
private PayloadNormalizer m_normalizePayload;
// private String buildBrowserChart(Map<String, Browser> map) {
// PieChart chart = new PieChart();
// List<Item> items = new ArrayList<Item>();
//
// for (Entry<String, Browser> entry : map.entrySet()) {
// String key = entry.getKey();
// Browser value = entry.getValue();
// for (Entry<String, BrowserVersion> versionEntry : value.getBrowserVersions().entrySet()) {
// String title = key + " " + versionEntry.getKey();
// long count = versionEntry.getValue().getCount();
// items.add(new Item().setTitle(title).setNumber(count));
// }
// }
// chart.addItems(items);
// return chart.getJsonString();
// }
//
// private void buildBrowserInfo(Model model, Payload payload) {
// BrowserReport report = queryBrowserReport(payload);
// model.setBrowserReport(report);
// DomainDetail detail = report.findDomainDetail(payload.getDomain());
//
// if (detail != null) {
// model.setBrowserChart(buildBrowserChart(detail.getBrowsers()));
// model.setOsChart(buildOsChart(detail.getOses()));
// }
//
// }
private void buildBugInfo(Model model, Payload payload) {
BugReport bugReport = queryBugReport(payload);
BugReportVisitor visitor = new BugReportVisitor();
......@@ -208,20 +92,6 @@ public class Handler implements PageHandler<Context> {
buildSortedHeavyInfo(model, heavyReport);
}
// private String buildOsChart(Map<String, Os> map) {
// PieChart chart = new PieChart();
// List<Item> items = new ArrayList<Item>();
//
// for (Entry<String, Os> entry : map.entrySet()) {
// String title = entry.getKey();
// Os value = entry.getValue();
// long count = value.getCount();
// items.add(new Item().setTitle(title).setNumber(count));
// }
// chart.addItems(items);
// return chart.getJsonString();
// }
private void buildServiceInfo(Model model, Payload payload) {
ServiceReport serviceReport = queryServiceReport(payload);
List<com.dianping.cat.home.service.entity.Domain> dHisList = sort(serviceReport, payload.getSortBy());
......@@ -263,6 +133,20 @@ public class Handler implements PageHandler<Context> {
}
}
// private String buildOsChart(Map<String, Os> map) {
// PieChart chart = new PieChart();
// List<Item> items = new ArrayList<Item>();
//
// for (Entry<String, Os> entry : map.entrySet()) {
// String title = entry.getKey();
// Os value = entry.getValue();
// long count = value.getCount();
// items.add(new Item().setTitle(title).setNumber(count));
// }
// chart.addItems(items);
// return chart.getJsonString();
// }
private void buildUtilizationInfo(Model model, Payload payload) {
UtilizationReport utilizationReport = queryUtilizationReport(payload);
List<com.dianping.cat.home.utilization.entity.Domain> dUList = sort(utilizationReport, payload.getSortBy());
......@@ -346,10 +230,9 @@ public class Handler implements PageHandler<Context> {
case UTILIZATION_HISTORY_REPORT:
buildUtilizationInfo(model, payload);
break;
// case BROWSER_REPORT:
// case BROWSER_HISTORY_REPORT:
// buildBrowserInfo(model, payload);
// break;
case BROWSER_REPORT:
case BROWSER_HISTORY_REPORT:
break;
}
model.setPage(ReportPage.STATISTICS);
m_jspViewer.view(ctx, model);
......@@ -361,19 +244,19 @@ public class Handler implements PageHandler<Context> {
return !bugConfig.contains(exception);
}
// private BrowserReport queryBrowserReport(Payload payload) {
// Pair<Date, Date> pair = queryStartEndTime(payload);
// Date start = pair.getKey();
// Date end = pair.getValue();
// BrowserReport report = m_reportService.queryBrowserReport("Cat", start, end);
// Set<String> domains = m_reportService.queryAllDomainNames(start, end, BrowserAnalyzer.ID);
// Set<String> domainNames = report.getDomainNames();
//
// report.setStartTime(start);
// report.setEndTime(end);
// domainNames.addAll(domains);
// return report;
// }
protected BrowserReport queryBrowserReport(Payload payload) {
Pair<Date, Date> pair = queryStartEndTime(payload);
Date start = pair.getKey();
Date end = pair.getValue();
BrowserReport report = m_reportService.queryBrowserReport(payload.getDomain(), start, end);
Set<String> domains = m_reportService.queryAllDomainNames(start, end, BrowserMetaAnalyzer.ID);
Set<String> domainNames = report.getDomainNames();
report.setStartTime(start);
report.setEndTime(end);
domainNames.addAll(domains);
return report;
}
private BugReport queryBugReport(Payload payload) {
Pair<Date, Date> pair = queryStartEndTime(payload);
......@@ -522,5 +405,94 @@ public class Handler implements PageHandler<Context> {
temp.setExceptions(MapUtils.sortMap(exceptions, compator));
}
return errors;
}
}
public class BugReportVisitor extends BaseVisitor {
private Domain m_currentDomain;
private Map<String, ErrorStatis> m_errors = new HashMap<String, ErrorStatis>();
public ErrorStatis findOrCreateErrorStatis(String productLine) {
ErrorStatis statis = m_errors.get(productLine);
if (statis == null) {
statis = new ErrorStatis();
m_errors.put(productLine, statis);
}
return statis;
}
public Map<String, ErrorStatis> getErrors() {
return m_errors;
}
@Override
public void visitDomain(Domain domain) {
m_currentDomain = domain;
super.visitDomain(domain);
}
@Override
public void visitExceptionItem(ExceptionItem exceptionItem) {
String exception = exceptionItem.getId();
int count = exceptionItem.getCount();
Project project = findProjectByDomain(m_currentDomain.getId());
if (project != null) {
String productLine = project.getProjectLine();
String department = project.getDepartment();
ErrorStatis statis = findOrCreateErrorStatis(productLine);
statis.setDepartment(department);
statis.setProductLine(productLine);
m_currentDomain.setDepartment(department);
m_currentDomain.setProductLine(productLine);
Map<String, ExceptionItem> items = null;
if (isBug(m_currentDomain.getId(), exception)) {
items = statis.getBugs();
} else {
items = statis.getExceptions();
}
ExceptionItem item = items.get(exception);
if (item == null) {
item = new ExceptionItem(exception);
item.setCount(count);
item.getMessages().addAll(exceptionItem.getMessages());
items.put(exception, item);
} else {
List<String> messages = item.getMessages();
item.setCount(item.getCount() + count);
messages.addAll(exceptionItem.getMessages());
if (messages.size() > 10) {
messages = messages.subList(0, 10);
}
}
}
}
}
public class ClearBugReport extends BaseVisitor {
@Override
public void visitDomain(Domain domain) {
String domainName = domain.getId();
Set<String> removes = new HashSet<String>();
Map<String, ExceptionItem> items = domain.getExceptionItems();
for (ExceptionItem item : items.values()) {
if (!isBug(domainName, item.getId())) {
removes.add(item.getId());
}
}
for (String remove : removes) {
items.remove(remove);
}
}
}
}
......@@ -9,7 +9,7 @@ import java.util.Set;
import org.unidal.web.mvc.view.annotation.EntityMeta;
import com.dianping.cat.Constants;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.bug.entity.BugReport;
import com.dianping.cat.home.bug.transform.DefaultJsonBuilder;
import com.dianping.cat.home.heavy.entity.HeavyReport;
......
......@@ -3,7 +3,7 @@ package com.dianping.cat.report.service;
import java.util.Date;
import java.util.Set;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.consumer.cross.model.entity.CrossReport;
import com.dianping.cat.consumer.dependency.model.entity.DependencyReport;
import com.dianping.cat.consumer.event.model.entity.EventReport;
......@@ -19,6 +19,7 @@ import com.dianping.cat.core.dal.DailyReport;
import com.dianping.cat.core.dal.HourlyReport;
import com.dianping.cat.core.dal.MonthlyReport;
import com.dianping.cat.core.dal.WeeklyReport;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.bug.entity.BugReport;
import com.dianping.cat.home.heavy.entity.HeavyReport;
import com.dianping.cat.home.service.entity.ServiceReport;
......@@ -67,4 +68,6 @@ public interface ReportService {
public UtilizationReport queryUtilizationReport(String domain, Date start, Date end);
public BrowserReport queryBrowserReport(String domain, Date start, Date end);
public BrowserMetaReport queryBrowserMetaReport(String domain, Date start, Date end);
}
package com.dianping.cat.report.service.impl;
import java.util.Date;
import java.util.List;
import java.util.Set;
import org.unidal.dal.jdbc.DalException;
import com.dianping.cat.Cat;
import com.dianping.cat.consumer.browser.BrowserMetaAnalyzer;
import com.dianping.cat.consumer.browser.BrowserMetaReportMerger;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.consumer.browsermeta.model.transform.DefaultNativeParser;
import com.dianping.cat.core.dal.HourlyReport;
import com.dianping.cat.core.dal.HourlyReportContent;
import com.dianping.cat.core.dal.HourlyReportContentEntity;
import com.dianping.cat.core.dal.HourlyReportEntity;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.report.service.AbstractReportService;
public class BrowserMetaReportService extends AbstractReportService<BrowserMetaReport> {
@Override
public BrowserMetaReport makeReport(String domain, Date start, Date end) {
BrowserMetaReport report = new BrowserMetaReport(domain);
report.setStartTime(start);
report.setEndTime(end);
return report;
}
private BrowserMetaReport queryFromHourlyBinary(int id, String domain) throws DalException {
HourlyReportContent content = m_hourlyReportContentDao.findByPK(id, HourlyReportContentEntity.READSET_FULL);
if (content != null) {
return DefaultNativeParser.parse(content.getContent());
} else {
return new BrowserMetaReport(domain);
}
}
@Override
public BrowserMetaReport queryDailyReport(String domain, Date start, Date end) {
throw new RuntimeException("BrowserMetaReport report don't support daily report");
}
@Override
public BrowserMetaReport queryHourlyReport(String domain, Date start, Date end) {
BrowserMetaReportMerger merger = new BrowserMetaReportMerger(new BrowserMetaReport(domain));
long startTime = start.getTime();
long endTime = end.getTime();
String name = BrowserMetaAnalyzer.ID;
for (; startTime < endTime; startTime = startTime + TimeUtil.ONE_HOUR) {
List<HourlyReport> reports = null;
try {
reports = m_hourlyReportDao.findAllByDomainNamePeriod(new Date(startTime), domain, name,
HourlyReportEntity.READSET_FULL);
} catch (DalException e) {
Cat.logError(e);
}
if (reports != null) {
for (HourlyReport report : reports) {
String xml = report.getContent();
try {
if (xml != null && xml.length() > 0) {// for old xml storage
BrowserMetaReport reportModel = com.dianping.cat.consumer.browsermeta.model.transform.DefaultSaxParser
.parse(xml);
reportModel.accept(merger);
} else {// for new binary storage, binary is same to report id
BrowserMetaReport reportModel = queryFromHourlyBinary(report.getId(), domain);
reportModel.accept(merger);
}
} catch (Exception e) {
Cat.logError(e);
}
}
}
}
BrowserMetaReport metaReport = merger.getBrowserMetaReport();
metaReport.setStartTime(start);
metaReport.setEndTime(new Date(end.getTime() - 1));
Set<String> domains = queryAllDomainNames(start, end, BrowserMetaAnalyzer.ID);
metaReport.getDomainNames().addAll(domains);
return metaReport;
}
@Override
public BrowserMetaReport queryMonthlyReport(String domain, Date start) {
throw new RuntimeException("BrowserMetaReport report don't support monthly report");
}
@Override
public BrowserMetaReport queryWeeklyReport(String domain, Date start) {
throw new RuntimeException("BrowserMetaReport report don't support weekly report");
}
}
......@@ -7,10 +7,6 @@ import org.unidal.dal.jdbc.DalException;
import com.dianping.cat.Cat;
import com.dianping.cat.Constants;
import com.dianping.cat.consumer.browser.BrowserReportMerger;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browser.model.transform.DefaultNativeParser;
import com.dianping.cat.consumer.browser.model.transform.DefaultSaxParser;
import com.dianping.cat.core.dal.DailyReport;
import com.dianping.cat.core.dal.DailyReportEntity;
import com.dianping.cat.core.dal.HourlyReport;
......@@ -22,6 +18,9 @@ import com.dianping.cat.core.dal.MonthlyReportEntity;
import com.dianping.cat.core.dal.WeeklyReport;
import com.dianping.cat.core.dal.WeeklyReportEntity;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.browser.transform.DefaultNativeParser;
import com.dianping.cat.home.browser.transform.DefaultSaxParser;
import com.dianping.cat.home.dal.report.DailyReportContent;
import com.dianping.cat.home.dal.report.DailyReportContentEntity;
import com.dianping.cat.home.dal.report.MonthlyReportContent;
......@@ -29,6 +28,7 @@ import com.dianping.cat.home.dal.report.MonthlyReportContentEntity;
import com.dianping.cat.home.dal.report.WeeklyReportContent;
import com.dianping.cat.home.dal.report.WeeklyReportContentEntity;
import com.dianping.cat.report.service.AbstractReportService;
import com.dianping.cat.report.task.browser.BrowserReportMerger;
public class BrowserReportService extends AbstractReportService<BrowserReport> {
......
......@@ -7,7 +7,7 @@ import org.unidal.dal.jdbc.DalException;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.Cat;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browsermeta.model.entity.BrowserMetaReport;
import com.dianping.cat.consumer.cross.model.entity.CrossReport;
import com.dianping.cat.consumer.dependency.model.entity.DependencyReport;
import com.dianping.cat.consumer.event.model.entity.EventReport;
......@@ -31,6 +31,7 @@ import com.dianping.cat.core.dal.MonthlyReportEntity;
import com.dianping.cat.core.dal.WeeklyReport;
import com.dianping.cat.core.dal.WeeklyReportDao;
import com.dianping.cat.core.dal.WeeklyReportEntity;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.bug.entity.BugReport;
import com.dianping.cat.home.dal.report.DailyReportContent;
import com.dianping.cat.home.dal.report.DailyReportContentDao;
......@@ -117,6 +118,9 @@ public class DefaultReportService implements ReportService {
@Inject
private BrowserReportService m_browserReportService;
@Inject
private BrowserMetaReportService m_browserMetaReportService;
@Override
public boolean insertDailyReport(DailyReport report, byte[] content) {
try {
......@@ -166,7 +170,7 @@ public class DefaultReportService implements ReportService {
} catch (Exception e) {
Cat.logError(e);
}
try {
m_monthlyReportDao.deleteReportByDomainNamePeriod(report);
m_monthlyReportDao.insert(report);
......@@ -284,4 +288,9 @@ public class DefaultReportService implements ReportService {
public BrowserReport queryBrowserReport(String domain, Date start, Date end) {
return m_browserReportService.queryReport(domain, start, end);
}
@Override
public BrowserMetaReport queryBrowserMetaReport(String domain, Date start, Date end) {
return m_browserMetaReportService.queryReport(domain, start, end);
}
}
......@@ -79,7 +79,6 @@ public abstract class TaskConsumer implements org.unidal.helper.Threads.Task {
} catch (Throwable e) {
Cat.logError(e);
}
}
this.stopped = true;
}
......
......@@ -5,17 +5,17 @@ import java.util.Date;
import org.unidal.lookup.annotation.Inject;
import com.dianping.cat.configuration.NetworkInterfaceManager;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browser.model.transform.DefaultNativeBuilder;
import com.dianping.cat.core.dal.DailyReport;
import com.dianping.cat.core.dal.MonthlyReport;
import com.dianping.cat.core.dal.WeeklyReport;
import com.dianping.cat.helper.TimeUtil;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.browser.transform.DefaultNativeBuilder;
import com.dianping.cat.report.service.ReportService;
import com.dianping.cat.report.task.TaskHelper;
import com.dianping.cat.report.task.spi.ReportTaskBuilder;
public class BrowserReportBuilder implements ReportTaskBuilder {
public class BrowserMetaReportBuilder implements ReportTaskBuilder {
@Inject
protected ReportService m_reportService;
......@@ -38,6 +38,7 @@ public class BrowserReportBuilder implements ReportTaskBuilder {
@Override
public boolean buildHourlyTask(String name, String domain, Date period) {
//TODO change browsermetareport to browserreport
throw new RuntimeException("Browser report don't support HourReport!");
}
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
import com.dianping.cat.consumer.browser.model.entity.BrowserReport;
import com.dianping.cat.consumer.browser.model.entity.UserAgent;
import com.dianping.cat.consumer.browser.model.transform.DefaultMerger;
import com.dianping.cat.home.browser.entity.BrowserReport;
import com.dianping.cat.home.browser.entity.UserAgent;
import com.dianping.cat.home.browser.transform.DefaultMerger;
public class BrowserReportMerger extends DefaultMerger {
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
public class UserAgentDetail {
private String m_browserName;
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
import org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable;
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
public class UserAgentParseException extends RuntimeException {
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
import java.util.ArrayList;
import java.util.List;
......
......@@ -25,7 +25,7 @@ import com.dianping.cat.consumer.state.StateAnalyzer;
import com.dianping.cat.consumer.transaction.TransactionAnalyzer;
import com.dianping.cat.core.dal.Task;
import com.dianping.cat.report.task.abtest.ABTestReportBuilder;
import com.dianping.cat.report.task.browser.BrowserReportBuilder;
import com.dianping.cat.report.task.browser.BrowserMetaReportBuilder;
import com.dianping.cat.report.task.bug.BugReportBuilder;
import com.dianping.cat.report.task.cross.CrossReportBuilder;
import com.dianping.cat.report.task.dependency.DependencyReportBuilder;
......@@ -89,7 +89,7 @@ public class ReportFacade implements LogEnabled, Initializable {
private UtilizationReportBuilder m_utilizationReportBuilder;
@Inject
private BrowserReportBuilder m_browserReportBuilder;
private BrowserMetaReportBuilder m_browserReportBuilder;
private Logger m_logger;
......
<?xml version="1.0" encoding="UTF-8"?>
<model>
<entity name="browser-report" root="true">
<attribute name="domain" value-type="String" />
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<entity-ref name="domain-detail" />
</entity>
<entity name="domain-detail">
<attribute name="id" value-type="String" />
<entity-ref name="user-agent" type="list" names="user-agents" />
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" />
<attribute name="count" value-type="int" />
</entity>
</model>
<?xml version="1.0" encoding="UTF-8"?>
<model>
<entity name="browser-report" root="true">
<attribute name="domain" value-type="String" />
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<entity-ref name="domain-detail" />
</entity>
<entity name="domain-detail">
<attribute name="id" value-type="String" />
<entity-ref name="user-agent" type="list" names="user-agents" />
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" />
<attribute name="count" value-type="int" />
</entity>
</model>
<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<file path="browser-report-codegen.xml" />
<file path="browser-report-model.xml" />
</manifest>
<?xml version="1.0" encoding="UTF-8"?>
<manifest>
<file path="browser-report-codegen.xml" />
<file path="browser-report-model.xml" />
</manifest>
<?xml version="1.0" encoding="UTF-8"?>
<model model-package="com.dianping.cat.consumer.browser.model" enable-merger="true" enable-sax-parser="true"
enable-base-visitor="true" enable-native-parser="true" enable-native-builder="true">
<entity name="browser-report" root="true">
<attribute name="domain" value-type="String" key="true"/>
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<element name="domain" value-type="String" type="set" names="domain-names" />
<entity-ref name="domain-detail" type="map" names="domain-details" method-find-or-create="true"/>
</entity>
<entity name="domain-detail">
<attribute name="id" value-type="String" key="true" />
<entity-ref name="user-agent" type="map" names="user-agents" method-find-or-create="true"/>
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" key="true" />
<attribute name="count" value-type="int" />
</entity>
</model>
<?xml version="1.0" encoding="UTF-8"?>
<model model-package="com.dianping.cat.home.browser" enable-merger="true" enable-sax-parser="true"
enable-base-visitor="true" enable-native-parser="true" enable-native-builder="true">
<entity name="browser-report" root="true">
<attribute name="domain" value-type="String" key="true"/>
<attribute name="startTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<attribute name="endTime" value-type="Date" format="yyyy-MM-dd HH:mm:ss" />
<element name="domain" value-type="String" type="set" names="domain-names" />
<entity-ref name="domain-detail" type="map" names="domain-details" method-find-or-create="true"/>
</entity>
<entity name="domain-detail">
<attribute name="id" value-type="String" key="true" />
<entity-ref name="user-agent" type="map" names="user-agents" method-find-or-create="true"/>
</entity>
<entity name="user-agent">
<attribute name="id" value-type="String" key="true" />
<attribute name="count" value-type="int" />
</entity>
</model>
......@@ -561,6 +561,18 @@
</requirement>
</requirements>
</component>
<component>
<role>com.dianping.cat.report.service.impl.BrowserMetaReportService</role>
<implementation>com.dianping.cat.report.service.impl.BrowserMetaReportService</implementation>
<requirements>
<requirement>
<role>com.dianping.cat.core.dal.HourlyReportDao</role>
</requirement>
<requirement>
<role>com.dianping.cat.core.dal.HourlyReportContentDao</role>
</requirement>
</requirements>
</component>
<component>
<role>com.dianping.cat.report.service.impl.TopReportService</role>
<implementation>com.dianping.cat.report.service.impl.TopReportService</implementation>
......@@ -685,6 +697,9 @@
<requirement>
<role>com.dianping.cat.report.service.impl.BrowserReportService</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.service.impl.BrowserMetaReportService</role>
</requirement>
</requirements>
</component>
<component>
......@@ -970,8 +985,8 @@
</requirements>
</component>
<component>
<role>com.dianping.cat.report.task.browser.BrowserReportBuilder</role>
<implementation>com.dianping.cat.report.task.browser.BrowserReportBuilder</implementation>
<role>com.dianping.cat.report.task.browser.BrowserMetaReportBuilder</role>
<implementation>com.dianping.cat.report.task.browser.BrowserMetaReportBuilder</implementation>
<requirements>
<requirement>
<role>com.dianping.cat.report.service.ReportService</role>
......@@ -1061,7 +1076,7 @@
<role>com.dianping.cat.report.task.abtest.ABTestReportBuilder</role>
</requirement>
<requirement>
<role>com.dianping.cat.report.task.browser.BrowserReportBuilder</role>
<role>com.dianping.cat.report.task.browser.BrowserMetaReportBuilder</role>
</requirement>
</requirements>
</component>
......
package com.dianping.cat.consumer.browser;
package com.dianping.cat.report.task.browser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
......
package com.dianping.cat.report.task.event;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.junit.Test;
import org.unidal.lookup.ComponentTestCase;
import com.dianping.cat.consumer.heartbeat.HeartbeatAnalyzer;
import com.dianping.cat.consumer.problem.ProblemAnalyzer;
import com.dianping.cat.consumer.transaction.TransactionAnalyzer;
import com.dianping.cat.report.task.heartbeat.HeartbeatReportBuilder;
import com.dianping.cat.report.task.problem.ProblemReportBuilder;
import com.dianping.cat.report.task.transaction.TransactionReportBuilder;
public class BuilderTest extends ComponentTestCase {
@Test
public void test() throws ParseException{
HeartbeatReportBuilder builder = lookup(HeartbeatReportBuilder.class);
Date period = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2013-12-16 16:00:00");
builder.buildHourlyTask(HeartbeatAnalyzer.ID, "Cat", period);
}
@Test
public void testProblem() throws ParseException{
ProblemReportBuilder builder = lookup(ProblemReportBuilder.class);
Date period = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2013-12-16 16:00:00");
builder.buildHourlyTask(ProblemAnalyzer.ID, "Cat", period);
}
@Test
public void testTransaction() throws ParseException{
TransactionReportBuilder builder = lookup(TransactionReportBuilder.class);
Date period = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2013-12-16 16:00:00");
builder.buildHourlyTask(TransactionAnalyzer.ID, "Cat", period);
}
}
......@@ -399,10 +399,10 @@ CREATE TABLE `baseline` (
CREATE TABLE `user_agent` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`userAgent` varchar(200) NOT NULL DEFAULT '',
`browser` varchar(100) DEFAULT '',
`version` varchar(100) DEFAULT '',
`os` varchar(100) DEFAULT '',
`userAgent` varchar(1024) NOT NULL DEFAULT '',
`browser` varchar(200) DEFAULT '',
`version` varchar(200) DEFAULT '',
`os` varchar(200) DEFAULT '',
PRIMARY KEY (`id`),
UNIQUE KEY `userAgent` (`userAgent`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册