diff --git a/cat-core/src/main/java/com/dianping/cat/message/io/TcpSocketReceiver.java b/cat-core/src/main/java/com/dianping/cat/message/io/TcpSocketReceiver.java index 5d6ecfc7bdaeda11add3956d723df569c254c080..ddf33f91906da5f436f656941d12187a84686fca 100644 --- a/cat-core/src/main/java/com/dianping/cat/message/io/TcpSocketReceiver.java +++ b/cat-core/src/main/java/com/dianping/cat/message/io/TcpSocketReceiver.java @@ -195,7 +195,8 @@ public class TcpSocketReceiver implements LogEnabled { int length = buffer.readInt(); buffer.resetReaderIndex(); - if (buffer.readableBytes() < length) { + + if (buffer.readableBytes() < length + 4) { return null; } diff --git a/cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java b/cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java index cb9535a6f8bb53c3a49e3796f381bba6c61c45d6..a7a06d44fd83169550c27073bda8ca54123b706d 100644 --- a/cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java +++ b/cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java @@ -61,6 +61,8 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag private int m_total; + private long m_totalSize = 0; + private Logger m_logger; private BlockingQueue m_messageBlocks = new LinkedBlockingQueue(1000); @@ -151,7 +153,7 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag } - private boolean isFit(String path) { + private boolean shouldMove(String path) { if (path.indexOf("draft") > -1 || path.indexOf("outbox") > -1) { return false; } @@ -259,7 +261,7 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag @Override public Direction matches(File base, String path) { if (new File(base, path).isFile()) { - if (isFit(path)) { + if (shouldMove(path)) { paths.add(path); } } @@ -317,6 +319,10 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag DefaultMessageTree defaultTree = (DefaultMessageTree) tree; ChannelBuffer buf = defaultTree.getBuf(); + + int size = buf.readableBytes(); + m_totalSize += size; + MessageBlock bolck = bucket.storeMessage(buf, id); if (bolck != null) { @@ -332,9 +338,9 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag m_total++; if (m_total % 100000 == 0) { - m_logger.info("Encode the message number " + m_total); + m_logger.info("Encode the message number: " + m_total + " Size:" + m_totalSize * 1.0 / 1024 / 1024 / 1024 + + "GB"); } - } class BlockDumper implements Task { diff --git a/cat-home/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java b/cat-home/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java index 934c16fb164ccafaa89a9a369d3c9ebc08dc1132..8431d088257290f8c20fa5123e39777f80842d75 100755 --- a/cat-home/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java +++ b/cat-home/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java @@ -125,8 +125,8 @@ public class ComponentsConfigurator extends AbstractResourceConfigurator { .req(TaskDao.class, ReportDao.class, DailyreportDao.class)); all.add(C(HealthReportBuilder.class) // - .req(GraphDao.class, ReportDao.class, DailyreportDao.class,// - HealthServiceCollector.class)); + .req(GraphDao.class, ReportDao.class, DailyreportDao.class)// + .req(WeeklyreportDao.class, MonthreportDao.class,HealthServiceCollector.class)); all.add(C(ReportFacade.class)// .req(TransactionReportBuilder.class, EventReportBuilder.class, ProblemReportBuilder.class,// diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/cross/CrossReportBuilder.java b/cat-home/src/main/java/com/dianping/cat/report/task/cross/CrossReportBuilder.java index 6af48afacbf7f37787f69ff511efb111f22b0fd3..5a699d0e2a7d19460c22de5d97689f6c7fd0551e 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/task/cross/CrossReportBuilder.java +++ b/cat-home/src/main/java/com/dianping/cat/report/task/cross/CrossReportBuilder.java @@ -1,5 +1,6 @@ package com.dianping.cat.report.task.cross; +import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Set; @@ -8,7 +9,14 @@ import com.dainping.cat.consumer.dal.report.Report; import com.dainping.cat.consumer.dal.report.ReportEntity; import com.dianping.cat.Cat; import com.dianping.cat.configuration.NetworkInterfaceManager; +import com.dianping.cat.consumer.cross.model.entity.CrossReport; +import com.dianping.cat.consumer.cross.model.transform.DefaultSaxParser; +import com.dianping.cat.helper.TimeUtil; import com.dianping.cat.home.dal.report.Dailyreport; +import com.dianping.cat.home.dal.report.DailyreportEntity; +import com.dianping.cat.home.dal.report.Monthreport; +import com.dianping.cat.home.dal.report.Weeklyreport; +import com.dianping.cat.report.page.model.cross.CrossReportMerger; import com.dianping.cat.report.task.TaskHelper; import com.dianping.cat.report.task.spi.AbstractReportBuilder; import com.dianping.cat.report.task.spi.ReportBuilder; @@ -74,13 +82,80 @@ public class CrossReportBuilder extends AbstractReportBuilder implements ReportB } @Override - public boolean buildWeeklyReport(String reportName, String reportDomain, Date reportPeriod) { - return false; - } + public boolean buildWeeklyReport(String reportName, String reportDomain, Date reportPeriod) { + Date start = reportPeriod; + Date end = new Date(start.getTime() + TimeUtil.ONE_DAY * 7); + + CrossReport crossReport = buildMergedDailyReport(reportDomain, start, end); + Weeklyreport report = m_weeklyreportDao.createLocal(); + String content = crossReport.toString(); + + report.setContent(content); + report.setCreationDate(new Date()); + report.setDomain(reportDomain); + report.setIp(NetworkInterfaceManager.INSTANCE.getLocalHostAddress()); + report.setName(reportName); + report.setPeriod(reportPeriod); + report.setType(1); + + try { + m_weeklyreportDao.insert(report); + } catch (DalException e) { + Cat.logError(e); + return false; + } + return true; + } @Override - public boolean buildMonthReport(String reportName, String reportDomain, Date reportPeriod) { - return false; - } + public boolean buildMonthReport(String reportName, String reportDomain, Date reportPeriod) { + Calendar cal = Calendar.getInstance(); + cal.setTime(reportPeriod); + cal.add(Calendar.MONTH, 1); + + Date start = reportPeriod; + Date end = cal.getTime(); + + CrossReport crossReport = buildMergedDailyReport(reportDomain, start, end); + Monthreport report = m_monthreportDao.createLocal(); + report.setContent(crossReport.toString()); + report.setCreationDate(new Date()); + report.setDomain(reportDomain); + report.setIp(NetworkInterfaceManager.INSTANCE.getLocalHostAddress()); + report.setName(reportName); + report.setPeriod(reportPeriod); + report.setType(1); + + try { + m_monthreportDao.insert(report); + } catch (DalException e) { + Cat.logError(e); + return false; + } + return true; + } + + private CrossReport buildMergedDailyReport(String domain, Date start, Date end) { + long startTime = start.getTime(); + long endTime = end.getTime(); + CrossReportMerger merger = new CrossReportMerger(new CrossReport(domain)); + + for (; startTime < endTime; startTime += TimeUtil.ONE_DAY) { + try { + Dailyreport dailyreport = m_dailyReportDao.findByNameDomainPeriod(new Date(startTime), domain, + "cross", DailyreportEntity.READSET_FULL); + String xml = dailyreport.getContent(); + + CrossReport reportModel = DefaultSaxParser.parse(xml); + reportModel.accept(merger); + } catch (Exception e) { + Cat.logError(e); + } + } + CrossReport crossReport = merger.getCrossReport(); + crossReport.setStartTime(start); + crossReport.setEndTime(end); + return crossReport; + } } diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskConsumer.java b/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskConsumer.java index 7f2bb2a8c8905cc76c18ad726bb1cc712848ae08..d9ea92d285f499d78d6ffe255dcddf5a58e9419a 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskConsumer.java +++ b/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskConsumer.java @@ -71,11 +71,11 @@ public abstract class TaskConsumer implements com.site.helper.Threads.Task { updateDoingToDone(task); } } - + t.addData(task.toString()); t.setStatus(Transaction.SUCCESS); } catch (Throwable e) { - t.setStatus(e); Cat.logError(e); + t.setStatus(e); } finally { t.complete(); } diff --git a/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskProducer.java b/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskProducer.java index 6146d75e21de6a34f73a4c32aa8cf9e1b02f3cb0..5b86f4ea3c3eff09021dc3e5826556412d7c929c 100644 --- a/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskProducer.java +++ b/cat-home/src/main/java/com/dianping/cat/report/task/thread/TaskProducer.java @@ -228,7 +228,7 @@ public class TaskProducer implements com.site.helper.Threads.Task, Initializable m_dailyReportNameSet.add("health"); Calendar cal = Calendar.getInstance(); - cal.add(Calendar.MONTH, -2); + cal.add(Calendar.MONTH, -3); cal.set(cal.get(Calendar.YEAR), cal.get(Calendar.MONTH), 0, 0, 0, 0); cal.set(Calendar.MILLISECOND, 0); diff --git a/cat-home/src/main/resources/META-INF/plexus/components.xml b/cat-home/src/main/resources/META-INF/plexus/components.xml index 6874f44830397e2fdc7acbc189b8f103ca49db79..92c45387a37e67d2528edbde7048cac9c295c6be 100755 --- a/cat-home/src/main/resources/META-INF/plexus/components.xml +++ b/cat-home/src/main/resources/META-INF/plexus/components.xml @@ -314,6 +314,12 @@ com.dianping.cat.home.dal.report.DailyreportDao + + com.dianping.cat.home.dal.report.WeeklyreportDao + + + com.dianping.cat.home.dal.report.MonthreportDao + com.dianping.cat.report.task.health.HealthServiceCollector diff --git a/script/Cat.sql b/script/Cat.sql index e77bec0f51836ba715ba0fe22be1e36f00e49c9f..96363fcb2d6a686d0abe7f64696ac0269f16e36f 100644 --- a/script/Cat.sql +++ b/script/Cat.sql @@ -2,7 +2,7 @@ CREATE TABLE `dailygraph` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL COMMENT '报表名称', `ip` varchar(20) NULL COMMENT '报表来自于哪台cat-client机器ip, 空串表示合并同domain所有ip', - `domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `period` datetime NOT NULL COMMENT '报表时间段', `type` tinyint(4) NOT NULL COMMENT '报表数据格式, 1/xml, 2/json, 3/csv, 默认3', `detail_content` mediumtext NOT NULL COMMENT '详细绘图内容', @@ -16,7 +16,7 @@ CREATE TABLE `dailyreport` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL COMMENT '报表名称, transaction, problem...', `ip` varchar(20) NOT NULL COMMENT '报表来自于哪台cat-consumer机器', - `domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `period` datetime NOT NULL COMMENT '报表时间段', `type` tinyint(4) NOT NULL COMMENT '报表数据格式, 1/xml, 2/json, 默认1', `content` mediumtext NOT NULL COMMENT '报表内容', @@ -29,7 +29,7 @@ CREATE TABLE `weeklyreport` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL COMMENT '报表名称, transaction, problem...', `ip` varchar(20) NOT NULL COMMENT '报表来自于哪台cat-consumer机器', - `domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `period` datetime NOT NULL COMMENT '报表时间段', `type` tinyint(4) NOT NULL COMMENT '报表数据格式, 1/xml, 2/json, 默认1', `content` mediumtext NOT NULL COMMENT '报表内容', @@ -42,7 +42,7 @@ CREATE TABLE `monthreport` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL COMMENT '报表名称, transaction, problem...', `ip` varchar(20) NOT NULL COMMENT '报表来自于哪台cat-consumer机器', - `domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `period` datetime NOT NULL COMMENT '报表时间段', `type` tinyint(4) NOT NULL COMMENT '报表数据格式, 1/xml, 2/json, 默认1', `content` mediumtext NOT NULL COMMENT '报表内容', @@ -68,7 +68,7 @@ CREATE TABLE `graph` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(20) NOT NULL COMMENT '报表名称', `ip` varchar(20) NULL COMMENT '报表来自于哪台cat-client机器ip, NULL表示合并同domain所有ip', - `domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `period` datetime NOT NULL COMMENT '报表时间段', `type` tinyint(4) NOT NULL COMMENT '报表数据格式, 1/xml, 2/json, 3/csv, 默认3', `detail_content` mediumtext NOT NULL COMMENT '详细绘图内容', @@ -99,17 +99,6 @@ CREATE TABLE `location` ( UNIQUE KEY `transaction_date_lat_lng` (`transaction_date`,`lat`,`lng`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='用于热点图地理位置表'; -CREATE TABLE `monthreport` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(50) DEFAULT NULL, - `domain` varchar(100) DEFAULT NULL, - `period` datetime DEFAULT NULL, - `content` mediumtext, - `creation_date` datetime DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `name_2` (`name`,`domain`,`period`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - CREATE TABLE `report` ( `id` int(11) NOT NULL AUTO_INCREMENT, @@ -179,7 +168,7 @@ CREATE TABLE `task` ( `consumer` varchar(20) NULL COMMENT '任务执行者ip', `failure_count` tinyint(4) NOT NULL COMMENT '任务失败次数', `report_name` varchar(20) NOT NULL COMMENT '报表名称, transaction, problem...', - `report_domain` varchar(20) NOT NULL COMMENT '报表处理的Domain信息', + `report_domain` varchar(50) NOT NULL COMMENT '报表处理的Domain信息', `report_period` datetime NOT NULL COMMENT '报表时间', `status` tinyint(4) NOT NULL COMMENT '执行状态: 1/todo, 2/doing, 3/done 4/failed', `task_type` tinyint(4) NOT NULL DEFAULT '1' COMMENT '0表示小时任务，1表示天任务',