Merge pull request #405 from zengwei0771/master

app data collect refactor

cat-broker/src/main/java/com/dianping/cat/broker/api/app/BucketHandler.java

 package com.dianping.cat.broker.api.app;
 
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map.Entry;
 
 import org.unidal.helper.Threads.Task;
@@ -16,8 +15,6 @@ import com.dianping.cat.config.app.AppDataService;
 
 public class BucketHandler implements Task {
 
-	private static final String FILEDIRECTORY = "/data/appdatas/cat/app/";
-
 	private static int ONE_MINUTE = 60 * 1000;
 
 	private static int ONE_DAY = 24 * 60 * ONE_MINUTE;
@@ -40,13 +37,39 @@ public class BucketHandler implements Task {
 	}
 
 	private void end() {
+		int minute = (int) (m_startTime % ONE_DAY / ONE_MINUTE);
+		Date period = new Date(m_startTime - minute * ONE_MINUTE);
+		List<AppDataCommand> appDataCommands = new ArrayList<AppDataCommand>();
+
 		for (Entry<Integer, HashMap<String, AppData>> outerEntry : m_mergedData.entrySet()) {
 			for (Entry<String, AppData> entry : outerEntry.getValue().entrySet()) {
 				AppData appData = entry.getValue();
-
-				saveToDataBase(appData);
+				AppDataCommand proto = new AppDataCommand();
+
+				proto.setPeriod(period);
+				proto.setMinuteOrder(minute);
+				proto.setCommandId(appData.getCommand());
+				proto.setCity(appData.getCity());
+				proto.setOperator(appData.getOperator());
+				proto.setNetwork(appData.getNetwork());
+				proto.setAppVersion(appData.getVersion());
+				proto.setConnnectType(appData.getConnectType());
+				proto.setCode(appData.getCode());
+				proto.setPlatform(appData.getPlatform());
+				proto.setAccessNumber(appData.getCount());
+				proto.setResponseSumTime(appData.getResponseTime());
+				proto.setRequestPackage(appData.getRequestByte());
+				proto.setResponsePackage(appData.getResponseByte());
+				proto.setCreationDate(new Date());
+				appDataCommands.add(proto);
 			}
 		}
+
+		try {
+			m_appDataService.insert((AppDataCommand[]) appDataCommands.toArray());
+		} catch (Exception e) {
+			Cat.logError(e);
+		}
 	}
 
 	public void enqueue(AppData appData) {
@@ -122,56 +145,6 @@ public class BucketHandler implements Task {
 		end();
 	}
 
-	private void saveToDataBase(AppData appData) {
-		int minute = (int) (m_startTime % ONE_DAY / ONE_MINUTE);
-		Date period = new Date(m_startTime - minute * ONE_MINUTE);
-
-		try {
-			AppDataCommand proto = new AppDataCommand();
-
-			proto.setPeriod(period);
-			proto.setMinuteOrder(minute);
-			proto.setCommandId(appData.getCommand());
-			proto.setCity(appData.getCity());
-			proto.setOperator(appData.getOperator());
-			proto.setNetwork(appData.getNetwork());
-			proto.setAppVersion(appData.getVersion());
-			proto.setConnnectType(appData.getConnectType());
-			proto.setCode(appData.getCode());
-			proto.setPlatform(appData.getPlatform());
-			proto.setAccessNumber(appData.getCount());
-			proto.setResponseSumTime(appData.getResponseTime());
-			proto.setRequestPackage(appData.getRequestByte());
-			proto.setResponsePackage(appData.getResponseByte());
-			proto.setCreationDate(new Date());
-			m_appDataService.insert(proto);
-		} catch (Exception e) {
-			Cat.logError(e);
-
-			saveToFile(appData);
-		}
-	}
-
-	private void saveToFile(AppData appData) {
-		Date date = new Date();
-		SimpleDateFormat formater = new SimpleDateFormat("yyyy-MM-dd");
-		String dateStr = formater.format(date);
-		String filePath = FILEDIRECTORY + dateStr;
-
-		try {
-			BufferedWriter writer = new BufferedWriter(new FileWriter(filePath));
-			String content = appData.getTimestamp() + "\t" + appData.getCity() + "\t" + appData.getOperator() + "\t"
-			      + appData.getNetwork() + "\t" + appData.getVersion() + "\t" + appData.getConnectType() + "\t"
-			      + appData.getCommand() + "\t" + appData.getCode() + "\t" + appData.getPlatform() + "\t"
-			      + appData.getRequestByte() + "\t" + appData.getResponseByte() + "\t" + appData.getResponseTime() + "\n";
-
-			writer.append(content);
-			writer.close();
-		} catch (Exception e) {
-			Cat.logError(e);
-		}
-	}
-
 	@Override
 	public void shutdown() {
 		synchronized (this) {

cat-broker/src/main/java/com/dianping/cat/broker/api/page/batch/Handler.java

@@ -126,7 +126,7 @@ public class Handler implements PageHandler<Context>, LogEnabled {
 				String operatorStr = ipInfo.getChannel();
 				Integer cityId = m_appConfigManager.getCities().get(province);
 				Integer operatorId = m_appConfigManager.getOperators().get(operatorStr);
-
+				
 				if (cityId != null && operatorId != null) {
 					for (String record : records) {
 						processOneRecord(cityId, operatorId, record);
@@ -146,7 +146,7 @@ public class Handler implements PageHandler<Context>, LogEnabled {
 
 			try {
 				appData.setTimestamp(Long.parseLong(items[0]));
-				Integer command = m_appConfigManager.getCommands().get(items[2]);
+				Integer command = m_appConfigManager.getCommands().get(items[1]);
 
 				if (command != null) {
 					appData.setCommand(command);

cat-broker/src/test/java/com/dianping/cat/broker/AppTest.java

@@ -17,7 +17,7 @@ public class AppTest {
 		long timestamp = System.currentTimeMillis();
 		
 		for (int i = 0; i < 10; i++) {
-			urls.add(url_pre + "?v=2&c=" + timestamp + URLEncoder.encode("\tshop.bin\t1\t1\t1\t1\t1\t1\t1\t1\n", "utf-8"));
+			urls.add(url_pre + "?v=2&c=" + timestamp + URLEncoder.encode("\thttp://www.dianping.com/\t1\t1\t1\t1\t1\t1\t1\t1\n", "utf-8"));
 		}
 
 		for (String url : urls) {

cat-core/src/main/java/com/dianping/cat/config/app/AppConfigManager.java

@@ -71,6 +71,7 @@ public class AppConfigManager implements Initializable {
 			m_configId = config.getId();
 			m_config = DefaultSaxParser.parse(content);
 			m_modifyTime = config.getModifyDate().getTime();
+			refreshData();
 		} catch (DalNotFoundException e) {
 			try {
 				String content = Files.forIO().readFrom(
@@ -132,14 +133,14 @@ public class AppConfigManager implements Initializable {
 	public Map<String, Integer> getCommands() {
 		return m_commands;
 	}
-	
+
 	public Map<String, Integer> getCities() {
-   	return m_cities;
-   }
+		return m_cities;
+	}
 
 	public Map<String, Integer> getOperators() {
-   	return m_operators;
-   }
+		return m_operators;
+	}
 
 	private void refreshData() {
 		Collection<Command> commands = m_config.getCommands().values();
@@ -149,19 +150,19 @@ public class AppConfigManager implements Initializable {
 			commandMap.put(c.getName(), c.getId());
 		}
 		m_commands = commandMap;
-		
+
 		Map<String, Integer> cityMap = new HashMap<String, Integer>();
 		ConfigItem cities = m_config.findConfigItem(CITY);
-		
-		for(Item item:cities.getItems().values()){
+
+		for (Item item : cities.getItems().values()) {
 			cityMap.put(item.getName(), item.getId());
 		}
 		m_cities = cityMap;
-		
+
 		Map<String, Integer> operatorMap = new HashMap<String, Integer>();
 		ConfigItem operations = m_config.findConfigItem(OPERATOR);
-		
-		for(Item item:operations.getItems().values()){
+
+		for (Item item : operations.getItems().values()) {
 			operatorMap.put(item.getName(), item.getId());
 		}
 		m_operators = operatorMap;

cat-home/src/main/resources/config/default-app-config.xml

@@ -5,8 +5,10 @@
 		<item id="3" name="4G"></item>
 	</config-item>
 	<config-item id="运营商">
-		<item id="1" name="移动"></item>
-		<item id="2" name="联通"></item>
+		<item id="1" name="中国移动"></item>
+		<item id="2" name="中国联通"></item>
+		<item id="3" name="中国电信"></item>
+		<item id="4" name="中国铁通"></item>
 	</config-item>
 	<config-item id="版本">
 		<item id="1" name="1.0"></item>