From 22f12c2085d8574f0181e27ecf52aa1b008fcef1 Mon Sep 17 00:00:00 2001 From: ascrutae Date: Fri, 29 Jan 2016 10:02:20 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=98=E9=87=8F=E5=90=8D?= =?UTF-8?q?=EF=BC=8C=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analysis/AnalysisServerDriver.java | 3 +- .../categorize2chain/ChainDetail.java | 3 +- .../categorize2chain/ChainRelationship.java | 4 +- .../ChainSpecificTimeWindowSummary.java | 3 +- .../skywalking/analysis/config/Config.java | 28 +------------- .../skywalking/analysis/config/Constants.java | 24 +++++++++++- .../skywalking/analysis/util/HBaseUtil.java | 37 ++++++++----------- 7 files changed, 49 insertions(+), 53 deletions(-) diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java index 273b753a0b..5eefd37d17 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/AnalysisServerDriver.java @@ -5,6 +5,7 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import com.ai.cloud.skywalking.analysis.config.Constants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hbase.client.Scan; @@ -50,7 +51,7 @@ public class AnalysisServerDriver extends Configured implements Tool { job.setJarByClass(AnalysisServerDriver.class); Scan scan = buildHBaseScan(args); - TableMapReduceUtil.initTableMapperJob(Config.HBase.TABLE_CALL_CHAIN, scan, Categorize2ChainMapper.class, + TableMapReduceUtil.initTableMapperJob(Constants.TABLE_CALL_CHAIN, scan, Categorize2ChainMapper.class, Text.class, ChainInfo.class, job); job.setReducerClass(Categorize2ChainReducer.class); diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainDetail.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainDetail.java index d7f5021af8..a522ab7c92 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainDetail.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainDetail.java @@ -3,6 +3,7 @@ package com.ai.cloud.skywalking.analysis.categorize2chain; import com.ai.cloud.skywalking.analysis.categorize2chain.model.ChainInfo; import com.ai.cloud.skywalking.analysis.categorize2chain.model.ChainNode; import com.ai.cloud.skywalking.analysis.config.Config; +import com.ai.cloud.skywalking.analysis.config.Constants; import com.google.gson.Gson; import org.apache.hadoop.hbase.client.Put; @@ -35,7 +36,7 @@ public class ChainDetail { public void save(Put put) throws SQLException { for (Map.Entry entry : chainNodeMap.entrySet()){ - put.addColumn(Config.HBase.TRACE_DETAIL_FAMILY_COLUMN.getBytes(),entry.getKey().getBytes(), + put.addColumn(Constants.COLUMN_FAMILY_NAME_TRACE_DETAIL.getBytes(),entry.getKey().getBytes(), entry.getValue().toString().getBytes()); } if (isNormal) { diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainRelationship.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainRelationship.java index c90006c022..86c253a47c 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainRelationship.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainRelationship.java @@ -107,11 +107,11 @@ public class ChainRelationship { private void saveChainRelationship() throws IOException { Put put = new Put(getKey().getBytes()); - put.addColumn(Config.HBase.CHAIN_RELATIONSHIP_COLUMN_FAMILY.getBytes(), Constants.UNCATEGORIZED_QUALIFIER_NAME.getBytes() + put.addColumn(Constants.COLUMN_FAMILY_CHAIN_RELATIONSHIP.getBytes(), Constants.UNCATEGORIZE_COLUMN_FAMILY.getBytes() , new GsonBuilder().excludeFieldsWithoutExposeAnnotation().create().toJson(getUncategorizeChainInfoList()).getBytes()); for (Map.Entry entry : getCategorizedChainInfoMap().entrySet()) { - put.addColumn(Config.HBase.CHAIN_RELATIONSHIP_COLUMN_FAMILY.getBytes(), entry.getKey().getBytes() + put.addColumn(Constants.COLUMN_FAMILY_CHAIN_RELATIONSHIP.getBytes(), entry.getKey().getBytes() , entry.getValue().toString().getBytes()); } diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainSpecificTimeWindowSummary.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainSpecificTimeWindowSummary.java index 426ca18538..cf7d60b005 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainSpecificTimeWindowSummary.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/categorize2chain/ChainSpecificTimeWindowSummary.java @@ -2,6 +2,7 @@ package com.ai.cloud.skywalking.analysis.categorize2chain; import com.ai.cloud.skywalking.analysis.categorize2chain.model.ChainNode; import com.ai.cloud.skywalking.analysis.config.Config; +import com.ai.cloud.skywalking.analysis.config.Constants; import com.ai.cloud.skywalking.analysis.util.HBaseUtil; import org.apache.hadoop.hbase.client.Put; @@ -51,7 +52,7 @@ public class ChainSpecificTimeWindowSummary { public void save(Put put) { for (Map.Entry entry : chainNodeSummaryResultMap.entrySet()) { - put.addColumn(Config.HBase.CHAIN_SUMMARY_COLUMN_FAMILY.getBytes(), entry.getKey().getBytes(), entry.getValue().toString().getBytes()); + put.addColumn(Constants.COLUMN_FAMILY_NAME_CHAIN_SUMMARY.getBytes(), entry.getKey().getBytes(), entry.getValue().toString().getBytes()); } } } diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java index a03c4c554c..bd4e87e72a 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Config.java @@ -1,38 +1,14 @@ package com.ai.cloud.skywalking.analysis.config; public class Config { - public static class Reducer{ + public static class Reducer { public static int REDUCER_NUMBER = 1; } public static class HBase { - public static String TRACE_DETAIL_FAMILY_COLUMN = "chain_detail"; - - public static String CHAIN_SUMMARY_COLUMN_FAMILY = "chain_summary"; - - public static String TRACE_INFO_COLUMN_FAMILY = "trace_info"; - public static String ZK_QUORUM; public static String ZK_CLIENT_PORT; - - public static String TRACE_INFO_TABLE_NAME = "trace-info"; - - public static String TABLE_CALL_CHAIN_RELATIONSHIP = "sw-chain-relationship"; - - public static String CHAIN_RELATIONSHIP_COLUMN_FAMILY = "chain-relationship"; - - public static String TABLE_CHAIN_INFO = "sw-chain-info"; - - public static String TABLE_CHAIN_SUMMARY = "sw-chain-summary"; - - public static String TABLE_CHAIN_DETAIL = "sw-chain-detail"; - - public static String TABLE_CALL_CHAIN = "sw-call-chain"; - } - - public static class TraceInfo { - public static String TRACE_INFO_COLUMN_CID = "cid"; } public static class MySql { @@ -51,6 +27,6 @@ public class Config { } public static class ChainNodeSummary { - public static long INTERVAL = 5L; + public static long INTERVAL = 1L; } } diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Constants.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Constants.java index e5d1562866..a6a101ff47 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Constants.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/config/Constants.java @@ -1,5 +1,27 @@ package com.ai.cloud.skywalking.analysis.config; public class Constants { - public static final String UNCATEGORIZED_QUALIFIER_NAME = "UNCATEGORIZED_CALL_CHAIN"; + public static final String UNCATEGORIZE_COLUMN_FAMILY = "UNCATEGORIZED_CALL_CHAIN"; + + // HBase 表：用于存放CID，TID的映射关系表 + public static String TABLE_CID_TID_MAPPING = "sw-cid-tid-mapping"; + + //HBase 表：用于存放调用链一分钟的汇总，汇总结果不包含关系汇总 + public static String TABLE_CHAIN_ONE_MINUTE_SUMMARY_EXCLUDE_RELATIONSHIP = "sw-chain-1min-summary-ex-rela"; + + public static String TABLE_CHAIN_DETAIL = "sw-chain-detail"; + + public static String TABLE_CALL_CHAIN = "sw-call-chain"; + + public static String TABLE_CALL_CHAIN_RELATIONSHIP = "sw-chain-relationship"; + + public static String COLUMN_FAMILY_CHAIN_RELATIONSHIP = "chain-relationship"; + + public static String COLUMN_FAMILY_NAME_TRACE_DETAIL = "chain_detail"; + + public static String COLUMN_FAMILY_NAME_CHAIN_SUMMARY = "chain_summary"; + + public static String COLUMN_FAMILY_NAME_TRACE_INFO = "trace_info"; + + public static String COLUMN_FAMILY_NAME_CID = "cid"; } diff --git a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/HBaseUtil.java b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/HBaseUtil.java index 1b621eefcf..46e3bbc14e 100644 --- a/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/HBaseUtil.java +++ b/skywalking-analysis/src/main/java/com/ai/cloud/skywalking/analysis/util/HBaseUtil.java @@ -1,16 +1,11 @@ package com.ai.cloud.skywalking.analysis.util; -import com.ai.cloud.skywalking.analysis.categorize2chain.CategorizedChainInfo; -import com.ai.cloud.skywalking.analysis.categorize2chain.ChainNodeSpecificTimeWindowSummary; -import com.ai.cloud.skywalking.analysis.categorize2chain.ChainRelationship; -import com.ai.cloud.skywalking.analysis.categorize2chain.ChainSpecificTimeWindowSummary; -import com.ai.cloud.skywalking.analysis.categorize2chain.UncategorizeChainInfo; +import com.ai.cloud.skywalking.analysis.categorize2chain.*; import com.ai.cloud.skywalking.analysis.categorize2chain.model.ChainInfo; import com.ai.cloud.skywalking.analysis.config.Config; import com.ai.cloud.skywalking.analysis.config.Constants; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -31,15 +26,15 @@ public class HBaseUtil { Table table = null; try { - table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CHAIN_INFO)); + table = connection.getTable(TableName.valueOf(Constants.TABLE_CID_TID_MAPPING)); } catch (IOException e) { - logger.error("Cannot found table[" + Config.HBase.TRACE_INFO_TABLE_NAME + "]", e); + logger.error("Cannot found table[" + Constants.TABLE_CID_TID_MAPPING + "]", e); } Put put = new Put(Bytes.toBytes(traceId)); - put.addColumn(Bytes.toBytes(Config.HBase.TRACE_INFO_COLUMN_FAMILY), - Bytes.toBytes(Config.TraceInfo.TRACE_INFO_COLUMN_CID), + put.addColumn(Bytes.toBytes(Constants.COLUMN_FAMILY_NAME_TRACE_INFO), + Bytes.toBytes(Constants.COLUMN_FAMILY_NAME_CID), Bytes.toBytes(chainInfo.getCID())); try { table.put(put); @@ -58,16 +53,16 @@ public class HBaseUtil { try { initHBaseClient(); // - createTableIfNeed(Config.HBase.TABLE_CHAIN_INFO, Config.HBase.TRACE_INFO_COLUMN_FAMILY); + createTableIfNeed(Constants.TABLE_CID_TID_MAPPING, Constants.COLUMN_FAMILY_NAME_TRACE_INFO); // - createTableIfNeed(Config.HBase.TABLE_CALL_CHAIN_RELATIONSHIP, Config.HBase.CHAIN_RELATIONSHIP_COLUMN_FAMILY); + createTableIfNeed(Constants.TABLE_CALL_CHAIN_RELATIONSHIP, Constants.COLUMN_FAMILY_CHAIN_RELATIONSHIP); - createTableIfNeed(Config.HBase.TABLE_CHAIN_SUMMARY, Config.HBase.CHAIN_SUMMARY_COLUMN_FAMILY); + createTableIfNeed(Constants.TABLE_CHAIN_ONE_MINUTE_SUMMARY_EXCLUDE_RELATIONSHIP, Constants.COLUMN_FAMILY_NAME_CHAIN_SUMMARY); - createTableIfNeed(Config.HBase.TABLE_CHAIN_DETAIL, Config.HBase.TRACE_DETAIL_FAMILY_COLUMN); + createTableIfNeed(Constants.TABLE_CHAIN_DETAIL, Constants.COLUMN_FAMILY_NAME_TRACE_DETAIL); } catch (IOException e) { - logger.error("Create table[{}] failed", Config.HBase.TRACE_INFO_TABLE_NAME, e); + logger.error("Create tables failed", e); } } @@ -97,7 +92,7 @@ public class HBaseUtil { public static ChainRelationship selectCallChainRelationship(String key) throws IOException { ChainRelationship chainRelate = new ChainRelationship(key); - Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CALL_CHAIN_RELATIONSHIP)); + Table table = connection.getTable(TableName.valueOf(Constants.TABLE_CALL_CHAIN_RELATIONSHIP)); Get g = new Get(Bytes.toBytes(key)); Result r = table.get(g); for (Cell cell : r.rawCells()) { @@ -105,7 +100,7 @@ public class HBaseUtil { String qualifierName = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); - if (Constants.UNCATEGORIZED_QUALIFIER_NAME.equals(qualifierName)) { + if (Constants.UNCATEGORIZE_COLUMN_FAMILY.equals(qualifierName)) { List uncategorizeChainInfoList = new Gson().fromJson(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()), new TypeToken>() { @@ -123,7 +118,7 @@ public class HBaseUtil { public static ChainSpecificTimeWindowSummary selectChainSummaryResult(String key) throws IOException { ChainSpecificTimeWindowSummary result = null; - Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CHAIN_SUMMARY)); + Table table = connection.getTable(TableName.valueOf(Constants.TABLE_CHAIN_ONE_MINUTE_SUMMARY_EXCLUDE_RELATIONSHIP)); Get g = new Get(Bytes.toBytes(key)); Result r = table.get(g); @@ -141,7 +136,7 @@ public class HBaseUtil { } public static void saveChainRelationship(Put put) throws IOException { - Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CALL_CHAIN_RELATIONSHIP)); + Table table = connection.getTable(TableName.valueOf(Constants.TABLE_CALL_CHAIN_RELATIONSHIP)); table.put(put); if (logger.isDebugEnabled()) { @@ -150,7 +145,7 @@ public class HBaseUtil { } public static void batchSaveChainSpecificTimeWindowSummary(List puts) throws IOException, InterruptedException { - Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CHAIN_SUMMARY)); + Table table = connection.getTable(TableName.valueOf(Constants.TABLE_CHAIN_ONE_MINUTE_SUMMARY_EXCLUDE_RELATIONSHIP)); Object[] resultArrays = new Object[puts.size()]; table.batch(puts, resultArrays); for (Object result : resultArrays) { @@ -161,7 +156,7 @@ public class HBaseUtil { } public static void saveChainDetails(List puts) throws IOException, InterruptedException { - Table table = connection.getTable(TableName.valueOf(Config.HBase.TABLE_CHAIN_DETAIL)); + Table table = connection.getTable(TableName.valueOf(Constants.TABLE_CHAIN_DETAIL)); if (puts != null && puts.size() > 0) { Object[] resultArrays = new Object[puts.size()]; table.batch(puts, resultArrays); -- GitLab