Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
梦中观雨
cat
提交
d83d0f41
C
cat
项目概览
梦中观雨
/
cat
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
cat
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d83d0f41
编写于
9月 03, 2012
作者:
F
Frankie Wu
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
dump performance tuning
上级
c21fe54a
变更
15
隐藏空白更改
内联
并排
Showing
15 changed file
with
364 addition
and
362 deletion
+364
-362
cat-consumer/src/main/java/com/dianping/cat/consumer/dump/DumpAnalyzer.java
...ain/java/com/dianping/cat/consumer/dump/DumpAnalyzer.java
+14
-76
cat-core/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java
...n/java/com/dianping/cat/build/ComponentsConfigurator.java
+5
-2
cat-core/src/main/java/com/dianping/cat/message/internal/MessageIdFactory.java
...a/com/dianping/cat/message/internal/MessageIdFactory.java
+5
-1
cat-core/src/main/java/com/dianping/cat/storage/dump/ChannelBufferManager.java
...a/com/dianping/cat/storage/dump/ChannelBufferManager.java
+28
-0
cat-core/src/main/java/com/dianping/cat/storage/dump/DumpItem.java
...src/main/java/com/dianping/cat/storage/dump/DumpItem.java
+0
-49
cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucket.java
...ava/com/dianping/cat/storage/dump/LocalMessageBucket.java
+70
-183
cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java
.../dianping/cat/storage/dump/LocalMessageBucketManager.java
+62
-49
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlock.java
...main/java/com/dianping/cat/storage/dump/MessageBlock.java
+47
-0
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlockReader.java
...ava/com/dianping/cat/storage/dump/MessageBlockReader.java
+65
-0
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlockWriter.java
...ava/com/dianping/cat/storage/dump/MessageBlockWriter.java
+48
-0
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBucket.java
...ain/java/com/dianping/cat/storage/dump/MessageBucket.java
+1
-1
cat-core/src/main/resources/META-INF/plexus/components.xml
cat-core/src/main/resources/META-INF/plexus/components.xml
+7
-0
cat-core/src/test/java/com/dianping/cat/storage/dump/LocalMessageBucketManagerTest.java
...nping/cat/storage/dump/LocalMessageBucketManagerTest.java
+8
-0
cat-core/src/test/java/com/dianping/cat/storage/dump/LocalMessageBucketTest.java
...com/dianping/cat/storage/dump/LocalMessageBucketTest.java
+2
-0
cat-hadoop/src/main/java/com/dianping/cat/hadoop/hdfs/HdfsMessageBucket.java
.../java/com/dianping/cat/hadoop/hdfs/HdfsMessageBucket.java
+2
-1
未找到文件。
cat-consumer/src/main/java/com/dianping/cat/consumer/dump/DumpAnalyzer.java
浏览文件 @
d83d0f41
...
...
@@ -4,27 +4,20 @@ import java.io.IOException;
import
java.util.Collections
;
import
java.util.Date
;
import
java.util.Set
;
import
java.util.concurrent.BlockingQueue
;
import
java.util.concurrent.LinkedBlockingQueue
;
import
java.util.concurrent.TimeUnit
;
import
org.codehaus.plexus.logging.LogEnabled
;
import
org.codehaus.plexus.logging.Logger
;
import
org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable
;
import
org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException
;
import
com.dianping.cat.Cat
;
import
com.dianping.cat.configuration.NetworkInterfaceManager
;
import
com.dianping.cat.configuration.ServerConfigManager
;
import
com.dianping.cat.message.internal.MessageId
;
import
com.dianping.cat.message.spi.AbstractMessageAnalyzer
;
import
com.dianping.cat.message.spi.MessagePathBuilder
;
import
com.dianping.cat.message.spi.MessageTree
;
import
com.dianping.cat.storage.dump.DumpItem
;
import
com.dianping.cat.storage.dump.LocalMessageBucketManager
;
import
com.dianping.cat.storage.dump.MessageBucketManager
;
import
com.site.helper.Threads
;
import
com.site.helper.Threads.Task
;
import
com.site.lookup.annotation.Inject
;
public
class
DumpAnalyzer
extends
AbstractMessageAnalyzer
<
Object
>
implements
Initializable
,
LogEnabled
{
...
...
@@ -47,14 +40,6 @@ public class DumpAnalyzer extends AbstractMessageAnalyzer<Object> implements Ini
private
Logger
m_logger
;
private
final
BlockingQueue
<
DumpItem
>
m_storeQueue
=
new
LinkedBlockingQueue
<
DumpItem
>(
10000
);
private
int
m_errors
;
public
DumpUploader
getDumpUploader
()
{
return
m_uploader
;
}
@Override
public
void
doCheckpoint
(
boolean
atEnd
)
{
if
(
atEnd
)
{
...
...
@@ -78,11 +63,24 @@ public class DumpAnalyzer extends AbstractMessageAnalyzer<Object> implements Ini
return
Collections
.
emptySet
();
}
public
DumpUploader
getDumpUploader
()
{
return
m_uploader
;
}
@Override
public
Object
getReport
(
String
domain
)
{
throw
new
UnsupportedOperationException
(
"This should not be called!"
);
}
@Override
public
void
initialize
()
throws
InitializationException
{
m_localMode
=
m_configManager
.
isLocalMode
();
if
(!
m_localMode
)
{
m_uploader
.
start
();
}
}
@Override
protected
boolean
isTimeout
()
{
long
currentTime
=
System
.
currentTimeMillis
();
...
...
@@ -122,17 +120,7 @@ public class DumpAnalyzer extends AbstractMessageAnalyzer<Object> implements Ini
}
}
else
{
try
{
DumpItem
item
=
m_bucketManager
.
buildStoreMetaInfo
(
tree
);
boolean
result
=
m_storeQueue
.
offer
(
item
);
if
(!
result
)
{
m_errors
++;
if
(
m_errors
==
1
||
m_errors
%
10000
==
0
)
{
m_logger
.
error
(
"Error when put dump item into queue, errors:"
+
m_errors
);
}
}
//m_bucketManager.storeMessage(tree);
m_bucketManager
.
storeMessage
(
tree
);
}
catch
(
IOException
e
)
{
m_logger
.
error
(
"Error when dumping to local file system!"
,
e
);
}
...
...
@@ -144,54 +132,4 @@ public class DumpAnalyzer extends AbstractMessageAnalyzer<Object> implements Ini
m_startTime
=
startTime
;
m_duration
=
duration
;
}
@Override
public
void
initialize
()
throws
InitializationException
{
m_localMode
=
m_configManager
.
isLocalMode
();
if
(!
m_localMode
)
{
m_uploader
.
start
();
}
Threads
.
forGroup
(
"Cat"
).
start
(
new
WriteMessageTree
());
}
public
class
WriteMessageTree
implements
Task
{
private
int
m_error
=
0
;
@Override
public
void
run
()
{
// int i = 0;
// long current = System.currentTimeMillis();
while
(
true
)
{
try
{
DumpItem
item
=
m_storeQueue
.
poll
(
5
,
TimeUnit
.
MILLISECONDS
);
if
(
item
!=
null
)
{
m_bucketManager
.
storeMessage
(
item
);
// i++;
// if (i % 10000 == 0) {
// long l = System.currentTimeMillis() - current;
// System.out.println("Total :" + i + " time " + l);
// System.out.println((double) i / l);
// }
}
}
catch
(
Exception
e
)
{
if
(
m_error
==
1
||
m_error
%
10000
==
0
)
{
Cat
.
logError
(
e
);
}
}
}
}
@Override
public
String
getName
()
{
return
"WriteMessageTree"
;
}
@Override
public
void
shutdown
()
{
}
}
}
cat-core/src/main/java/com/dianping/cat/build/ComponentsConfigurator.java
浏览文件 @
d83d0f41
...
...
@@ -37,6 +37,7 @@ import com.dianping.cat.message.spi.internal.DefaultMessagePathBuilder;
import
com.dianping.cat.message.spi.internal.DefaultMessageStatistics
;
import
com.dianping.cat.message.spi.internal.DefaultMessageStorage
;
import
com.dianping.cat.status.StatusUpdateTask
;
import
com.dianping.cat.storage.dump.ChannelBufferManager
;
import
com.dianping.cat.storage.dump.LocalMessageBucket
;
import
com.dianping.cat.storage.dump.LocalMessageBucketManager
;
import
com.dianping.cat.storage.dump.MessageBucket
;
...
...
@@ -95,13 +96,15 @@ public class ComponentsConfigurator extends AbstractResourceConfigurator {
.
req
(
ServerConfigManager
.
class
));
all
.
add
(
C
(
MessageStatistics
.
class
,
DefaultMessageStatistics
.
class
));
all
.
add
(
C
(
StatusUpdateTask
.
class
)
//
.
req
(
MessageStatistics
.
class
,
ClientConfigManager
.
class
));
.
req
(
MessageStatistics
.
class
,
ClientConfigManager
.
class
));
all
.
add
(
C
(
MessageBucket
.
class
,
LocalMessageBucket
.
ID
,
LocalMessageBucket
.
class
)
//
.
is
(
PER_LOOKUP
)
//
.
req
(
MessageCodec
.
class
,
PlainTextMessageCodec
.
ID
));
.
req
(
MessageCodec
.
class
,
PlainTextMessageCodec
.
ID
)
//
.
req
(
ChannelBufferManager
.
class
));
all
.
add
(
C
(
MessageBucketManager
.
class
,
LocalMessageBucketManager
.
ID
,
LocalMessageBucketManager
.
class
)
//
.
req
(
ServerConfigManager
.
class
,
MessagePathBuilder
.
class
));
all
.
add
(
C
(
ChannelBufferManager
.
class
));
all
.
add
(
C
(
Module
.
class
,
CatCoreModule
.
ID
,
CatCoreModule
.
class
));
...
...
cat-core/src/main/java/com/dianping/cat/message/internal/MessageIdFactory.java
浏览文件 @
d83d0f41
...
...
@@ -94,7 +94,7 @@ public class MessageIdFactory {
m_markFile
=
new
RandomAccessFile
(
mark
,
"rw"
);
m_byteBuffer
=
m_markFile
.
getChannel
().
map
(
MapMode
.
READ_WRITE
,
0
,
20
);
if
(
m_byteBuffer
.
limit
()
>
0
)
{
int
index
=
m_byteBuffer
.
getInt
();
long
timestamp
=
m_byteBuffer
.
getLong
();
...
...
@@ -126,4 +126,8 @@ public class MessageIdFactory {
public
void
setIpAddress
(
String
ipAddress
)
{
m_ipAddress
=
ipAddress
;
}
protected
void
resetIndex
()
{
m_index
=
0
;
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/ChannelBufferManager.java
0 → 100644
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.util.concurrent.BlockingQueue
;
import
java.util.concurrent.LinkedBlockingQueue
;
import
org.jboss.netty.buffer.ChannelBuffer
;
import
org.jboss.netty.buffer.ChannelBuffers
;
public
class
ChannelBufferManager
{
private
BlockingQueue
<
ChannelBuffer
>
m_pool
=
new
LinkedBlockingQueue
<
ChannelBuffer
>(
100
);
public
ChannelBuffer
allocate
()
{
ChannelBuffer
buffer
=
m_pool
.
poll
();
if
(
buffer
!=
null
)
{
return
buffer
;
}
else
{
return
ChannelBuffers
.
dynamicBuffer
(
16384
);
}
}
public
void
revoke
(
ChannelBuffer
buffer
)
{
if
(
buffer
.
capacity
()
<=
16384
)
{
// get rid of big buffer
buffer
.
clear
();
m_pool
.
offer
(
buffer
);
}
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/DumpItem.java
已删除
100644 → 0
浏览文件 @
c21fe54a
package
com.dianping.cat.storage.dump
;
public
class
DumpItem
{
private
int
m_index
;
private
String
m_fileName
;
private
int
m_size
;
private
byte
[]
m_bytes
;
public
int
getIndex
()
{
return
m_index
;
}
public
DumpItem
setIndex
(
int
index
)
{
m_index
=
index
;
return
this
;
}
public
String
getFileName
()
{
return
m_fileName
;
}
public
DumpItem
setFileName
(
String
fileName
)
{
m_fileName
=
fileName
;
return
this
;
}
public
int
getSize
()
{
return
m_size
;
}
public
DumpItem
setSize
(
int
size
)
{
m_size
=
size
;
return
this
;
}
public
byte
[]
getBytes
()
{
return
m_bytes
;
}
public
void
setBytes
(
byte
[]
bytes
)
{
m_bytes
=
bytes
;
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucket.java
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.io.ByteArrayInputStream
;
import
java.io.ByteArrayOutputStream
;
import
java.io.DataInputStream
;
import
java.io.DataOutputStream
;
import
java.io.EOFException
;
import
java.io.File
;
import
java.io.IOException
;
import
java.io.RandomAccessFile
;
import
java.util.concurrent.atomic.AtomicBoolean
;
import
java.util.zip.GZIPInputStream
;
import
java.util.zip.GZIPOutputStream
;
import
org.jboss.netty.buffer.ChannelBuffer
;
...
...
@@ -25,9 +20,14 @@ import com.site.lookup.annotation.Inject;
public
class
LocalMessageBucket
implements
MessageBucket
{
public
static
final
String
ID
=
"local"
;
private
static
final
int
MAX_BLOCK_SIZE
=
1
<<
16
;
// 64K
@Inject
private
MessageCodec
m_codec
;
@Inject
private
ChannelBufferManager
m_bufferManager
;
@Inject
private
File
m_baseDir
=
new
File
(
"."
);
...
...
@@ -43,11 +43,31 @@ public class LocalMessageBucket implements MessageBucket {
private
long
m_lastAccessTime
;
private
void
checkDirty
()
throws
IOException
{
if
(
m_dirty
.
get
())
{
m_writer
.
flushBlock
();
m_dirty
.
set
(
false
);
}
private
GZIPOutputStream
m_out
;
private
ByteArrayOutputStream
m_buf
;
private
MessageBlock
m_block
;
private
int
m_blockSize
;
public
void
archive
()
throws
IOException
{
File
outbox
=
new
File
(
m_baseDir
,
"outbox"
);
File
from
=
new
File
(
m_baseDir
,
m_dataFile
);
File
to
=
new
File
(
outbox
,
m_dataFile
);
File
fromIndex
=
new
File
(
m_baseDir
,
m_dataFile
+
".idx"
);
File
toIndex
=
new
File
(
outbox
,
m_dataFile
+
".idx"
);
to
.
getParentFile
().
mkdirs
();
Files
.
forDir
().
copyFile
(
from
,
to
);
Files
.
forDir
().
copyFile
(
fromIndex
,
toIndex
);
Files
.
forDir
().
delete
(
from
);
Files
.
forDir
().
delete
(
fromIndex
);
File
parentFile
=
from
.
getParentFile
();
parentFile
.
delete
();
// delete it if empty
parentFile
.
getParentFile
().
delete
();
// delete it if empty
}
@Override
...
...
@@ -65,8 +85,6 @@ public class LocalMessageBucket implements MessageBucket {
@Override
public
MessageTree
findByIndex
(
int
index
)
throws
IOException
{
checkDirty
();
try
{
m_lastAccessTime
=
System
.
currentTimeMillis
();
...
...
@@ -82,6 +100,32 @@ public class LocalMessageBucket implements MessageBucket {
}
}
protected
synchronized
MessageBlock
flushBlock
()
throws
IOException
{
if
(
m_dirty
.
get
())
{
m_out
.
close
();
byte
[]
data
=
m_buf
.
toByteArray
();
try
{
m_block
.
setData
(
data
);
m_blockSize
=
0
;
m_buf
.
reset
();
m_out
=
new
GZIPOutputStream
(
m_buf
);
m_dirty
.
set
(
false
);
return
m_block
;
}
finally
{
m_block
=
new
MessageBlock
(
m_dataFile
);
}
}
return
null
;
}
public
MessageBlockWriter
getWriter
()
{
return
m_writer
;
}
public
double
getCompressionRate
()
{
return
m_rawSize
*
1.0
/
m_dataFile
.
length
();
}
...
...
@@ -99,6 +143,9 @@ public class LocalMessageBucket implements MessageBucket {
m_writer
=
new
MessageBlockWriter
(
file
);
m_reader
=
new
MessageBlockReader
(
file
);
m_block
=
new
MessageBlock
(
m_dataFile
);
m_buf
=
new
ByteArrayOutputStream
(
16384
);
m_out
=
new
GZIPOutputStream
(
m_buf
);
}
public
void
setBaseDir
(
File
baseDir
)
{
...
...
@@ -109,185 +156,25 @@ public class LocalMessageBucket implements MessageBucket {
m_codec
=
codec
;
}
public
ChannelBuffer
getChannelBuf
(
MessageTree
tree
)
{
// int index = MessageId.parse(tree.getMessageId()).getIndex();
ChannelBuffer
buf
=
ChannelBuffers
.
dynamicBuffer
(
8192
);
m_codec
.
encode
(
tree
,
buf
);
return
buf
;
}
public
void
storeChannelBuf
(
int
treeIndex
,
int
size
,
byte
[]
data
)
throws
IOException
{
m_lastAccessTime
=
System
.
currentTimeMillis
();
m_writer
.
writeMessage
(
treeIndex
,
data
);
m_dirty
.
set
(
true
);
m_rawSize
+=
data
.
length
;
}
@Override
public
void
store
(
MessageTree
tree
)
throws
IOException
{
int
index
=
MessageId
.
parse
(
tree
.
getMessageId
()).
getIndex
();
ChannelBuffer
buf
=
ChannelBuffers
.
dynamicBuffer
(
8192
);
public
synchronized
MessageBlock
store
(
MessageTree
tree
)
throws
IOException
{
ChannelBuffer
buf
=
m_bufferManager
.
allocate
();
m_lastAccessTime
=
System
.
currentTimeMillis
();
m_codec
.
encode
(
tree
,
buf
);
buf
.
readInt
();
// get rid of length
int
size
=
buf
.
readableBytes
();
byte
[]
data
=
new
byte
[
size
]
;
MessageId
id
=
MessageId
.
parse
(
tree
.
getMessageId
())
;
buf
.
readBytes
(
data
);
m_lastAccessTime
=
System
.
currentTimeMillis
();
m_writer
.
writeMessage
(
index
,
data
);
m_dirty
.
set
(
true
);
m_rawSize
+=
data
.
length
;
}
static
class
MessageBlockReader
{
private
RandomAccessFile
m_indexFile
;
private
RandomAccessFile
m_dataFile
;
public
MessageBlockReader
(
File
dataFile
)
throws
IOException
{
File
indexFile
=
new
File
(
dataFile
.
getAbsolutePath
()
+
".idx"
);
m_indexFile
=
new
RandomAccessFile
(
indexFile
,
"r"
);
m_dataFile
=
new
RandomAccessFile
(
dataFile
,
"r"
);
}
public
void
close
()
throws
IOException
{
synchronized
(
m_indexFile
)
{
m_indexFile
.
close
();
m_dataFile
.
close
();
}
}
public
byte
[]
readMessage
(
int
index
)
throws
IOException
{
int
blockAddress
;
int
blockOffset
;
byte
[]
buf
;
synchronized
(
m_indexFile
)
{
m_indexFile
.
seek
(
index
*
6
);
blockAddress
=
m_indexFile
.
readInt
();
blockOffset
=
m_indexFile
.
readShort
()
&
0xFFFF
;
}
synchronized
(
m_dataFile
)
{
m_dataFile
.
seek
(
blockAddress
);
buf
=
new
byte
[
m_dataFile
.
readInt
()];
m_dataFile
.
readFully
(
buf
);
}
ByteArrayInputStream
bais
=
new
ByteArrayInputStream
(
buf
);
DataInputStream
in
=
new
DataInputStream
(
new
GZIPInputStream
(
bais
));
try
{
in
.
skip
(
blockOffset
);
int
len
=
in
.
readInt
();
byte
[]
data
=
new
byte
[
len
];
in
.
readFully
(
data
);
return
data
;
}
finally
{
try
{
in
.
close
();
}
catch
(
Exception
e
)
{
// ignore it
}
}
}
}
static
class
MessageBlockWriter
{
private
static
final
int
MAX_BLOCK_SIZE
=
1
<<
16
;
// 64K
private
RandomAccessFile
m_indexFile
;
private
RandomAccessFile
m_dataFile
;
private
int
m_blockAddress
;
private
int
m_blockSize
;
private
ByteArrayOutputStream
m_buf
;
private
DataOutputStream
m_out
;
public
MessageBlockWriter
(
File
dataFile
)
throws
IOException
{
File
indexFile
=
new
File
(
dataFile
.
getAbsolutePath
()
+
".idx"
);
dataFile
.
getParentFile
().
mkdirs
();
m_indexFile
=
new
RandomAccessFile
(
indexFile
,
"rw"
);
m_dataFile
=
new
RandomAccessFile
(
dataFile
,
"rw"
);
m_buf
=
new
ByteArrayOutputStream
(
8192
);
m_out
=
new
DataOutputStream
(
new
GZIPOutputStream
(
m_buf
));
m_blockAddress
=
(
int
)
m_dataFile
.
length
();
m_dataFile
.
seek
(
m_blockAddress
);
// move to end
}
public
synchronized
void
close
()
throws
IOException
{
if
(
m_out
!=
null
)
{
try
{
flushBlock
();
}
finally
{
m_out
.
close
();
m_indexFile
.
close
();
m_dataFile
.
close
();
m_out
=
null
;
}
}
}
protected
synchronized
void
flushBlock
()
throws
IOException
{
m_out
.
close
();
m_blockSize
+=
size
;
m_block
.
addIndex
(
id
.
getIndex
(),
size
);
buf
.
getBytes
(
0
,
m_out
,
size
);
// write buffer and compress it
byte
[]
data
=
m_buf
.
toByteArray
();
if
(
data
.
length
>
0
)
{
m_dataFile
.
writeInt
(
data
.
length
);
m_dataFile
.
write
(
data
);
m_blockAddress
+=
data
.
length
+
4
;
m_blockSize
=
0
;
}
m_buf
.
reset
();
m_out
=
new
DataOutputStream
(
new
GZIPOutputStream
(
m_buf
));
}
public
synchronized
void
writeMessage
(
int
index
,
byte
[]
data
)
throws
IOException
{
if
(
m_blockSize
+
data
.
length
>
MAX_BLOCK_SIZE
)
{
flushBlock
();
}
m_indexFile
.
seek
(
index
*
6
);
m_indexFile
.
writeInt
(
m_blockAddress
);
m_indexFile
.
writeShort
(
m_blockSize
&
0xFFFF
);
m_out
.
writeInt
(
data
.
length
);
m_out
.
write
(
data
);
m_blockSize
+=
data
.
length
+
4
;
if
(
m_blockSize
>=
MAX_BLOCK_SIZE
)
{
return
flushBlock
();
}
else
{
return
null
;
}
}
public
void
archive
()
throws
IOException
{
File
outbox
=
new
File
(
m_baseDir
,
"outbox"
);
File
from
=
new
File
(
m_baseDir
,
m_dataFile
);
File
to
=
new
File
(
outbox
,
m_dataFile
);
File
fromIndex
=
new
File
(
m_baseDir
,
m_dataFile
+
".idx"
);
File
toIndex
=
new
File
(
outbox
,
m_dataFile
+
".idx"
);
to
.
getParentFile
().
mkdirs
();
Files
.
forDir
().
copyFile
(
from
,
to
);
Files
.
forDir
().
copyFile
(
fromIndex
,
toIndex
);
Files
.
forDir
().
delete
(
from
);
Files
.
forDir
().
delete
(
fromIndex
);
File
parentFile
=
from
.
getParentFile
();
parentFile
.
delete
();
// delete it if empty
parentFile
.
getParentFile
().
delete
();
// delete it if empty
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/LocalMessageBucketManager.java
浏览文件 @
d83d0f41
...
...
@@ -7,10 +7,13 @@ import java.util.Date;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.concurrent.BlockingQueue
;
import
java.util.concurrent.LinkedBlockingQueue
;
import
java.util.concurrent.TimeUnit
;
import
java.util.concurrent.locks.LockSupport
;
import
org.codehaus.plexus.personality.plexus.lifecycle.phase.Initializable
;
import
org.codehaus.plexus.personality.plexus.lifecycle.phase.InitializationException
;
import
org.jboss.netty.buffer.ChannelBuffer
;
import
com.dianping.cat.Cat
;
import
com.dianping.cat.configuration.ServerConfigManager
;
...
...
@@ -40,6 +43,8 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
private
Map
<
String
,
LocalMessageBucket
>
m_buckets
=
new
HashMap
<
String
,
LocalMessageBucket
>();
private
BlockingQueue
<
MessageBlock
>
m_blockQueue
=
new
LinkedBlockingQueue
<
MessageBlock
>(
1000
);
public
void
archive
(
long
startTime
)
throws
IOException
{
String
path
=
m_pathBuilder
.
getPath
(
new
Date
(
startTime
),
""
);
List
<
String
>
keys
=
new
ArrayList
<
String
>();
...
...
@@ -98,6 +103,7 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
m_baseDir
=
new
File
(
m_configManager
.
getHdfsLocalBaseDir
(
"dump"
));
}
Threads
.
forGroup
(
"Cat"
).
start
(
new
BlockDumper
());
Threads
.
forGroup
(
"Cat"
).
start
(
new
IdleChecker
());
}
...
...
@@ -141,7 +147,16 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
}
if
(
bucket
!=
null
)
{
MessageTree
tree
=
bucket
.
findById
(
messageId
);
// flush the buffer if have data
MessageBlock
block
=
bucket
.
flushBlock
();
if
(
block
!=
null
)
{
m_blockQueue
.
offer
(
block
);
LockSupport
.
parkNanos
(
50
*
1000
*
1000L
);
// wait 50 ms
}
MessageTree
tree
=
bucket
.
findByIndex
(
id
.
getIndex
());
if
(
tree
!=
null
&&
tree
.
getMessageId
().
equals
(
messageId
))
{
t
.
addData
(
"path"
,
dataFile
);
...
...
@@ -159,6 +174,10 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
t
.
setStatus
(
e
);
cat
.
logError
(
e
);
throw
e
;
}
catch
(
Error
e
)
{
t
.
setStatus
(
e
);
cat
.
logError
(
e
);
throw
e
;
}
finally
{
t
.
complete
();
}
...
...
@@ -168,28 +187,9 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
m_baseDir
=
baseDir
;
}
public
DumpItem
buildStoreMetaInfo
(
MessageTree
tree
)
throws
IOException
{
DumpItem
item
=
new
DumpItem
();
MessageId
messageId
=
MessageId
.
parse
(
tree
.
getMessageId
());
int
index
=
messageId
.
getIndex
();
String
fileName
=
getBucketName
(
messageId
,
tree
);
ChannelBuffer
buf
=
getChannelBuf
(
index
,
fileName
,
tree
);
// get rid of length
buf
.
readInt
();
int
size
=
buf
.
readableBytes
();
byte
[]
data
=
new
byte
[
size
];
buf
.
readBytes
(
data
);
item
.
setFileName
(
fileName
).
setIndex
(
index
).
setSize
(
size
).
setBytes
(
data
);
return
item
;
}
private
String
getBucketName
(
MessageId
id
,
MessageTree
tree
)
throws
IOException
{
@Override
public
void
storeMessage
(
MessageTree
tree
)
throws
IOException
{
MessageId
id
=
MessageId
.
parse
(
tree
.
getMessageId
());
// <callee domain> - <caller domain> - <callee ip>
String
name
=
tree
.
getDomain
()
+
"-"
+
id
.
getDomain
()
+
"-"
+
tree
.
getIpAddress
();
String
dataFile
=
m_pathBuilder
.
getPath
(
new
Date
(
id
.
getTimestamp
()),
name
);
...
...
@@ -201,37 +201,51 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
bucket
.
initialize
(
dataFile
);
m_buckets
.
put
(
dataFile
,
bucket
);
}
return
dataFile
;
}
private
ChannelBuffer
getChannelBuf
(
int
index
,
String
fileName
,
MessageTree
tree
)
throws
IOException
{
LocalMessageBucket
bucket
=
m_buckets
.
get
(
fileName
);
return
bucket
.
getChannelBuf
(
tree
);
}
MessageBlock
block
=
bucket
.
store
(
tree
);
public
void
storeMessage
(
DumpItem
item
)
throws
IOException
{
LocalMessageBucket
bucket
=
m_buckets
.
get
(
item
.
getFileName
());
bucket
.
storeChannelBuf
(
item
.
getIndex
(),
item
.
getSize
(),
item
.
getBytes
());
if
(
block
!=
null
)
{
m_blockQueue
.
offer
(
block
);
}
}
@Override
public
void
storeMessage
(
MessageTree
tree
)
throws
IOException
{
MessageId
id
=
MessageId
.
parse
(
tree
.
getMessageId
());
// <callee domain> - <caller domain> - <callee ip>
String
name
=
tree
.
getDomain
()
+
"-"
+
id
.
getDomain
()
+
"-"
+
tree
.
getIpAddress
();
String
dataFile
=
m_pathBuilder
.
getPath
(
new
Date
(
id
.
getTimestamp
()),
name
);
LocalMessageBucket
bucket
=
m_buckets
.
get
(
dataFile
);
class
BlockDumper
implements
Task
{
private
int
m_errors
;
if
(
bucket
==
null
)
{
bucket
=
(
LocalMessageBucket
)
lookup
(
MessageBucket
.
class
,
LocalMessageBucket
.
ID
);
bucket
.
setBaseDir
(
m_baseDir
);
bucket
.
initialize
(
dataFile
);
m_buckets
.
put
(
dataFile
,
bucket
);
@Override
public
String
getName
()
{
return
"LocalMessageBucketManager-BlockDumper"
;
}
@Override
public
void
run
()
{
try
{
while
(
true
)
{
MessageBlock
block
=
m_blockQueue
.
poll
(
5
,
TimeUnit
.
MILLISECONDS
);
if
(
block
!=
null
)
{
String
dataFile
=
block
.
getDataFile
();
LocalMessageBucket
bucket
=
m_buckets
.
get
(
dataFile
);
try
{
bucket
.
getWriter
().
writeBlock
(
block
);
}
catch
(
Throwable
e
)
{
m_errors
++;
if
(
m_errors
==
1
||
m_errors
%
1000
==
0
)
{
Cat
.
getProducer
().
logError
(
new
RuntimeException
(
"Error when dumping for bucket: "
+
dataFile
+
"."
,
e
));
}
}
}
}
}
catch
(
InterruptedException
e
)
{
// ignore it
}
}
bucket
.
store
(
tree
);
@Override
public
void
shutdown
()
{
}
}
class
IdleChecker
implements
Task
{
...
...
@@ -261,5 +275,4 @@ public class LocalMessageBucketManager extends ContainerHolder implements Messag
public
void
shutdown
()
{
}
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlock.java
0 → 100644
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.util.ArrayList
;
import
java.util.List
;
public
class
MessageBlock
{
private
String
m_dataFile
;
private
byte
[]
m_data
;
private
List
<
Integer
>
m_indexes
=
new
ArrayList
<
Integer
>(
32
);
private
List
<
Integer
>
m_sizes
=
new
ArrayList
<
Integer
>(
32
);
public
MessageBlock
(
String
dataFile
)
{
m_dataFile
=
dataFile
;
}
public
void
addIndex
(
int
index
,
int
size
)
{
m_indexes
.
add
(
index
);
m_sizes
.
add
(
size
);
}
public
int
getBlockSize
()
{
return
m_indexes
.
size
();
}
public
byte
[]
getData
()
{
return
m_data
;
}
public
String
getDataFile
()
{
return
m_dataFile
;
}
public
int
getIndex
(
int
index
)
{
return
m_indexes
.
get
(
index
);
}
public
int
getSize
(
int
index
)
{
return
m_sizes
.
get
(
index
);
}
public
void
setData
(
byte
[]
data
)
{
m_data
=
data
;
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlockReader.java
0 → 100644
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.io.ByteArrayInputStream
;
import
java.io.DataInputStream
;
import
java.io.File
;
import
java.io.IOException
;
import
java.io.RandomAccessFile
;
import
java.util.zip.GZIPInputStream
;
class
MessageBlockReader
{
private
RandomAccessFile
m_indexFile
;
private
RandomAccessFile
m_dataFile
;
public
MessageBlockReader
(
File
dataFile
)
throws
IOException
{
File
indexFile
=
new
File
(
dataFile
.
getAbsolutePath
()
+
".idx"
);
m_indexFile
=
new
RandomAccessFile
(
indexFile
,
"r"
);
m_dataFile
=
new
RandomAccessFile
(
dataFile
,
"r"
);
}
public
void
close
()
throws
IOException
{
synchronized
(
m_indexFile
)
{
m_indexFile
.
close
();
m_dataFile
.
close
();
}
}
public
byte
[]
readMessage
(
int
index
)
throws
IOException
{
int
blockAddress
;
int
blockOffset
;
byte
[]
buf
;
synchronized
(
m_indexFile
)
{
m_indexFile
.
seek
(
index
*
6
);
blockAddress
=
m_indexFile
.
readInt
();
blockOffset
=
m_indexFile
.
readShort
()
&
0xFFFF
;
}
synchronized
(
m_dataFile
)
{
m_dataFile
.
seek
(
blockAddress
);
buf
=
new
byte
[
m_dataFile
.
readInt
()];
m_dataFile
.
readFully
(
buf
);
}
ByteArrayInputStream
bais
=
new
ByteArrayInputStream
(
buf
);
DataInputStream
in
=
new
DataInputStream
(
new
GZIPInputStream
(
bais
));
try
{
in
.
skip
(
blockOffset
);
int
len
=
in
.
readInt
();
byte
[]
data
=
new
byte
[
len
];
in
.
readFully
(
data
);
return
data
;
}
finally
{
try
{
in
.
close
();
}
catch
(
Exception
e
)
{
// ignore it
}
}
}
}
\ No newline at end of file
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBlockWriter.java
0 → 100644
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.io.File
;
import
java.io.IOException
;
import
java.io.RandomAccessFile
;
class
MessageBlockWriter
{
private
RandomAccessFile
m_indexFile
;
private
RandomAccessFile
m_dataFile
;
private
int
m_blockAddress
;
public
MessageBlockWriter
(
File
dataFile
)
throws
IOException
{
File
indexFile
=
new
File
(
dataFile
.
getAbsolutePath
()
+
".idx"
);
dataFile
.
getParentFile
().
mkdirs
();
m_indexFile
=
new
RandomAccessFile
(
indexFile
,
"rw"
);
m_dataFile
=
new
RandomAccessFile
(
dataFile
,
"rw"
);
m_blockAddress
=
(
int
)
m_dataFile
.
length
();
m_dataFile
.
seek
(
m_blockAddress
);
// move to end
}
public
synchronized
void
close
()
throws
IOException
{
m_indexFile
.
close
();
m_dataFile
.
close
();
}
public
synchronized
void
writeBlock
(
MessageBlock
block
)
throws
IOException
{
int
len
=
block
.
getBlockSize
();
byte
[]
data
=
block
.
getData
();
int
blockSize
=
0
;
for
(
int
i
=
0
;
i
<
len
;
i
++)
{
int
seq
=
block
.
getIndex
(
i
);
int
size
=
block
.
getSize
(
i
);
m_indexFile
.
seek
(
seq
*
6
);
m_indexFile
.
writeInt
(
m_blockAddress
);
m_indexFile
.
writeShort
(
blockSize
);
blockSize
+=
size
;
}
m_dataFile
.
writeInt
(
data
.
length
);
m_dataFile
.
write
(
data
);
m_blockAddress
+=
data
.
length
+
4
;
}
}
cat-core/src/main/java/com/dianping/cat/storage/dump/MessageBucket.java
浏览文件 @
d83d0f41
...
...
@@ -15,5 +15,5 @@ public interface MessageBucket {
public
void
initialize
(
String
dataFile
)
throws
IOException
;
public
void
store
(
MessageTree
tree
)
throws
IOException
;
public
MessageBlock
store
(
MessageTree
tree
)
throws
IOException
;
}
cat-core/src/main/resources/META-INF/plexus/components.xml
浏览文件 @
d83d0f41
...
...
@@ -200,6 +200,9 @@
<role>
com.dianping.cat.message.spi.MessageCodec
</role>
<role-hint>
plain-text
</role-hint>
</requirement>
<requirement>
<role>
com.dianping.cat.storage.dump.ChannelBufferManager
</role>
</requirement>
</requirements>
</component>
<component>
...
...
@@ -215,6 +218,10 @@
</requirement>
</requirements>
</component>
<component>
<role>
com.dianping.cat.storage.dump.ChannelBufferManager
</role>
<implementation>
com.dianping.cat.storage.dump.ChannelBufferManager
</implementation>
</component>
<component>
<role>
com.site.initialization.Module
</role>
<role-hint>
cat-core
</role-hint>
...
...
cat-core/src/test/java/com/dianping/cat/storage/dump/LocalMessageBucketManagerTest.java
浏览文件 @
d83d0f41
package
com.dianping.cat.storage.dump
;
import
java.io.IOException
;
import
org.junit.Assert
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
...
...
@@ -71,5 +73,11 @@ public class LocalMessageBucketManagerTest extends ComponentTestCase {
protected
long
getTimestamp
()
{
return
1343532130488L
/
3600
/
1000
;
}
@Override
public
void
initialize
(
String
domain
)
throws
IOException
{
super
.
initialize
(
domain
);
super
.
resetIndex
();
}
}
}
cat-core/src/test/java/com/dianping/cat/storage/dump/LocalMessageBucketTest.java
浏览文件 @
d83d0f41
...
...
@@ -7,6 +7,7 @@ import java.util.Random;
import
org.jboss.netty.buffer.ChannelBuffer
;
import
org.junit.Assert
;
import
org.junit.BeforeClass
;
import
org.junit.Ignore
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
import
org.junit.runners.JUnit4
;
...
...
@@ -19,6 +20,7 @@ import com.dianping.cat.message.spi.internal.DefaultMessageTree;
import
com.site.lookup.ComponentTestCase
;
@RunWith
(
JUnit4
.
class
)
@Ignore
public
class
LocalMessageBucketTest
extends
ComponentTestCase
{
@BeforeClass
public
static
void
beforeClass
()
{
...
...
cat-hadoop/src/main/java/com/dianping/cat/hadoop/hdfs/HdfsMessageBucket.java
浏览文件 @
d83d0f41
...
...
@@ -16,6 +16,7 @@ import com.dianping.cat.message.internal.MessageId;
import
com.dianping.cat.message.spi.MessageCodec
;
import
com.dianping.cat.message.spi.MessageTree
;
import
com.dianping.cat.message.spi.internal.DefaultMessageTree
;
import
com.dianping.cat.storage.dump.MessageBlock
;
import
com.dianping.cat.storage.dump.MessageBucket
;
import
com.site.lookup.annotation.Inject
;
...
...
@@ -75,7 +76,7 @@ public class HdfsMessageBucket implements MessageBucket {
}
@Override
public
void
store
(
MessageTree
tree
)
throws
IOException
{
public
MessageBlock
store
(
MessageTree
tree
)
throws
IOException
{
throw
new
UnsupportedOperationException
(
"Not supported by HDFS!"
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录