Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
4f5b68d6
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4f5b68d6
编写于
7月 14, 2014
作者:
Y
Yingjun Wu
提交者:
Stephan Ewen
8月 18, 2014
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[streaming] update internal state abstraction
上级
68ebfc48
变更
12
隐藏空白更改
内联
并排
Showing
12 changed file
with
156 addition
and
80 deletion
+156
-80
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamrecord/StreamRecord.java
...stratosphere/streaming/api/streamrecord/StreamRecord.java
+69
-18
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/window/wordcount/WindowWordCountCounter.java
...ing/examples/window/wordcount/WindowWordCountCounter.java
+10
-6
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountKvCounter.java
...here/streaming/examples/wordcount/WordCountKvCounter.java
+2
-2
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/LogTableState.java
...n/java/eu/stratosphere/streaming/state/LogTableState.java
+5
-0
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/LogTableStateIterator.java
...u/stratosphere/streaming/state/LogTableStateIterator.java
+5
-0
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/MutableTableState.java
...va/eu/stratosphere/streaming/state/MutableTableState.java
+15
-3
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/MutableTableStateIterator.java
...ratosphere/streaming/state/MutableTableStateIterator.java
+2
-2
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/TableState.java
...main/java/eu/stratosphere/streaming/state/TableState.java
+4
-2
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/TableStateIterator.java
...a/eu/stratosphere/streaming/state/TableStateIterator.java
+1
-1
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/WindowState.java
...ain/java/eu/stratosphere/streaming/state/WindowState.java
+40
-41
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/WindowStateIterator.java
.../eu/stratosphere/streaming/state/WindowStateIterator.java
+1
-3
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/streamrecord/StreamRecordTest.java
...tosphere/streaming/api/streamrecord/StreamRecordTest.java
+2
-2
未找到文件。
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamrecord/StreamRecord.java
浏览文件 @
4f5b68d6
...
...
@@ -78,25 +78,31 @@ public class StreamRecord implements IOReadableWritable, Serializable {
Tuple12
.
class
,
Tuple13
.
class
,
Tuple14
.
class
,
Tuple15
.
class
,
Tuple16
.
class
,
Tuple17
.
class
,
Tuple18
.
class
,
Tuple19
.
class
,
Tuple20
.
class
,
Tuple21
.
class
,
Tuple22
.
class
};
// TODO implement equals, clone
/**
* Creates a new empty instance for read
*/
public
StreamRecord
()
{
}
public
StreamRecord
()
{}
public
StreamRecord
(
int
numOfFields
)
{
this
.
numOfFields
=
numOfFields
;
this
.
numOfTuples
=
0
;
tupleBatch
=
new
ArrayList
<
Tuple
>();
}
public
StreamRecord
(
int
numOfFields
,
int
batchSize
)
{
this
.
numOfFields
=
numOfFields
;
this
.
numOfTuples
=
0
;
tupleBatch
=
new
ArrayList
<
Tuple
>(
batchSize
);
}
public
StreamRecord
(
StreamRecord
record
)
{
this
.
numOfFields
=
record
.
getNumOfFields
();
this
.
numOfTuples
=
0
;
tupleBatch
=
new
ArrayList
<
Tuple
>();
this
.
uid
=
new
UID
(
Arrays
.
copyOf
(
record
.
getId
().
getId
(),
20
));
for
(
int
i
=
0
;
i
<
record
.
getNumOfTuples
();
++
i
)
{
this
.
tupleBatch
.
add
(
copyTuple
(
record
.
getTuple
(
i
)));
}
}
/**
...
...
@@ -127,6 +133,10 @@ public class StreamRecord implements IOReadableWritable, Serializable {
this
(
tuple
,
1
);
}
public
boolean
isEmpty
(){
return
(
this
.
numOfTuples
==
0
);
}
/**
* @return Number of fields in the tuples
*/
...
...
@@ -200,19 +210,20 @@ public class StreamRecord implements IOReadableWritable, Serializable {
}
}
public
Object
getFieldFast
(
int
tupleNumber
,
int
fieldNumber
)
throws
NoSuchTupleException
,
NoSuchFieldException
{
Tuple
tuple
;
try
{
tuple
=
tupleBatch
.
get
(
tupleNumber
);
}
catch
(
IndexOutOfBoundsException
e
)
{
throw
(
new
NoSuchTupleException
());
}
try
{
return
tuple
.
getFieldFast
(
fieldNumber
);
}
catch
(
IndexOutOfBoundsException
e
)
{
throw
(
new
NoSuchFieldException
());
}
}
//I haven't seen any difference between getField() and getFieldFast...
// public Object getFieldFast(int tupleNumber, int fieldNumber) throws NoSuchTupleException, NoSuchFieldException {
// Tuple tuple;
// try {
// tuple = tupleBatch.get(tupleNumber);
// } catch (IndexOutOfBoundsException e) {
// throw (new NoSuchTupleException());
// }
// try {
// return tuple.getFieldFast(fieldNumber);
// } catch (IndexOutOfBoundsException e) {
// throw (new NoSuchFieldException());
// }
// }
/**
* Get a Boolean from the given field of the first Tuple of the batch
...
...
@@ -901,6 +912,35 @@ public class StreamRecord implements IOReadableWritable, Serializable {
throw
new
TupleSizeMismatchException
();
}
}
/**
* Checks if the number of fields are equal to the batch field size then
* adds the shadow copy of Tuple to the end of the batch
*
* @param tuple
* Tuple to be added as the next record of the batch
*/
public
void
addShadowTuple
(
Tuple
tuple
)
throws
TupleSizeMismatchException
{
addShadowTuple
(
numOfTuples
,
tuple
);
}
/**
* Checks if the number of fields are equal to the batch field size then
* inserts the shadow copy of Tuple to the given position into the recordbatch
*
* @param index
* Position of the added tuple
* @param tuple
* Tuple to be added as the next record of the batch
*/
public
void
addShadowTuple
(
int
index
,
Tuple
tuple
)
throws
TupleSizeMismatchException
{
if
(
tuple
.
getArity
()
==
numOfFields
)
{
tupleBatch
.
add
(
index
,
tuple
);
numOfTuples
++;
}
else
{
throw
new
TupleSizeMismatchException
();
}
}
public
StreamRecord
copySerialized
()
{
...
...
@@ -959,6 +999,17 @@ public class StreamRecord implements IOReadableWritable, Serializable {
return
newTuple
;
}
/**
* copy tuples from the given record and append them to the end.
*
* @param record
*/
public
void
appendRecord
(
StreamRecord
record
){
for
(
int
i
=
0
;
i
<
record
.
getNumOfTuples
();
++
i
){
this
.
addTuple
(
record
.
getTuple
(
i
));
}
}
/**
* Converts tuple field types to a byte array
*
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/window/wordcount/WindowWordCountCounter.java
浏览文件 @
4f5b68d6
...
...
@@ -18,16 +18,18 @@ package eu.stratosphere.streaming.examples.window.wordcount;
import
eu.stratosphere.api.java.tuple.Tuple3
;
import
eu.stratosphere.streaming.api.invokable.UserTaskInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.state.Mutable
Internal
State
;
import
eu.stratosphere.streaming.state.Window
Internal
State
;
import
eu.stratosphere.streaming.state.Mutable
Table
State
;
import
eu.stratosphere.streaming.state.WindowState
;
public
class
WindowWordCountCounter
extends
UserTaskInvokable
{
private
int
windowSize
;
private
int
slidingStep
;
private
int
computeGranularity
;
private
int
windowFieldId
;
private
Window
Internal
State
<
Integer
>
window
;
private
Mutable
Internal
State
<
String
,
Integer
>
wordCounts
;
private
WindowState
<
Integer
>
window
;
private
Mutable
Table
State
<
String
,
Integer
>
wordCounts
;
private
String
word
=
""
;
private
Integer
count
=
0
;
...
...
@@ -38,8 +40,10 @@ public class WindowWordCountCounter extends UserTaskInvokable {
public
WindowWordCountCounter
()
{
windowSize
=
100
;
slidingStep
=
20
;
window
=
new
WindowInternalState
<
Integer
>(
windowSize
,
slidingStep
);
wordCounts
=
new
MutableInternalState
<
String
,
Integer
>();
computeGranularity
=
10
;
windowFieldId
=
2
;
window
=
new
WindowState
<
Integer
>(
windowSize
,
slidingStep
,
computeGranularity
,
windowFieldId
);
wordCounts
=
new
MutableTableState
<
String
,
Integer
>();
}
private
void
incrementCompute
(
StreamRecord
record
)
{
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountKvCounter.java
浏览文件 @
4f5b68d6
...
...
@@ -18,12 +18,12 @@ package eu.stratosphere.streaming.examples.wordcount;
import
eu.stratosphere.api.java.tuple.Tuple2
;
import
eu.stratosphere.streaming.api.invokable.UserTaskInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.state.Mutable
Internal
State
;
import
eu.stratosphere.streaming.state.Mutable
Table
State
;
import
eu.stratosphere.streaming.util.PerformanceCounter
;
public
class
WordCountKvCounter
extends
UserTaskInvokable
{
private
Mutable
InternalState
<
String
,
Integer
>
wordCounts
=
new
MutableInternal
State
<
String
,
Integer
>();
private
Mutable
TableState
<
String
,
Integer
>
wordCounts
=
new
MutableTable
State
<
String
,
Integer
>();
private
String
word
=
""
;
private
Integer
count
=
0
;
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/LogTableState.java
0 → 100644
浏览文件 @
4f5b68d6
package
eu.stratosphere.streaming.state
;
public
class
LogTableState
{
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/LogTableStateIterator.java
0 → 100644
浏览文件 @
4f5b68d6
package
eu.stratosphere.streaming.state
;
public
class
LogTableStateIterator
{
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/Mutable
Internal
State.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/Mutable
Table
State.java
浏览文件 @
4f5b68d6
...
...
@@ -21,7 +21,7 @@ import java.util.Map;
/**
* The most general internal state that stores data in a mutable map.
*/
public
class
Mutable
InternalState
<
K
,
V
>
implements
Internal
State
<
K
,
V
>
{
public
class
Mutable
TableState
<
K
,
V
>
implements
Table
State
<
K
,
V
>
{
private
Map
<
K
,
V
>
state
=
new
LinkedHashMap
<
K
,
V
>();
@Override
...
...
@@ -49,9 +49,21 @@ public class MutableInternalState<K, V> implements InternalState<K, V> {
}
@Override
public
StateIterator
<
K
,
V
>
getIterator
()
{
public
Table
StateIterator
<
K
,
V
>
getIterator
()
{
// TODO Auto-generated method stub
return
new
MutableStateIterator
<
K
,
V
>(
state
.
entrySet
().
iterator
());
return
new
MutableTableStateIterator
<
K
,
V
>(
state
.
entrySet
().
iterator
());
}
@Override
public
String
serialize
()
{
// TODO Auto-generated method stub
return
null
;
}
@Override
public
void
deserialize
(
String
str
)
{
// TODO Auto-generated method stub
}
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/MutableStateIterator.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/Mutable
Table
StateIterator.java
浏览文件 @
4f5b68d6
...
...
@@ -20,10 +20,10 @@ import java.util.Map.Entry;
import
eu.stratosphere.api.java.tuple.Tuple2
;
public
class
Mutable
StateIterator
<
K
,
V
>
implements
StateIterator
<
K
,
V
>{
public
class
Mutable
TableStateIterator
<
K
,
V
>
implements
Table
StateIterator
<
K
,
V
>{
private
Iterator
<
Entry
<
K
,
V
>>
iterator
;
public
MutableStateIterator
(
Iterator
<
Entry
<
K
,
V
>>
iter
){
public
Mutable
Table
StateIterator
(
Iterator
<
Entry
<
K
,
V
>>
iter
){
iterator
=
iter
;
}
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/
Internal
State.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/
Table
State.java
浏览文件 @
4f5b68d6
...
...
@@ -18,10 +18,12 @@ package eu.stratosphere.streaming.state;
/**
* An internal state interface that supports stateful operator.
*/
public
interface
Internal
State
<
K
,
V
>
{
public
interface
Table
State
<
K
,
V
>
{
public
void
put
(
K
key
,
V
value
);
public
V
get
(
K
key
);
public
void
delete
(
K
key
);
public
boolean
containsKey
(
K
key
);
public
StateIterator
<
K
,
V
>
getIterator
();
public
String
serialize
();
public
void
deserialize
(
String
str
);
public
TableStateIterator
<
K
,
V
>
getIterator
();
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/StateIterator.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/
Table
StateIterator.java
浏览文件 @
4f5b68d6
...
...
@@ -20,7 +20,7 @@ import eu.stratosphere.api.java.tuple.Tuple2;
/**
* the iterator for internal states.
*/
public
interface
StateIterator
<
K
,
V
>{
public
interface
Table
StateIterator
<
K
,
V
>{
public
boolean
hasNext
();
public
Tuple2
<
K
,
V
>
next
();
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/Window
Internal
State.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/WindowState.java
浏览文件 @
4f5b68d6
...
...
@@ -25,26 +25,55 @@ import eu.stratosphere.streaming.api.streamrecord.StreamRecord;
* compose time based window operator by extending this class by splitting the
* stream into multiple mini batches.
*/
public
class
WindowInternalState
<
K
>
implements
InternalState
<
K
,
StreamRecord
>
{
public
class
WindowState
<
K
>
{
private
int
windowSize
;
private
int
slidingStep
;
private
int
computeGranularity
;
private
int
windowFieldId
;
private
int
initTimestamp
;
private
int
nextTimestamp
;
private
int
currentRecordNum
;
private
int
fullRecordNum
;
private
int
slideRecordNum
;
CircularFifoBuffer
buffer
;
public
WindowInternalState
(
int
windowSize
,
int
slidingStep
)
{
currentRecordNum
=
0
;
fullRecordNum
=
windowSize
;
slideRecordNum
=
slidingStep
;
buffer
=
new
CircularFifoBuffer
(
windowSize
);
StreamRecord
tempRecord
;
public
WindowState
(
int
windowSize
,
int
slidingStep
,
int
computeGranularity
,
int
windowFieldId
)
{
this
.
windowSize
=
windowSize
;
this
.
slidingStep
=
slidingStep
;
this
.
computeGranularity
=
computeGranularity
;
this
.
windowFieldId
=
windowFieldId
;
this
.
initTimestamp
=
-
1
;
this
.
nextTimestamp
=
-
1
;
this
.
currentRecordNum
=
0
;
//here we assume that windowSize and slidingStep is divisible by computeGranularity.
this
.
fullRecordNum
=
windowSize
/
computeGranularity
;
this
.
slideRecordNum
=
slidingStep
/
computeGranularity
;
this
.
buffer
=
new
CircularFifoBuffer
(
fullRecordNum
);
}
public
void
pushBack
(
StreamRecord
records
)
{
buffer
.
add
(
records
);
currentRecordNum
+=
1
;
public
void
pushBack
(
StreamRecord
record
)
{
if
(
initTimestamp
==
-
1
){
initTimestamp
=
record
.
getTuple
(
0
).
getField
(
windowFieldId
);
nextTimestamp
=
initTimestamp
+
computeGranularity
;
tempRecord
=
new
StreamRecord
(
record
.
getNumOfFields
());
}
for
(
int
i
=
0
;
i
<
record
.
getNumOfTuples
();
++
i
)
{
while
((
Integer
)
record
.
getTuple
(
i
).
getField
(
windowFieldId
)
>
nextTimestamp
){
buffer
.
add
(
tempRecord
);
currentRecordNum
+=
1
;
tempRecord
=
new
StreamRecord
(
record
.
getNumOfFields
());
}
tempRecord
.
addShadowTuple
(
record
.
getTuple
(
i
));
}
}
public
StreamRecord
popFront
()
{
StreamRecord
frontRecord
=
(
StreamRecord
)
buffer
.
get
();
StreamRecord
frontRecord
=
(
StreamRecord
)
buffer
.
get
();
buffer
.
remove
();
return
frontRecord
;
}
...
...
@@ -61,34 +90,4 @@ public class WindowInternalState<K> implements InternalState<K, StreamRecord> {
return
false
;
}
@Override
public
void
put
(
K
key
,
StreamRecord
value
)
{
// TODO Auto-generated method stub
}
@Override
public
StreamRecord
get
(
K
key
)
{
// TODO Auto-generated method stub
return
null
;
}
@Override
public
void
delete
(
K
key
)
{
// TODO Auto-generated method stub
}
@Override
public
boolean
containsKey
(
K
key
)
{
// TODO Auto-generated method stub
return
false
;
}
@Override
public
StateIterator
<
K
,
StreamRecord
>
getIterator
()
{
// TODO Auto-generated method stub
return
null
;
}
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/state/WindowStateIterator.java
浏览文件 @
4f5b68d6
...
...
@@ -18,15 +18,13 @@ package eu.stratosphere.streaming.state;
import
eu.stratosphere.api.java.tuple.Tuple2
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
public
class
WindowStateIterator
<
K
>
implements
StateIterator
<
K
,
StreamRecord
>
{
public
class
WindowStateIterator
<
K
>{
@Override
public
boolean
hasNext
()
{
// TODO Auto-generated method stub
return
false
;
}
@Override
public
Tuple2
<
K
,
StreamRecord
>
next
()
{
// TODO Auto-generated method stub
return
null
;
...
...
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/streamrecord/StreamRecordTest.java
浏览文件 @
4f5b68d6
...
...
@@ -241,7 +241,7 @@ public class StreamRecordTest {
t
=
System
.
nanoTime
();
for
(
int
i
=
0
;
i
<
ITERATION
;
i
++)
{
record
.
getField
Fast
(
0
,
i
%
4
);
record
.
getField
(
0
,
i
%
4
);
}
t2
=
System
.
nanoTime
()
-
t
;
System
.
out
.
println
(
"getFieldFast:\t"
+
t2
+
" ns"
);
...
...
@@ -262,7 +262,7 @@ public class StreamRecordTest {
t
=
System
.
nanoTime
();
for
(
int
i
=
0
;
i
<
ITERATION
;
i
++)
{
record20
.
getField
Fast
(
0
,
i
%
20
);
record20
.
getField
(
0
,
i
%
20
);
}
t2
=
System
.
nanoTime
()
-
t
;
System
.
out
.
println
(
"getFieldFast:\t"
+
t2
+
" ns"
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录