Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
FinClip
ligase
提交
a7773d3d
ligase
项目概览
FinClip
/
ligase
通知
33
Star
8
Fork
1
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
ligase
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
a7773d3d
编写于
9月 21, 2017
作者:
M
Mark Haines
提交者:
GitHub
9月 21, 2017
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add context to the partition offset table (#249)
上级
e7cf2ae0
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
40 addition
and
58 deletion
+40
-58
src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go
...x-org/dendrite/clientapi/auth/storage/accounts/storage.go
+2
-12
src/github.com/matrix-org/dendrite/common/consumers.go
src/github.com/matrix-org/dendrite/common/consumers.go
+5
-4
src/github.com/matrix-org/dendrite/common/partition_offset_table.go
....com/matrix-org/dendrite/common/partition_offset_table.go
+28
-7
src/github.com/matrix-org/dendrite/federationsender/storage/storage.go
...m/matrix-org/dendrite/federationsender/storage/storage.go
+0
-10
src/github.com/matrix-org/dendrite/publicroomsapi/storage/storage.go
...com/matrix-org/dendrite/publicroomsapi/storage/storage.go
+2
-12
src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
...hub.com/matrix-org/dendrite/syncapi/storage/syncserver.go
+3
-13
未找到文件。
src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go
浏览文件 @
a7773d3d
...
...
@@ -29,8 +29,8 @@ import (
// Database represents an account database
type
Database
struct
{
db
*
sql
.
DB
partitions
common
.
PartitionOffsetStatements
db
*
sql
.
DB
common
.
PartitionOffsetStatements
accounts
accountsStatements
profiles
profilesStatements
memberships
membershipStatements
...
...
@@ -127,16 +127,6 @@ func (d *Database) CreateAccount(
return
d
.
accounts
.
insertAccount
(
ctx
,
localpart
,
hash
)
}
// PartitionOffsets implements common.PartitionStorer
func
(
d
*
Database
)
PartitionOffsets
(
topic
string
)
([]
common
.
PartitionOffset
,
error
)
{
return
d
.
partitions
.
SelectPartitionOffsets
(
topic
)
}
// SetPartitionOffset implements common.PartitionStorer
func
(
d
*
Database
)
SetPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
{
return
d
.
partitions
.
UpsertPartitionOffset
(
topic
,
partition
,
offset
)
}
// SaveMembership saves the user matching a given localpart as a member of a given
// room. It also stores the ID of the membership event and a flag on whether the user
// is still in the room.
...
...
src/github.com/matrix-org/dendrite/common/consumers.go
浏览文件 @
a7773d3d
...
...
@@ -15,6 +15,7 @@
package
common
import
(
"context"
"fmt"
sarama
"gopkg.in/Shopify/sarama.v1"
...
...
@@ -31,9 +32,9 @@ type PartitionOffset struct {
// A PartitionStorer has the storage APIs needed by the consumer.
type
PartitionStorer
interface
{
// PartitionOffsets returns the offsets the consumer has reached for each partition.
PartitionOffsets
(
topic
string
)
([]
PartitionOffset
,
error
)
PartitionOffsets
(
ctx
context
.
Context
,
topic
string
)
([]
PartitionOffset
,
error
)
// SetPartitionOffset records where the consumer has reached for a partition.
SetPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
SetPartitionOffset
(
ctx
context
.
Context
,
topic
string
,
partition
int32
,
offset
int64
)
error
}
// A ContinualConsumer continually consumes logs even across restarts. It requires a PartitionStorer to
...
...
@@ -75,7 +76,7 @@ func (c *ContinualConsumer) Start() error {
offsets
[
partition
]
=
sarama
.
OffsetOldest
}
storedOffsets
,
err
:=
c
.
PartitionStore
.
PartitionOffsets
(
c
.
Topic
)
storedOffsets
,
err
:=
c
.
PartitionStore
.
PartitionOffsets
(
c
ontext
.
TODO
(),
c
.
Topic
)
if
err
!=
nil
{
return
err
}
...
...
@@ -110,7 +111,7 @@ func (c *ContinualConsumer) consumePartition(pc sarama.PartitionConsumer) {
for
message
:=
range
pc
.
Messages
()
{
msgErr
:=
c
.
ProcessMessage
(
message
)
// Advance our position in the stream so that we will start at the right position after a restart.
if
err
:=
c
.
PartitionStore
.
SetPartitionOffset
(
c
.
Topic
,
message
.
Partition
,
message
.
Offset
);
err
!=
nil
{
if
err
:=
c
.
PartitionStore
.
SetPartitionOffset
(
c
ontext
.
TODO
(),
c
.
Topic
,
message
.
Partition
,
message
.
Offset
);
err
!=
nil
{
panic
(
fmt
.
Errorf
(
"the ContinualConsumer failed to SetPartitionOffset: %s"
,
err
))
}
// Shutdown if we were told to do so.
...
...
src/github.com/matrix-org/dendrite/common/partition_offset_table.go
浏览文件 @
a7773d3d
...
...
@@ -14,8 +14,11 @@
package
common
import
"database/sql"
import
"strings"
import
(
"context"
"database/sql"
"strings"
)
const
partitionOffsetsSchema
=
`
-- The offsets that the server has processed up to.
...
...
@@ -66,9 +69,25 @@ func (s *PartitionOffsetStatements) Prepare(db *sql.DB, prefix string) (err erro
return
}
// SelectPartitionOffsets returns all the partition offsets for the given topic.
func
(
s
*
PartitionOffsetStatements
)
SelectPartitionOffsets
(
topic
string
)
([]
PartitionOffset
,
error
)
{
rows
,
err
:=
s
.
selectPartitionOffsetsStmt
.
Query
(
topic
)
// PartitionOffsets implements PartitionStorer
func
(
s
*
PartitionOffsetStatements
)
PartitionOffsets
(
ctx
context
.
Context
,
topic
string
,
)
([]
PartitionOffset
,
error
)
{
return
s
.
selectPartitionOffsets
(
ctx
,
topic
)
}
// SetPartitionOffset implements PartitionStorer
func
(
s
*
PartitionOffsetStatements
)
SetPartitionOffset
(
ctx
context
.
Context
,
topic
string
,
partition
int32
,
offset
int64
,
)
error
{
return
s
.
upsertPartitionOffset
(
ctx
,
topic
,
partition
,
offset
)
}
// selectPartitionOffsets returns all the partition offsets for the given topic.
func
(
s
*
PartitionOffsetStatements
)
selectPartitionOffsets
(
ctx
context
.
Context
,
topic
string
,
)
([]
PartitionOffset
,
error
)
{
rows
,
err
:=
s
.
selectPartitionOffsetsStmt
.
QueryContext
(
ctx
,
topic
)
if
err
!=
nil
{
return
nil
,
err
}
...
...
@@ -85,7 +104,9 @@ func (s *PartitionOffsetStatements) SelectPartitionOffsets(topic string) ([]Part
}
// UpsertPartitionOffset updates or inserts the partition offset for the given topic.
func
(
s
*
PartitionOffsetStatements
)
UpsertPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
{
_
,
err
:=
s
.
upsertPartitionOffsetStmt
.
Exec
(
topic
,
partition
,
offset
)
func
(
s
*
PartitionOffsetStatements
)
upsertPartitionOffset
(
ctx
context
.
Context
,
topic
string
,
partition
int32
,
offset
int64
,
)
error
{
_
,
err
:=
s
.
upsertPartitionOffsetStmt
.
ExecContext
(
ctx
,
topic
,
partition
,
offset
)
return
err
}
src/github.com/matrix-org/dendrite/federationsender/storage/storage.go
浏览文件 @
a7773d3d
...
...
@@ -61,16 +61,6 @@ func (d *Database) prepare() error {
return
nil
}
// PartitionOffsets implements common.PartitionStorer
func
(
d
*
Database
)
PartitionOffsets
(
topic
string
)
([]
common
.
PartitionOffset
,
error
)
{
return
d
.
SelectPartitionOffsets
(
topic
)
}
// SetPartitionOffset implements common.PartitionStorer
func
(
d
*
Database
)
SetPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
{
return
d
.
UpsertPartitionOffset
(
topic
,
partition
,
offset
)
}
// UpdateRoom updates the joined hosts for a room and returns what the joined
// hosts were before the update.
func
(
d
*
Database
)
UpdateRoom
(
...
...
src/github.com/matrix-org/dendrite/publicroomsapi/storage/storage.go
浏览文件 @
a7773d3d
...
...
@@ -27,8 +27,8 @@ import (
// PublicRoomsServerDatabase represents a public rooms server database.
type
PublicRoomsServerDatabase
struct
{
db
*
sql
.
DB
partitions
common
.
PartitionOffsetStatements
db
*
sql
.
DB
common
.
PartitionOffsetStatements
statements
publicRoomsStatements
}
...
...
@@ -52,16 +52,6 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerData
return
&
PublicRoomsServerDatabase
{
db
,
partitions
,
statements
},
nil
}
// PartitionOffsets implements common.PartitionStorer
func
(
d
*
PublicRoomsServerDatabase
)
PartitionOffsets
(
topic
string
)
([]
common
.
PartitionOffset
,
error
)
{
return
d
.
partitions
.
SelectPartitionOffsets
(
topic
)
}
// SetPartitionOffset implements common.PartitionStorer
func
(
d
*
PublicRoomsServerDatabase
)
SetPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
{
return
d
.
partitions
.
UpsertPartitionOffset
(
topic
,
partition
,
offset
)
}
// GetRoomVisibility returns the room visibility as a boolean: true if the room
// is publicly visible, false if not.
// Returns an error if the retrieval failed.
...
...
src/github.com/matrix-org/dendrite/syncapi/storage/syncserver.go
浏览文件 @
a7773d3d
...
...
@@ -42,8 +42,8 @@ type streamEvent struct {
// SyncServerDatabase represents a sync server database
type
SyncServerDatabase
struct
{
db
*
sql
.
DB
partitions
common
.
PartitionOffsetStatements
db
*
sql
.
DB
common
.
PartitionOffsetStatements
accountData
accountDataStatements
events
outputRoomEventsStatements
roomstate
currentRoomStateStatements
...
...
@@ -57,7 +57,7 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) {
if
d
.
db
,
err
=
sql
.
Open
(
"postgres"
,
dataSourceName
);
err
!=
nil
{
return
nil
,
err
}
if
err
=
d
.
partition
s
.
Prepare
(
d
.
db
,
"syncapi"
);
err
!=
nil
{
if
err
=
d
.
PartitionOffsetStatement
s
.
Prepare
(
d
.
db
,
"syncapi"
);
err
!=
nil
{
return
nil
,
err
}
if
err
=
d
.
accountData
.
prepare
(
d
.
db
);
err
!=
nil
{
...
...
@@ -162,16 +162,6 @@ func (d *SyncServerDatabase) GetStateEvent(
return
d
.
roomstate
.
selectStateEvent
(
ctx
,
evType
,
roomID
,
stateKey
)
}
// PartitionOffsets implements common.PartitionStorer
func
(
d
*
SyncServerDatabase
)
PartitionOffsets
(
topic
string
)
([]
common
.
PartitionOffset
,
error
)
{
return
d
.
partitions
.
SelectPartitionOffsets
(
topic
)
}
// SetPartitionOffset implements common.PartitionStorer
func
(
d
*
SyncServerDatabase
)
SetPartitionOffset
(
topic
string
,
partition
int32
,
offset
int64
)
error
{
return
d
.
partitions
.
UpsertPartitionOffset
(
topic
,
partition
,
offset
)
}
// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet.
func
(
d
*
SyncServerDatabase
)
SyncStreamPosition
(
ctx
context
.
Context
)
(
types
.
StreamPosition
,
error
)
{
return
d
.
syncStreamPositionTx
(
ctx
,
nil
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录