Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
41af738a
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
41af738a
编写于
6月 12, 2017
作者:
H
Helin Wang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix according to comments
上级
54e8263c
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
37 addition
and
32 deletion
+37
-32
go/connection/conn.go
go/connection/conn.go
+1
-0
go/master/service.go
go/master/service.go
+36
-32
未找到文件。
go/connection/conn.go
浏览文件 @
41af738a
...
...
@@ -62,6 +62,7 @@ func (c *Conn) Connect(addr string) error {
c
.
waitConn
=
nil
}
}
else
{
client
.
Close
()
return
errors
.
New
(
"client already set from a concurrent goroutine"
)
}
...
...
go/master/service.go
浏览文件 @
41af738a
...
...
@@ -11,10 +11,6 @@ import (
"github.com/PaddlePaddle/recordio"
)
const
(
targetTaskCount
=
300
)
// Service is the master server service.
type
Service
struct
{
chunksPerTask
int
...
...
@@ -23,7 +19,7 @@ type Service struct {
ready
chan
struct
{}
mu
sync
.
Mutex
init
Began
bool
init
Done
bool
taskQueues
taskQueues
}
...
...
@@ -104,54 +100,35 @@ func (s *Service) snapshot() error {
return
nil
}
// SetDataset sets dataset to dispatch for the master server.
//
// SetDataset can be call multiple times. But only the first call will
// be honored.
func
(
s
*
Service
)
SetDataset
(
globPaths
[]
string
,
dummy
*
int
)
error
{
if
len
(
globPaths
)
==
0
{
return
errors
.
New
(
"no dataset specified"
)
}
s
.
mu
.
Lock
()
defer
s
.
mu
.
Unlock
()
if
s
.
initBegan
{
// SetDataset already called. All trainer will call
// SetDataset, but we only handle the first one. Treat
// other calls as successful but do nothing.
return
nil
}
s
.
initBegan
=
true
func
getChunks
(
globPaths
[]
string
)
([]
Chunk
,
error
)
{
var
chunks
[]
Chunk
var
paths
[]
string
for
_
,
s
:=
range
globPaths
{
match
,
err
:=
filepath
.
Glob
(
s
)
if
err
!=
nil
{
panic
(
err
)
return
nil
,
err
}
paths
=
append
(
paths
,
match
...
)
}
if
len
(
paths
)
==
0
{
return
errors
.
New
(
"no valid datset specified"
)
return
nil
,
errors
.
New
(
"no valid datset specified"
)
}
for
_
,
path
:=
range
paths
{
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
panic
(
err
)
return
nil
,
err
}
index
,
err
:=
recordio
.
LoadIndex
(
f
)
if
err
!=
nil
{
return
err
return
nil
,
err
}
err
=
f
.
Close
()
if
err
!=
nil
{
return
err
return
nil
,
err
}
count
:=
index
.
NumChunks
()
...
...
@@ -164,14 +141,41 @@ func (s *Service) SetDataset(globPaths []string, dummy *int) error {
}
}
return
chunks
,
nil
}
// SetDataset sets dataset to dispatch for the master server.
//
// SetDataset can be call multiple times. But only the first call will
// be honored.
func
(
s
*
Service
)
SetDataset
(
globPaths
[]
string
,
dummy
*
int
)
error
{
if
len
(
globPaths
)
==
0
{
return
errors
.
New
(
"no dataset specified"
)
}
s
.
mu
.
Lock
()
defer
s
.
mu
.
Unlock
()
if
s
.
initDone
{
// Already initialized. All trainer will call
// SetDataset, but we only handle the first one. Treat
// other calls as successful but do nothing.
return
nil
}
chunks
,
err
:=
getChunks
(
globPaths
)
if
err
!=
nil
{
return
err
}
s
.
taskQueues
.
Todo
=
partition
(
chunks
,
s
.
chunksPerTask
)
err
:
=
s
.
snapshot
()
err
=
s
.
snapshot
()
if
err
!=
nil
{
return
err
}
close
(
s
.
ready
)
s
.
initDone
=
true
return
nil
}
...
...
@@ -193,7 +197,7 @@ func (s *Service) GetTask(dummy int, task *Task) error {
// TODO(helin): client need to retry in this
// error case. Gotcha: RPC client can't
// compare returned error with predefined
// erros like io.EOF. Because interface don't
// erro
r
s like io.EOF. Because interface don't
// have same dynamic value when in different
// process.
return
errors
.
New
(
"no more available task"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录