Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wzpei9
Paddle
提交
9b11e17d
P
Paddle
项目概览
wzpei9
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9b11e17d
编写于
5月 25, 2017
作者:
H
Helin Wang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix according to comments
上级
6ce7c8bc
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
45 addition
and
25 deletion
+45
-25
paddle/go/cmd/master/master.go
paddle/go/cmd/master/master.go
+28
-13
paddle/go/cmd/pserver/pserver.go
paddle/go/cmd/pserver/pserver.go
+3
-2
paddle/go/master/service.go
paddle/go/master/service.go
+6
-7
paddle/go/master/service_internal_test.go
paddle/go/master/service_internal_test.go
+8
-3
未找到文件。
paddle/go/cmd/master/master.go
浏览文件 @
9b11e17d
package
main
import
(
"f
lag
"
"f
mt
"
"net"
"net/http"
"net/rpc"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/namsral/flag"
"github.com/PaddlePaddle/Paddle/paddle/go/master"
"github.com/PaddlePaddle/Paddle/paddle/go/recordio"
)
const
(
taskTimeoutDur
=
20
*
time
.
Minute
taskTimeoutMax
=
3
)
func
main
()
{
port
:=
flag
.
Int
(
"p"
,
0
,
"port of the master server"
)
dataset
:=
flag
.
String
(
"d"
,
""
,
"dataset: comma separated path to RecordIO files"
)
faultTolerant
:=
flag
.
Bool
(
"fault-tolerance"
,
false
,
"enable fault tolerance (requires etcd)."
)
port
:=
flag
.
Int
(
"port"
,
8080
,
"port of the master server."
)
dataset
:=
flag
.
String
(
"training_dataset"
,
""
,
"dataset: comma separated path to RecordIO paths, supports golb patterns."
)
faultTolerance
:=
flag
.
Bool
(
"fault_tolerance"
,
false
,
"enable fault tolerance (requires etcd)."
)
taskTimeoutDur
:=
flag
.
Duration
(
"task_timout_dur"
,
20
*
time
.
Minute
,
"task timout duration."
)
taskTimeoutMax
:=
flag
.
Int
(
"task_timeout_max"
,
3
,
"max timtout count for each task before it being declared failed task."
)
chunkPerTask
:=
flag
.
Int
(
"chunk_per_task"
,
10
,
"chunk per task."
)
flag
.
Parse
()
if
*
dataset
==
""
{
panic
(
"no dataset specified."
)
}
if
*
faultToleran
t
{
panic
(
"fault tolerna
t
not implemented."
)
if
*
faultToleran
ce
{
panic
(
"fault tolerna
nce
not implemented."
)
}
var
chunks
[]
master
.
Chunk
paths
:=
strings
.
Split
(
*
dataset
,
","
)
var
paths
[]
string
ss
:=
strings
.
Split
(
*
dataset
,
","
)
fmt
.
Println
(
ss
)
for
_
,
s
:=
range
ss
{
match
,
err
:=
filepath
.
Glob
(
s
)
if
err
!=
nil
{
panic
(
err
)
}
paths
=
append
(
paths
,
match
...
)
}
if
len
(
paths
)
==
0
{
panic
(
"no valid datset specified."
)
}
idx
:=
0
for
_
,
path
:=
range
paths
{
f
,
err
:=
os
.
Open
(
path
)
...
...
@@ -59,7 +74,7 @@ func main() {
}
}
s
:=
master
.
NewService
(
chunks
,
taskTimeoutDur
,
taskTimeoutMax
)
s
:=
master
.
NewService
(
chunks
,
*
chunkPerTask
,
*
taskTimeoutDur
,
*
taskTimeoutMax
)
err
:=
rpc
.
Register
(
s
)
if
err
!=
nil
{
panic
(
err
)
...
...
paddle/go/cmd/pserver/pserver.go
浏览文件 @
9b11e17d
package
main
import
(
"flag"
"net"
"net/http"
"net/rpc"
"strconv"
"github.com/namsral/flag"
"github.com/PaddlePaddle/Paddle/paddle/go/pserver"
)
func
main
()
{
port
:=
flag
.
Int
(
"p"
,
0
,
"port of the pserver"
)
port
:=
flag
.
Int
(
"p
ort
"
,
0
,
"port of the pserver"
)
flag
.
Parse
()
s
:=
pserver
.
NewService
()
...
...
paddle/go/master/service.go
浏览文件 @
9b11e17d
...
...
@@ -34,17 +34,16 @@ func Recover() (*Service, error) {
return
nil
,
nil
}
func
partition
(
chunks
[]
Chunk
,
targetTaskCount
int
)
[]
taskEntry
{
func
partition
(
chunks
[]
Chunk
,
chunksPerTask
int
)
[]
taskEntry
{
id
:=
0
chunkPerTask
:=
len
(
chunks
)
/
targetTaskCount
if
chunkPerTask
<=
0
{
chunkPerTask
=
1
if
chunksPerTask
<=
0
{
chunksPerTask
=
1
}
var
result
[]
taskEntry
var
cur
taskEntry
for
i
,
c
:=
range
chunks
{
if
i
%
chunkPerTask
==
0
&&
len
(
cur
.
Task
.
Chunks
)
>
0
{
if
i
%
chunk
s
PerTask
==
0
&&
len
(
cur
.
Task
.
Chunks
)
>
0
{
cur
.
Task
.
ID
=
id
id
++
result
=
append
(
result
,
cur
)
...
...
@@ -64,13 +63,13 @@ func partition(chunks []Chunk, targetTaskCount int) []taskEntry {
}
// NewService creates a new service.
func
NewService
(
chunks
[]
Chunk
,
timeoutDur
time
.
Duration
,
timeoutMax
int
)
*
Service
{
func
NewService
(
chunks
[]
Chunk
,
chunksPerTask
int
,
timeoutDur
time
.
Duration
,
timeoutMax
int
)
*
Service
{
s
:=
&
Service
{}
s
.
timeoutDur
=
timeoutDur
s
.
timeoutMax
=
timeoutMax
s
.
taskQueues
=
taskQueues
{}
s
.
taskQueues
.
Pending
=
make
(
map
[
int
]
taskEntry
)
s
.
taskQueues
.
Todo
=
partition
(
chunks
,
targetTaskCount
)
s
.
taskQueues
.
Todo
=
partition
(
chunks
,
chunksPerTask
)
return
s
}
...
...
paddle/go/master/service_internal_test.go
浏览文件 @
9b11e17d
...
...
@@ -4,18 +4,23 @@ import "testing"
func
TestPartitionCount
(
t
*
testing
.
T
)
{
cs
:=
make
([]
Chunk
,
100
)
ts
:=
partition
(
cs
,
20
)
ts
:=
partition
(
cs
,
5
)
if
len
(
ts
)
!=
20
{
t
.
Error
(
len
(
ts
))
}
cs
=
make
([]
Chunk
,
101
)
ts
=
partition
(
cs
,
20
)
ts
=
partition
(
cs
,
5
)
if
len
(
ts
)
!=
21
{
t
.
Error
(
len
(
ts
))
}
ts
=
partition
(
cs
,
200
)
ts
=
partition
(
cs
,
1
)
if
len
(
ts
)
!=
101
{
t
.
Error
(
len
(
ts
))
}
ts
=
partition
(
cs
,
0
)
if
len
(
ts
)
!=
101
{
t
.
Error
(
len
(
ts
))
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录