Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
weixin_52226395
Paddle
提交
7b9080ef
P
Paddle
项目概览
weixin_52226395
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7b9080ef
编写于
6月 15, 2017
作者:
H
Helin Wang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Implement master client, cgo and Python part
上级
fa5c3f1f
变更
10
显示空白变更内容
内联
并排
Showing
10 changed file
with
124 addition
and
23 deletion
+124
-23
go/master/c/client.go
go/master/c/client.go
+39
-10
go/master/client.go
go/master/client.go
+10
-5
go/master/client_test.go
go/master/client_test.go
+1
-1
go/master/python/.gitignore
go/master/python/.gitignore
+1
-0
go/master/python/build.sh
go/master/python/build.sh
+4
-0
go/master/python/paddle_master/.gitignore
go/master/python/paddle_master/.gitignore
+2
-0
go/master/python/paddle_master/__init__.py
go/master/python/paddle_master/__init__.py
+3
-0
go/master/python/paddle_master/client.py
go/master/python/paddle_master/client.py
+39
-0
go/master/python/setup.py
go/master/python/setup.py
+19
-0
go/pserver/cclient/cclient.go
go/pserver/cclient/cclient.go
+6
-7
未找到文件。
go/master/c/client.go
浏览文件 @
7b9080ef
package
main
/*
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#define PADDLE_MASTER_OK 0
#define PADDLE_MASTER_ERROR -1
typedef int paddle_master_client;
*/
...
...
@@ -14,6 +20,7 @@ import (
"github.com/PaddlePaddle/Paddle/go/master"
)
var
nullPtr
=
unsafe
.
Pointer
(
uintptr
(
0
))
var
mu
sync
.
Mutex
var
handleMap
=
make
(
map
[
C
.
paddle_master_client
]
*
master
.
Client
)
var
curHandle
C
.
paddle_master_client
...
...
@@ -47,17 +54,16 @@ func (a addresser) Address() string {
return
string
(
a
)
}
//paddle_new_master_client
func
paddle_new_master_client
(
addr
*
C
.
char
,
buf_size
C
.
int
)
C
.
paddle_master_client
{
//
export
paddle_new_master_client
func
paddle_new_master_client
(
addr
*
C
.
char
)
C
.
paddle_master_client
{
a
:=
C
.
GoString
(
addr
)
c
:=
master
.
NewClient
(
addresser
(
a
)
,
int
(
buf_size
)
)
c
:=
master
.
NewClient
(
addresser
(
a
))
return
add
(
c
)
}
//export paddle_new_etcd_master_client
func
paddle_new_etcd_master_client
(
etcd_addr
*
C
.
char
)
C
.
paddle_master_client
{
// TODO(helin): fault tolerant master client using etcd.
panic
(
"not implemented."
)
//export paddle_release_master_client
func
paddle_release_master_client
(
client
C
.
paddle_master_client
)
{
remove
(
client
)
}
//export paddle_set_dataset
...
...
@@ -65,17 +71,40 @@ func paddle_set_dataset(client C.paddle_master_client, path **C.char, size C.int
c
:=
get
(
client
)
var
paths
[]
string
for
i
:=
0
;
i
<
int
(
size
);
i
++
{
ptr
:=
(
**
C
.
char
)(
unsafe
.
Pointer
(
uintptr
(
unsafe
.
Pointer
(
path
))
+
uintptr
(
size
)))
ptr
:=
(
**
C
.
char
)(
unsafe
.
Pointer
(
uintptr
(
unsafe
.
Pointer
(
path
))
+
uintptr
(
i
)
*
unsafe
.
Sizeof
(
*
path
)))
str
:=
C
.
GoString
(
*
ptr
)
paths
=
append
(
paths
,
str
)
}
err
:=
c
.
SetDataset
(
paths
)
if
err
!=
nil
{
log
.
Println
(
err
)
return
-
1
return
C
.
PADDLE_MASTER_ERROR
}
return
C
.
PADDLE_MASTER_OK
}
//export paddle_next_record
func
paddle_next_record
(
client
C
.
paddle_master_client
,
record
**
C
.
uchar
)
C
.
int
{
c
:=
get
(
client
)
r
:=
c
.
NextRecord
()
if
len
(
r
)
==
0
{
*
record
=
(
*
C
.
uchar
)(
nullPtr
)
return
0
}
size
:=
C
.
size_t
(
len
(
r
))
*
record
=
(
*
C
.
uchar
)(
C
.
malloc
(
size
))
C
.
memcpy
(
unsafe
.
Pointer
(
*
record
),
unsafe
.
Pointer
(
&
r
[
0
]),
size
)
return
C
.
int
(
size
)
}
//export mem_free
func
mem_free
(
p
unsafe
.
Pointer
)
{
// "free" may be a better name for this function, but doing so
// will cause calling any function of this library from Python
// ctypes hanging.
C
.
free
(
p
)
}
func
main
()
{}
go/master/client.go
浏览文件 @
7b9080ef
...
...
@@ -21,13 +21,10 @@ type Client struct {
}
// NewClient creates a new Client.
//
// bufSize is the record buffer size. NextRecord will read from the
// buffer.
func
NewClient
(
addr
Addresser
,
bufSize
int
)
*
Client
{
func
NewClient
(
addr
Addresser
)
*
Client
{
c
:=
&
Client
{}
c
.
conn
=
connection
.
New
()
c
.
ch
=
make
(
chan
[]
byte
,
bufSize
)
c
.
ch
=
make
(
chan
[]
byte
)
go
c
.
monitorMaster
(
addr
)
go
c
.
getRecords
()
return
c
...
...
@@ -53,11 +50,19 @@ func (c *Client) getRecords() {
c
.
ch
<-
s
.
Record
()
}
if
s
.
Err
()
!=
nil
{
log
.
Println
(
err
,
chunk
.
Path
)
}
err
=
f
.
Close
()
if
err
!=
nil
{
log
.
Println
(
err
)
}
}
// We treat a task as finished whenever the last data
// instance of the task is read. This is not exactly
// correct, but a reasonable approximation.
c
.
taskFinished
(
t
.
ID
)
}
}
...
...
go/master/client_test.go
浏览文件 @
7b9080ef
...
...
@@ -60,7 +60,7 @@ func TestNextRecord(t *testing.T) {
w
.
Close
()
f
.
Close
()
c
:=
master
.
NewClient
(
master
.
TestAddresser
(
fmt
.
Sprintf
(
":%d"
,
p
))
,
10
)
c
:=
master
.
NewClient
(
master
.
TestAddresser
(
fmt
.
Sprintf
(
":%d"
,
p
)))
c
.
SetDataset
([]
string
{
path
})
for
pass
:=
0
;
pass
<
50
;
pass
++
{
...
...
go/master/python/.gitignore
0 → 100644
浏览文件 @
7b9080ef
*.whl
go/master/python/build.sh
0 → 100755
浏览文件 @
7b9080ef
#!/bin/bash
go build
-buildmode
=
c-shared ../c
&&
rm
c.h
&&
mv
c paddle_master/libmaster.so
pip wheel
.
go/master/python/paddle_master/.gitignore
0 → 100644
浏览文件 @
7b9080ef
*.so
*.pyc
go/master/python/paddle_master/__init__.py
0 → 100644
浏览文件 @
7b9080ef
from
client
import
*
__all__
=
[
'client'
]
go/master/python/paddle_master/client.py
0 → 100644
浏览文件 @
7b9080ef
import
ctypes
import
os
path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
"libmaster.so"
)
lib
=
ctypes
.
cdll
.
LoadLibrary
(
path
)
class
client
(
object
):
"""
client is a client to the master server.
"""
def
__init__
(
self
,
addr
,
buf_size
):
self
.
c
=
lib
.
paddle_new_master_client
(
addr
,
buf_size
)
def
close
(
self
):
lib
.
paddle_release_master_client
(
self
.
c
)
self
.
c
=
None
def
set_dataset
(
self
,
paths
):
holder_type
=
ctypes
.
c_char_p
*
len
(
paths
)
holder
=
holder_type
()
print
paths
for
idx
,
path
in
enumerate
(
paths
):
c_ptr
=
ctypes
.
c_char_p
(
path
)
holder
[
idx
]
=
c_ptr
lib
.
paddle_set_dataset
(
self
.
c
,
holder
,
len
(
paths
))
def
next_record
(
self
):
p
=
ctypes
.
c_char_p
()
ret
=
ctypes
.
pointer
(
p
)
size
=
lib
.
paddle_next_record
(
self
.
c
,
ret
)
if
size
==
0
:
# empty record
return
""
record
=
ret
.
contents
.
value
[:
size
]
# memory created from C should be freed.
lib
.
mem_free
(
ret
.
contents
)
return
record
go/master/python/setup.py
0 → 100644
浏览文件 @
7b9080ef
from
setuptools
import
setup
,
Distribution
class
BinaryDistribution
(
Distribution
):
def
has_ext_modules
(
foo
):
return
True
setup
(
name
=
'paddle_master'
,
version
=
'0.1'
,
description
=
'The client of the master server of PaddlePaddle.'
,
url
=
'https://github.com/PaddlePaddle/Paddle/go/master/python'
,
author
=
'PaddlePaddle Authors'
,
author_email
=
'paddle-dev@baidu.com'
,
license
=
'Apache 2.0'
,
packages
=
[
'paddle_master'
],
package_data
=
{
'master'
:
[
'libmaster.so'
],
},
distclass
=
BinaryDistribution
)
go/pserver/cclient/cclient.go
浏览文件 @
7b9080ef
package
main
/*
#include <stdlib.h>
#include <string.h>
typedef enum {
PADDLE_ELEMENT_TYPE_INT32 = 0,
...
...
@@ -223,14 +222,14 @@ func paddle_get_params(client C.paddle_pserver_client, dst **C.paddle_parameter,
if
unsafe
.
Pointer
(
param
)
==
nullPtr
{
log
.
Println
(
"must pre-allocate parameter."
)
return
C
.
PSERVER_ERROR
}
else
{
}
if
unsafe
.
Pointer
(
param
.
content
)
!=
nullPtr
{
if
int
(
param
.
content_len
)
!=
len
(
p
.
Content
)
{
log
.
Printf
(
"the pre-allocated content len does not match parameter content len. Pre-allocated len: %d, returned len: %d"
,
param
.
content_len
,
len
(
p
.
Content
))
return
C
.
PSERVER_ERROR
}
}
}
C
.
memcpy
(
unsafe
.
Pointer
(
param
.
content
),
unsafe
.
Pointer
(
&
p
.
Content
[
0
]),
C
.
size_t
(
len
(
p
.
Content
)))
param
.
content_len
=
C
.
int
(
len
(
p
.
Content
))
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录