Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleFL
提交
08628d88
P
PaddleFL
项目概览
PaddlePaddle
/
PaddleFL
通知
35
Star
5
Fork
1
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
6
列表
看板
标记
里程碑
合并请求
4
Wiki
3
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleFL
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
6
Issue
6
列表
看板
标记
里程碑
合并请求
4
合并请求
4
Pages
分析
分析
仓库分析
DevOps
Wiki
3
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
08628d88
编写于
1月 13, 2020
作者:
Q
qjing666
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix dataset conflict issue in single machine simulation
上级
e205e518
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
30 addition
and
23 deletion
+30
-23
paddle_fl/core/trainer/fl_trainer.py
paddle_fl/core/trainer/fl_trainer.py
+6
-6
paddle_fl/dataset/femnist.py
paddle_fl/dataset/femnist.py
+22
-15
paddle_fl/examples/femnist_demo/fl_trainer.py
paddle_fl/examples/femnist_demo/fl_trainer.py
+2
-2
未找到文件。
paddle_fl/core/trainer/fl_trainer.py
浏览文件 @
08628d88
paddle_fl/dataset/femnist.py
浏览文件 @
08628d88
...
...
@@ -3,32 +3,34 @@ import os
import
json
import
tarfile
import
random
url
=
"https://paddlefl.bj.bcebos.com/leaf/"
target_path
=
"femnist_data"
tar_path
=
target_path
+
".tar.gz"
print
(
tar_path
)
def
download
(
url
):
def
download
(
url
,
tar_path
):
r
=
requests
.
get
(
url
)
with
open
(
tar_path
,
'wb'
)
as
f
:
f
.
write
(
r
.
content
)
def
extract
(
tar_path
):
def
extract
(
tar_path
,
target_path
):
tar
=
tarfile
.
open
(
tar_path
,
"r:gz"
)
file_names
=
tar
.
getnames
()
for
file_name
in
file_names
:
tar
.
extract
(
file_name
)
tar
.
extract
(
file_name
,
target_path
)
tar
.
close
()
def
train
(
trainer_id
,
inner_step
,
batch_size
,
count_by_step
):
target_path
=
"trainer%d_data"
%
trainer_id
data_path
=
target_path
+
"/femnist_data"
tar_path
=
data_path
+
".tar.gz"
if
not
os
.
path
.
exists
(
target_path
):
os
.
system
(
"mkdir trainer%d_data"
%
trainer_id
)
if
not
os
.
path
.
exists
(
data_path
):
print
(
"Preparing data..."
)
if
not
os
.
path
.
exists
(
tar_path
):
download
(
url
+
tar_path
)
extract
(
tar_path
)
download
(
"https://paddlefl.bj.bcebos.com/leaf/femnist_data.tar.gz"
,
tar_path
)
extract
(
tar_path
,
target_path
)
def
train_data
():
train_file
=
open
(
"./
femnist_data/train/all_data_%d_niid_0_keep_0_train_9.json"
%
trainer_id
,
'r'
)
train_file
=
open
(
"./
trainer%d_data/femnist_data/train/all_data_%d_niid_0_keep_0_train_9.json"
%
(
trainer_id
,
trainer_id
)
,
'r'
)
json_train
=
json
.
load
(
train_file
)
users
=
json_train
[
"users"
]
rand
=
random
.
randrange
(
0
,
len
(
users
))
# random choose a user from each trainer
...
...
@@ -48,13 +50,18 @@ def train(trainer_id,inner_step,batch_size,count_by_step):
return
train_data
def
test
(
trainer_id
,
inner_step
,
batch_size
,
count_by_step
):
target_path
=
"trainer%d_data"
%
trainer_id
data_path
=
target_path
+
"/femnist_data"
tar_path
=
data_path
+
".tar.gz"
if
not
os
.
path
.
exists
(
target_path
):
os
.
system
(
"mkdir trainer%d_data"
%
trainer_id
)
if
not
os
.
path
.
exists
(
data_path
):
print
(
"Preparing data..."
)
if
not
os
.
path
.
exists
(
tar_path
):
download
(
url
+
tar_path
)
extract
(
tar_path
)
download
(
"https://paddlefl.bj.bcebos.com/leaf/femnist_data.tar.gz"
,
tar_path
)
extract
(
tar_path
,
target_path
)
def
test_data
():
test_file
=
open
(
"./
femnist_data/test/all_data_%d_niid_0_keep_0_test_9.json"
%
trainer_id
,
'r'
)
test_file
=
open
(
"./
trainer%d_data/femnist_data/test/all_data_%d_niid_0_keep_0_test_9.json"
%
(
trainer_id
,
trainer_id
)
,
'r'
)
json_test
=
json
.
load
(
test_file
)
users
=
json_test
[
"users"
]
for
user
in
users
:
...
...
paddle_fl/examples/femnist_demo/fl_trainer.py
浏览文件 @
08628d88
...
...
@@ -40,7 +40,7 @@ def train_test(train_test_program, train_test_feed, train_test_reader):
epoch_id
=
0
step
=
0
epoch
=
3000
count_by_step
=
Tru
e
count_by_step
=
Fals
e
if
count_by_step
:
output_folder
=
"model_node%d"
%
trainer_id
else
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录