Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
66c7a076
P
Paddle
项目概览
PaddlePaddle
/
Paddle
12 个月 前同步成功
通知
2292
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
66c7a076
编写于
6月 29, 2021
作者:
T
Thunderbrook
提交者:
GitHub
6月 29, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Remove HeterBox (#33718)
* remove heterbox * remove heterbox
上级
5c514f5e
变更
9
展开全部
显示空白变更内容
内联
并排
Showing
9 changed file
with
8 addition
and
1211 deletion
+8
-1211
paddle/fluid/framework/CMakeLists.txt
paddle/fluid/framework/CMakeLists.txt
+5
-5
paddle/fluid/framework/device_worker.h
paddle/fluid/framework/device_worker.h
+0
-101
paddle/fluid/framework/heterbox_trainer.cc
paddle/fluid/framework/heterbox_trainer.cc
+0
-275
paddle/fluid/framework/heterbox_worker.cc
paddle/fluid/framework/heterbox_worker.cc
+0
-753
paddle/fluid/framework/trainer.h
paddle/fluid/framework/trainer.h
+0
-49
paddle/fluid/framework/trainer_factory.cc
paddle/fluid/framework/trainer_factory.cc
+0
-1
python/paddle/fluid/__init__.py
python/paddle/fluid/__init__.py
+1
-1
python/paddle/fluid/trainer_desc.py
python/paddle/fluid/trainer_desc.py
+1
-25
python/paddle/fluid/trainer_factory.py
python/paddle/fluid/trainer_factory.py
+1
-1
未找到文件。
paddle/fluid/framework/CMakeLists.txt
浏览文件 @
66c7a076
...
...
@@ -261,7 +261,7 @@ if(WITH_DISTRIBUTE)
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
heterxpu_trainer.cc
data_feed.cc device_worker.cc hogwild_worker.cc hetercpu_worker.cc ps_gpu_worker.cc
heterbox_worker.cc heterbox_trainer.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
device_context scope framework_proto trainer_desc_proto glog fs shell
fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer
...
...
@@ -282,7 +282,7 @@ if(WITH_DISTRIBUTE)
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
heterxpu_trainer.cc
data_feed.cc device_worker.cc hogwild_worker.cc hetercpu_worker.cc
heterbox_worker.cc heterbox_trainer.cc
downpour_worker.cc downpour_worker_opt.cc
downpour_worker.cc downpour_worker_opt.cc
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
device_context scope framework_proto data_feed_proto heter_service_proto trainer_desc_proto glog
lod_rank_table fs shell fleet_wrapper heter_wrapper box_wrapper lodtensor_printer feed_fetch_method
...
...
@@ -296,7 +296,7 @@ if(WITH_DISTRIBUTE)
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
heterxpu_trainer.cc
data_feed.cc device_worker.cc hogwild_worker.cc hetercpu_worker.cc ps_gpu_worker.cc
heterbox_worker.cc heterbox_trainer.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
device_context scope framework_proto data_feed_proto heter_service_proto trainer_desc_proto glog
lod_rank_table fs shell fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer feed_fetch_method
...
...
@@ -316,7 +316,7 @@ elseif(WITH_PSLIB)
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
heterxpu_trainer.cc
data_feed.cc device_worker.cc hogwild_worker.cc hetercpu_worker.cc ps_gpu_worker.cc
heterbox_worker.cc heterbox_trainer.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
device_context scope framework_proto data_feed_proto heter_service_proto trainer_desc_proto glog
lod_rank_table fs shell fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer feed_fetch_method
...
...
@@ -326,7 +326,7 @@ else()
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
heterxpu_trainer.cc
data_feed.cc device_worker.cc hogwild_worker.cc hetercpu_worker.cc ps_gpu_worker.cc
heterbox_worker.cc heterbox_trainer.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
ps_gpu_trainer.cc downpour_worker.cc downpour_worker_opt.cc
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
device_context scope framework_proto data_feed_proto heter_service_proto trainer_desc_proto glog
lod_rank_table fs shell fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer feed_fetch_method
...
...
paddle/fluid/framework/device_worker.h
浏览文件 @
66c7a076
...
...
@@ -444,107 +444,6 @@ class HeterCpuWorker : public HogwildWorker {
};
#endif
#if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
class
HeterBoxWorker
:
public
HogwildWorker
{
public:
HeterBoxWorker
()
{}
virtual
~
HeterBoxWorker
()
{}
virtual
void
Initialize
(
const
TrainerDesc
&
desc
);
virtual
void
TrainFiles
();
virtual
void
SetNeedDump
(
bool
need_dump_field
);
virtual
void
SetChannelWriter
(
ChannelObject
<
std
::
string
>*
queue
);
virtual
void
SetWorkerNum
(
int
num
)
{
worker_num_
=
num
;
}
virtual
void
CacheProgram
(
const
ProgramDesc
&
main_program
)
{
new
(
&
program_
)
ProgramDesc
(
main_program
);
}
void
ProduceTasks
()
override
;
virtual
void
SetStream
(
const
gpuStream_t
stream
)
{
copy_stream_
=
stream
;
}
virtual
void
SetEvent
(
const
gpuEvent_t
event
)
{
event_
=
event
;
}
virtual
void
TrainFilesWithProfiler
()
{}
void
ResetStat
();
protected:
std
::
shared_ptr
<
paddle
::
framework
::
FleetWrapper
>
fleet_ptr_
;
void
FillSparseValue
(
std
::
shared_ptr
<
HeterTask
>
task
,
size_t
table_id
);
void
PushGradients
();
void
CollectLabelInfo
(
std
::
shared_ptr
<
HeterTask
>
task
,
size_t
table_id
);
void
AdjustInsWeight
(
std
::
shared_ptr
<
HeterTask
>
task
);
void
DumpParam
();
void
CopySparseTable
();
void
CopyDenseTable
();
void
CopyDenseVars
();
private:
int
mpi_rank_
;
std
::
mutex
mutex_
;
std
::
vector
<
std
::
string
>
send_var_list_
;
int
worker_num_
;
ProgramDesc
program_
;
HeterObjectPool
<
HeterTask
>
object_pool_
;
bool
need_dump_param_
;
std
::
vector
<
std
::
string
>
dump_param_
;
bool
need_to_push_dense_
;
bool
need_dump_field_
;
bool
dump_slot_
;
bool
need_to_push_sparse_
;
std
::
vector
<
std
::
string
>
dump_fields_
;
ChannelWriter
<
std
::
string
>
writer_
;
DownpourWorkerParameter
param_
;
float
scale_datanorm_
;
// just save the value in param_ for easy access
std
::
map
<
uint64_t
,
std
::
string
>
label_var_name_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
sparse_key_names_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
sparse_value_names_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
sparse_grad_names_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
dense_value_names_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
dense_grad_names_
;
platform
::
Place
root_place_
;
// actually pushed feasign of each table
std
::
map
<
uint64_t
,
std
::
vector
<
uint64_t
>>
sparse_push_keys_
;
// skipped ops
std
::
vector
<
std
::
string
>
skip_ops_
;
std
::
vector
<::
std
::
future
<
int32_t
>>
push_sparse_status_
;
std
::
vector
<::
std
::
future
<
int32_t
>>
push_dense_status_
;
// adjust ins weight
AdjustInsWeightConfig
adjust_ins_weight_config_
;
std
::
vector
<
float
>
nid_show_
;
// check nan and inf during training
std
::
vector
<
std
::
string
>
check_nan_var_names_
;
// copy table
CopyTableConfig
copy_table_config_
;
std
::
map
<
uint64_t
,
uint64_t
>
table_dependency_
;
std
::
vector
<
std
::
pair
<
uint64_t
,
uint64_t
>>
copy_sparse_tables_
;
std
::
vector
<
std
::
pair
<
uint64_t
,
uint64_t
>>
copy_dense_tables_
;
std
::
unordered_map
<
uint64_t
,
std
::
unordered_set
<
uint64_t
>>
feasign_set_
;
paddle
::
framework
::
Channel
<
std
::
shared_ptr
<
HeterTask
>>
pull_queue_
;
paddle
::
framework
::
Channel
<
std
::
shared_ptr
<
HeterTask
>>
push_queue_
;
gpuEvent_t
event_
;
gpuStream_t
copy_stream_
;
int
batch_cnt_
{
0
};
std
::
atomic
<
int
>
done_cnt_
{
0
};
double
total_time_
;
double
read_time_
;
double
pack_time_
;
double
pull_sparse_local_time_
;
double
op_all_time_
;
double
xpu_op_time_
;
double
xpu_wait_time_
;
double
cpu_op_time_
;
double
collect_label_time_
;
double
fill_sparse_time_
;
double
push_sparse_time_
;
double
gpu_2_cpu_time_
;
double
cpu_2_gpu_time_
;
uint64_t
total_inst_
;
};
#endif
#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \
(defined PADDLE_WITH_PSLIB)
class
PSGPUWorker
:
public
HogwildWorker
{
...
...
paddle/fluid/framework/heterbox_trainer.cc
已删除
100644 → 0
浏览文件 @
5c514f5e
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <cstdlib>
#include <string>
#include <vector>
#include "io/fs.h"
#include "paddle/fluid/framework/data_feed_factory.h"
#include "paddle/fluid/framework/data_set.h"
#include "paddle/fluid/framework/device_worker_factory.h"
#include "paddle/fluid/framework/fleet/fleet_wrapper.h"
#include "paddle/fluid/framework/trainer.h"
#if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#include "paddle/fluid/platform/cuda_device_guard.h"
#endif
namespace
paddle
{
namespace
framework
{
void
HeterBoxTrainer
::
Initialize
(
const
TrainerDesc
&
trainer_desc
,
Dataset
*
dataset
)
{
thread_num_
=
trainer_desc
.
thread_num
();
param_
=
trainer_desc
.
downpour_param
();
for
(
int
i
=
0
;
i
<
param_
.
dense_table_size
();
++
i
)
{
uint64_t
table_id
=
static_cast
<
uint64_t
>
(
param_
.
dense_table
(
i
).
table_id
());
auto
table
=
param_
.
dense_table
(
i
);
dense_grad_names_
[
table_id
].
resize
(
table
.
dense_grad_name_size
());
for
(
int
j
=
0
;
j
<
table
.
dense_grad_name_size
();
++
j
)
{
dense_grad_names_
[
table_id
][
j
]
=
table
.
dense_grad_name
(
j
);
}
}
RegisterHeterCallback
();
scale_datanorm_
=
trainer_desc
.
scale_datanorm
();
int
place_num
=
trainer_desc
.
worker_places_size
();
const
std
::
vector
<
paddle
::
framework
::
DataFeed
*>
readers
=
dataset
->
GetReaders
();
for
(
int
i
=
0
;
i
<
place_num
;
++
i
)
{
int
num
=
trainer_desc
.
worker_places
(
i
);
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
platform
::
CUDAPlace
place
=
platform
::
CUDAPlace
(
num
);
platform
::
CUDADeviceGuard
guard
(
place
.
device
);
gpuStream_t
stream
;
#ifdef PADDLE_WITH_HIP
PADDLE_ENFORCE_CUDA_SUCCESS
(
hipStreamCreate
(
&
stream
));
#else
PADDLE_ENFORCE_CUDA_SUCCESS
(
cudaStreamCreate
(
&
stream
));
#endif
copy_streams_
.
push_back
(
stream
);
places_
.
push_back
(
place
);
gpuEvent_t
event
;
#ifdef PADDLE_WITH_HIP
PADDLE_ENFORCE_CUDA_SUCCESS
(
hipEventCreateWithFlags
(
&
event
,
hipEventDisableTiming
));
#else
PADDLE_ENFORCE_CUDA_SUCCESS
(
cudaEventCreateWithFlags
(
&
event
,
cudaEventDisableTiming
));
#endif
events_
.
push_back
(
event
);
#endif
#ifdef PADDLE_WITH_XPU
platform
::
XPUPlace
place
=
platform
::
XPUPlace
(
num
);
places_
.
push_back
(
place
);
#endif
}
for
(
int
i
=
0
;
i
<
trainer_desc
.
downpour_param
().
stat_var_names_size
();
i
++
)
{
need_merge_var_names_
.
push_back
(
trainer_desc
.
downpour_param
().
stat_var_names
(
i
));
}
VLOG
(
3
)
<<
"going to initialize pull dense worker"
;
pull_dense_worker_
=
PullDenseWorker
::
GetInstance
();
pull_dense_worker_
->
Initialize
(
trainer_desc
);
VLOG
(
3
)
<<
"initialize pull dense worker"
;
SetDebug
(
trainer_desc
.
debug
());
fleet_ptr_
=
FleetWrapper
::
GetInstance
();
trainer_desc_
=
trainer_desc
;
workers_
.
resize
(
place_num
);
for
(
int
i
=
0
;
i
<
place_num
;
++
i
)
{
workers_
[
i
]
=
DeviceWorkerFactory
::
CreateDeviceWorker
(
trainer_desc
.
device_worker_name
());
workers_
[
i
]
->
SetDeviceIndex
(
i
);
workers_
[
i
]
->
SetDataFeed
(
readers
[
i
]);
workers_
[
i
]
->
Initialize
(
trainer_desc
);
workers_
[
i
]
->
SetWorkerNum
(
place_num
);
}
}
void
HeterBoxTrainer
::
DumpWork
(
int
tid
)
{}
void
HeterBoxTrainer
::
RegisterHeterCallback
()
{
auto
fleet_ptr
=
FleetWrapper
::
GetInstance
();
fleet_ptr
->
RegisterHeterCallback
([
this
](
int
worker
,
int
taskid
)
{
// workers_[worker]->Schedule(taskid);
});
}
void
HeterBoxTrainer
::
InitTrainerEnv
(
const
ProgramDesc
&
main_program
,
const
platform
::
Place
&
place
)
{
for
(
size_t
i
=
0
;
i
<
places_
.
size
();
++
i
)
{
workers_
[
i
]
->
SetPlace
(
places_
[
i
]);
workers_
[
i
]
->
SetStream
(
copy_streams_
[
i
]);
workers_
[
i
]
->
SetEvent
(
events_
[
i
]);
workers_
[
i
]
->
SetReaderPlace
(
platform
::
CPUPlace
());
workers_
[
i
]
->
SetRootScope
(
root_scope_
);
workers_
[
i
]
->
CreateDeviceResource
(
main_program
);
// Program
workers_
[
i
]
->
BindingDataFeedMemory
();
#ifdef PADDLE_WITH_PSLIB
workers_
[
i
]
->
CacheProgram
(
main_program
);
#endif
}
for
(
size_t
num
=
0
;
num
<
places_
.
size
();
++
num
)
{
auto
place
=
places_
[
num
];
Scope
*
scope
=
workers_
[
num
]
->
GetThreadScope
();
auto
stream
=
copy_streams_
[
num
];
auto
event
=
events_
[
num
];
auto
dev_id
=
BOOST_GET_CONST
(
platform
::
CUDAPlace
,
place
).
device
;
platform
::
CUDADeviceGuard
guard
(
dev_id
);
auto
&
block
=
main_program
.
Block
(
0
);
for
(
auto
&
var
:
block
.
AllVars
())
{
if
(
var
->
Persistable
())
{
auto
name
=
var
->
Name
();
Variable
*
root_var
=
root_scope_
->
FindVar
(
name
);
if
(
!
root_var
)
{
continue
;
}
LoDTensor
*
root_tensor
=
root_var
->
GetMutable
<
LoDTensor
>
();
auto
*
ptr
=
scope
->
Var
(
name
);
InitializeVariable
(
ptr
,
proto
::
VarType
::
LOD_TENSOR
);
LoDTensor
*
thread_tensor
=
ptr
->
GetMutable
<
LoDTensor
>
();
#define HeterMemcpyFunc(cpp_type, proto_type) \
do { \
if (root_tensor->type() == proto_type) { \
HeterMemCpy<cpp_type>(thread_tensor, root_tensor, place, stream); \
} \
} while (0)
_ForEachDataType_
(
HeterMemcpyFunc
);
}
}
#ifdef PADDLE_WITH_HIP
PADDLE_ENFORCE_CUDA_SUCCESS
(
hipEventRecord
(
event
,
stream
));
hipEventSynchronize
(
event
);
#else
PADDLE_ENFORCE_CUDA_SUCCESS
(
cudaEventRecord
(
event
,
stream
));
cudaEventSynchronize
(
event
);
#endif
}
place_
=
place
;
}
template
<
typename
T
>
void
HeterBoxTrainer
::
HeterMemCpy
(
LoDTensor
*
thread_tensor
,
LoDTensor
*
root_tensor
,
const
paddle
::
platform
::
Place
&
thread_place
,
gpuStream_t
stream
)
{
T
*
thread_ptr
=
thread_tensor
->
mutable_data
<
T
>
(
root_tensor
->
dims
(),
thread_place
);
T
*
root_ptr
=
root_tensor
->
data
<
T
>
();
if
(
platform
::
is_cpu_place
(
root_tensor
->
place
()))
{
memory
::
Copy
(
BOOST_GET_CONST
(
platform
::
CUDAPlace
,
thread_place
),
thread_ptr
,
platform
::
CPUPlace
(),
root_ptr
,
sizeof
(
T
)
*
root_tensor
->
numel
(),
stream
);
}
else
{
memory
::
Copy
(
BOOST_GET_CONST
(
platform
::
CUDAPlace
,
thread_place
),
thread_ptr
,
BOOST_GET_CONST
(
platform
::
CUDAPlace
,
root_tensor
->
place
()),
root_ptr
,
sizeof
(
T
)
*
root_tensor
->
numel
(),
stream
);
}
}
void
HeterBoxTrainer
::
InitOtherEnv
(
const
ProgramDesc
&
main_program
)
{
pull_dense_worker_
->
SetRootScope
(
root_scope_
);
pull_dense_worker_
->
CreatePinVar
();
for
(
size_t
i
=
0
;
i
<
places_
.
size
();
++
i
)
{
pull_dense_worker_
->
AddThreadScope
(
workers_
[
i
]
->
GetThreadScope
());
pull_dense_worker_
->
AddPlace
(
places_
[
i
]);
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
pull_dense_worker_
->
AddStream
(
copy_streams_
[
i
]);
#endif
}
VLOG
(
3
)
<<
"init other env done."
;
}
void
HeterBoxTrainer
::
Run
()
{
int
pull_thread_num
=
3
*
places_
.
size
();
for
(
size_t
thidx
=
0
;
thidx
<
places_
.
size
();
++
thidx
)
{
workers_
[
thidx
]
->
device_reader_
->
Start
();
std
::
dynamic_pointer_cast
<
paddle
::
framework
::
HeterBoxWorker
>
(
workers_
[
thidx
])
->
ResetStat
();
}
for
(
int
i
=
0
;
i
<
pull_thread_num
;
++
i
)
{
int
worker_id
=
i
%
places_
.
size
();
pull_threads_
.
push_back
(
std
::
thread
(
&
DeviceWorker
::
ProduceTasks
,
workers_
[
worker_id
].
get
()));
}
for
(
size_t
thidx
=
0
;
thidx
<
places_
.
size
();
++
thidx
)
{
threads_
.
push_back
(
std
::
thread
(
&
DeviceWorker
::
TrainFiles
,
workers_
[
thidx
].
get
()));
}
}
template
<
typename
T
>
void
HeterBoxTrainer
::
MergeToRootScope
(
LoDTensor
*
root_tensor
,
LoDTensor
*
tensor
)
{
LoDTensor
tmp_root
;
TensorCopy
(
*
root_tensor
,
platform
::
CPUPlace
(),
&
tmp_root
);
T
*
tmp_root_data
=
tmp_root
.
data
<
T
>
();
LoDTensor
tmp_tensor
;
TensorCopy
(
*
tensor
,
platform
::
CPUPlace
(),
&
tmp_tensor
);
T
*
data
=
tmp_tensor
.
data
<
T
>
();
for
(
int
i
=
0
;
i
<
tmp_tensor
.
numel
();
i
++
)
{
tmp_root_data
[
i
]
+=
data
[
i
];
}
TensorCopy
(
tmp_root
,
platform
::
CPUPlace
(),
root_tensor
);
}
Scope
*
HeterBoxTrainer
::
GetWorkerScope
(
int
thread_id
)
{
return
nullptr
;
}
void
HeterBoxTrainer
::
Finalize
()
{
for
(
auto
&
th
:
pull_threads_
)
{
th
.
join
();
}
for
(
auto
&
th
:
threads_
)
{
th
.
join
();
}
for
(
size_t
i
=
0
;
i
<
need_merge_var_names_
.
size
();
i
++
)
{
Variable
*
root_var
=
root_scope_
->
FindVar
(
need_merge_var_names_
[
i
]);
if
(
root_var
==
nullptr
)
{
continue
;
}
LoDTensor
*
root_tensor
=
root_var
->
GetMutable
<
LoDTensor
>
();
for
(
size_t
j
=
0
;
j
<
places_
.
size
();
j
++
)
{
Scope
*
cur_thread_scope
=
workers_
[
j
]
->
GetThreadScope
();
Variable
*
thread_var
=
cur_thread_scope
->
FindVar
(
need_merge_var_names_
[
i
]);
if
(
thread_var
==
nullptr
)
{
continue
;
}
LoDTensor
*
thread_tensor
=
thread_var
->
GetMutable
<
LoDTensor
>
();
#define MergeCallback(cpp_type, proto_type) \
do { \
if (root_tensor->type() == proto_type) { \
if (thread_tensor->type() != proto_type) { \
VLOG(0) << "Error: thread id=" << j << ", need_merge_var_names_[" << i \
<< "] " << need_merge_var_names_[i] \
<< ", root tensor type=" << root_tensor->type() \
<< ", thread tensor type=" << thread_tensor->type(); \
exit(-1); \
} \
MergeToRootScope<cpp_type>(root_tensor, thread_tensor); \
} \
} while (0)
_ForEachDataType_
(
MergeCallback
);
}
}
pull_dense_worker_
->
MergeDenseParam
();
root_scope_
->
DropKids
();
}
}
// namespace framework
}
// namespace paddle
#endif
paddle/fluid/framework/heterbox_worker.cc
已删除
100644 → 0
浏览文件 @
5c514f5e
此差异已折叠。
点击以展开。
paddle/fluid/framework/trainer.h
浏览文件 @
66c7a076
...
...
@@ -243,55 +243,6 @@ class HeterXpuTrainer : public TrainerBase {
#endif
};
class
HeterBoxTrainer
:
public
TrainerBase
{
public:
HeterBoxTrainer
()
{}
virtual
~
HeterBoxTrainer
()
{}
virtual
void
Initialize
(
const
TrainerDesc
&
trainer_desc
,
Dataset
*
data_set
);
virtual
void
InitTrainerEnv
(
const
ProgramDesc
&
main_program
,
const
platform
::
Place
&
place
);
virtual
void
InitOtherEnv
(
const
ProgramDesc
&
main_program
);
virtual
void
Run
();
virtual
void
Finalize
();
virtual
void
RegisterHeterCallback
();
virtual
void
DumpWork
(
int
tid
);
virtual
Scope
*
GetWorkerScope
(
int
thread_id
);
virtual
void
CacheProgram
(
const
ProgramDesc
&
main_program
)
{
new
(
&
program_
)
ProgramDesc
(
main_program
);
}
virtual
std
::
string
GetDumpPath
(
int
tid
)
{
return
""
;
}
virtual
void
InitDumpEnv
()
{}
template
<
typename
T
>
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
void
HeterMemCpy
(
LoDTensor
*
tensor
,
LoDTensor
*
root_tensor
,
const
paddle
::
platform
::
Place
&
thread_place
,
gpuStream_t
stream
);
#endif
void
CreateThreadParam
(
const
ProgramDesc
&
program
,
int
num
);
template
<
typename
T
>
void
MergeToRootScope
(
LoDTensor
*
root_tensor
,
LoDTensor
*
thread_tensor
);
protected:
DownpourWorkerParameter
param_
;
std
::
map
<
uint64_t
,
std
::
vector
<
std
::
string
>>
dense_grad_names_
;
std
::
vector
<
std
::
string
>
need_merge_var_names_
;
float
scale_datanorm_
;
paddle
::
platform
::
Place
place_
;
ProgramDesc
program_
;
std
::
shared_ptr
<
paddle
::
framework
::
FleetWrapper
>
fleet_ptr_
;
std
::
shared_ptr
<
paddle
::
framework
::
PullDenseWorker
>
pull_dense_worker_
;
std
::
vector
<
std
::
shared_ptr
<
DeviceWorker
>>
workers_
;
std
::
vector
<
platform
::
Place
>
places_
;
// ps-gpu
std
::
vector
<
std
::
thread
>
pull_threads_
;
std
::
vector
<
std
::
thread
>
threads_
;
int
use_ps_gpu_
;
int
thread_num_
;
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
std
::
vector
<
gpuStream_t
>
copy_streams_
;
std
::
vector
<
gpuEvent_t
>
events_
;
#endif
};
#endif
#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \
...
...
paddle/fluid/framework/trainer_factory.cc
浏览文件 @
66c7a076
...
...
@@ -70,7 +70,6 @@ REGISTER_TRAINER_CLASS(DistMultiTrainer);
defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
REGISTER_TRAINER_CLASS
(
HeterXpuTrainer
);
REGISTER_TRAINER_CLASS
(
HeterBoxTrainer
);
#endif
#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \
(defined PADDLE_WITH_PSLIB)
...
...
python/paddle/fluid/__init__.py
浏览文件 @
66c7a076
...
...
@@ -93,7 +93,7 @@ from .dygraph.varbase_patch_methods import monkey_patch_varbase
from
.
import
generator
from
.core
import
_cuda_synchronize
from
.generator
import
Generator
from
.trainer_desc
import
TrainerDesc
,
DistMultiTrainer
,
PipelineTrainer
,
MultiTrainer
,
HeterXpuTrainer
,
HeterBoxTrainer
from
.trainer_desc
import
TrainerDesc
,
DistMultiTrainer
,
PipelineTrainer
,
MultiTrainer
,
HeterXpuTrainer
from
.transpiler
import
HashName
,
RoundRobin
from
.backward
import
append_backward
...
...
python/paddle/fluid/trainer_desc.py
浏览文件 @
66c7a076
...
...
@@ -17,7 +17,7 @@ import sys
import
os
__all__
=
[
'TrainerDesc'
,
'MultiTrainer'
,
'DistMultiTrainer'
,
'PipelineTrainer'
,
'HeterXpuTrainer'
,
'HeterBoxTrainer'
'HeterXpuTrainer'
]
...
...
@@ -346,30 +346,6 @@ class HeterXpuTrainer(TrainerDesc):
self
.
_device_worker
.
_gen_worker_desc
(
self
.
proto_desc
)
class
HeterBoxTrainer
(
TrainerDesc
):
"""
Implement of HeterBoxTrainer.
It's for Distributed training.
"""
def
__init__
(
self
):
super
(
HeterBoxTrainer
,
self
).
__init__
()
pass
def
_set_program
(
self
,
program
):
super
(
HeterBoxTrainer
,
self
).
_set_program
(
program
)
self
.
_program
=
program
def
_gen_trainer_desc
(
self
):
super
(
HeterBoxTrainer
,
self
).
_gen_trainer_desc
()
self
.
proto_desc
.
class_name
=
"HeterBoxTrainer"
if
self
.
_program
==
None
:
raise
RuntimeError
(
"None Program"
)
self
.
_device_worker
.
_set_infer
(
self
.
_infer
)
self
.
_device_worker
.
_set_program
(
self
.
_program
)
self
.
_device_worker
.
_gen_worker_desc
(
self
.
proto_desc
)
class
PSGPUTrainer
(
TrainerDesc
):
"""
Implement of PSGPUTrainer.
...
...
python/paddle/fluid/trainer_factory.py
浏览文件 @
66c7a076
...
...
@@ -22,7 +22,7 @@ from paddle.fluid.log_helper import get_logger
local_logger
=
get_logger
(
__name__
,
logging
.
INFO
,
fmt
=
'%(asctime)s-%(levelname)s: %(message)s'
)
from
.trainer_desc
import
MultiTrainer
,
DistMultiTrainer
,
PipelineTrainer
,
HeterXpuTrainer
,
HeterBoxTrainer
,
PSGPUTrainer
from
.trainer_desc
import
MultiTrainer
,
DistMultiTrainer
,
PipelineTrainer
,
HeterXpuTrainer
,
PSGPUTrainer
from
.device_worker
import
Hogwild
,
DownpourSGD
,
Section
,
DownpourSGDOPT
from
.framework
import
Variable
from
multiprocessing
import
Process
,
Manager
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录