Adapt learning rate decay with Fluid distributed training
Created by: Yancey1989
2trainers + 2pservers, following is pserver logs:
E0201 06:20:34.945315 5910 listen_and_serv_op.cc:156] run sub program error enforce framework::product(lr_dims) == 1 failed, 0 != 1
Learning rate should have 1 element at [/paddle/paddle/operators/sgd_op.cc:36]
PaddlePaddle Call Stacks:
0 0x7f3a9cecfde7p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 727
1 0x7f3a9d4d4eaap paddle::operators::SGDOp::InferShape(paddle::framework::InferShapeContext*) const + 986
2 0x7f3a9d57602fp paddle::framework::OperatorWithKernel::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 2527
3 0x7f3a9cf95630p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 1504
4 0x7f3a9d51e143p paddle::operators::ListenAndServOp::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 3443
5 0x7f3a9cf95630p paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool) + 1504
6 0x7f3a9ceeeac3p void pybind11::cpp_function::initialize<pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}, void, paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(pybind11::cpp_function::initialize<void, paddle::framework::Executor, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, pybind11::name, pybind11::is_method, pybind11::sibling>(void (paddle::framework::Executor::*)(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool)#1}&&, void (*)(paddle::framework::Executor*, paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call) + 579
7 0x7f3a9ceec754p pybind11::cpp_function::dispatcher(_object*, _object*, _object*) + 1236
8 0x4cad00p PyEval_EvalFrameEx + 28048
9 0x4c2705p PyEval_EvalCodeEx + 597
10 0x4ca088p PyEval_EvalFrameEx + 24856
11 0x4c9d7fp PyEval_EvalFrameEx + 24079
12 0x4c2705p PyEval_EvalCodeEx + 597
13 0x4c24a9p PyEval_EvalCode + 25
14 0x4f19efp
15 0x4ec372p PyRun_FileExFlags + 130
16 0x4eaaf1p PyRun_SimpleFileExFlags + 401
17 0x49e208p Py_Main + 1736
18 0x7f3ae5dd8830p __libc_start_main + 240
19 0x49da59p _start + 41