optimize operations cannot be depended by forward or backward node momentum -> elementwise_mul
Created by: kolinwei
开启内存优化后,OCR报如下错误 File "/work/weike/svn/baidu/paddle/test/cts_test/test_ocr_recognition.py", line 180, in test_train_ocr_recognition_crnn_ctc_multi_gpu train_info = executor_network(memory_optimize=True) File "/work/weike/svn/baidu/paddle/test/cts_test/test_ocr_recognition.py", line 97, in executor_network build_strategy=build_strategy) File "/usr/local/lib/python2.7/dist-packages/paddle/fluid/parallel_executor.py", line 166, in init build_strategy, num_trainers, trainer_id) EnforceNotMet: optimize operations cannot be depended by forward or backward node momentum -> elementwise_mul at [/paddle/paddle/fluid/framework/details/multi_devices_graph_pass.cc:273] PaddlePaddle Call Stacks: 0 0x7fa27938e2b6p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 486 1 0x7fa27a43a485p paddle::framework::details::SortOpsAndDelayOptimizeOp(paddle::framework::ir::Graph const&) + 2053 2 0x7fa27a440cc6p paddle::framework::details::MultiDevSSAGraphBuilder::ApplyImpl(std::unique_ptr<paddle::framework::ir::Graph, std::default_deletepaddle::framework::ir::Graph >) const + 86 3 0x7fa27a4d4888p paddle::framework::ir::Pass::Apply(std::unique_ptr<paddle::framework::ir::Graph, std::default_deletepaddle::framework::ir::Graph >) const + 216 4 0x7fa279467a88p paddle::framework::ApplyParallelExecutorPass(paddle::framework::ProgramDesc const&, std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, std::string const&, std::unordered_set<std::string, std::hashstd::string, std::equal_tostd::string, std::allocatorstd::string > const&, std::vector<paddle::framework::Scope*, std::allocatorpaddle::framework::Scope* > const&, bool, paddle::framework::details::BuildStrategy const&, paddle::platform::NCCLContextMap*) + 1400