Created by: abcdvzz

----------- Configuration Arguments ----------- batch_size: 16 data_dir: data epoc_num: 160 learning_rate: 0.001 mean_BGR: 104., 117., 123. model_save_dir: output parallel: True pretrained_model: vgg_ilsvrc_16_fc_reduced resize_h: 640 resize_w: 640 use_gpu: True use_pyramidbox: True with_mem_opt: True

W1209 20:38:51.853999 28655 device_context.cc:213] Please NOTE: device: 0, CUDA Capability: 61, Driver Version: 9.2, Runtime Version: 9.0 W1209 20:38:51.854032 28655 device_context.cc:220] device: 0, cuDNN Version: 7.0. W1209 20:38:52.456357 28655 dynamic_loader.cc:101] Can not find library: libnccl.so. Please try to add the lib path to LD_LIBRARY_PATH. Traceback (most recent call last): File "train.py", line 233, in train(args, config, train_parameters, train_file_list) File "train.py", line 162, in train loss_name=loss.name) File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/parallel_executor.py", line 176, in init build_strategy, num_trainers, trainer_id) paddle.fluid.core.EnforceNotMet: Failed to find dynamic library: libnccl.so ( libcudart.so.9.2: cannot open shared object file: No such file or directory ) Please specify its path correctly using following ways: Method. set environment variable LD_LIBRARY_PATH on Linux or DYLD_LIBRARY_PATH on Mac OS. For instance, issue command: export LD_LIBRARY_PATH=... Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is impossible unless System Integrity Protection (SIP) is disabled. at [/paddle/paddle/fluid/platform/dynload/dynamic_loader.cc:157] PaddlePaddle Call Stacks: 0 0x7f400bcba626p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 486 1 0x7f400d5ae12ep paddle::platform::dynload::GetNCCLDsoHandle() + 1822 2 0x7f400bdb2099p void std::once_call_impl<std::Bind_simple<decltype (ncclCommInitAll({parm#1}...)) paddle::platform::dynload::DynLoad__ncclCommInitAll::operator()<ncclComm**, int, int*>(ncclComm**, int, int*)::{lambda()#1 (closed)} ()> >() + 9 3 0x7f407ae4ea99p 4 0x7f400bdb575bp paddle::platform::NCCLContextMap::NCCLContextMap(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, ncclUniqueId*, unsigned long, unsigned long) + 2235 5 0x7f400bdb109ep paddle::framework::ParallelExecutor::ParallelExecutor(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, std::unordered_set<std::string, std::hashstd::string, std::equal_tostd::string, std::allocatorstd::string > const&, std::unordered_set<std::string, std::hashstd::string, std::equal_tostd::string, std::allocatorstd::string > const&, paddle::framework::ProgramDesc const&, std::string const&, paddle::framework::Scope*, std::vector<paddle::framework::Scope*, std::allocatorpaddle::framework::Scope* > const&, paddle::framework::details::ExecutionStrategy const&, paddle::framework::details::BuildStrategy const&, unsigned long, unsigned long) + 3854 6 0x7f400bd1340cp 7 0x7f400bd134fep 8 0x7f400bce238ep 9 0x4ea137p PyCFunction_Call + 119 10 0x5c20e7p PyObject_Call + 71 11 0x4fbfcep 12 0x5c20e7p PyObject_Call + 71 13 0x574c19p 14 0x57f58cp 15 0x5c20e7p PyObject_Call + 71 16 0x53b656p PyEval_EvalFrameEx + 20182 17 0x540b0bp PyEval_EvalCodeEx + 315 18 0x4ec3f7p 19 0x5c20e7p PyObject_Call + 71 20 0x4fbfcep 21 0x5c20e7p PyObject_Call + 71 22 0x574c19p 23 0x57f58cp 24 0x5c20e7p PyObject_Call + 71 25 0x53b656p PyEval_EvalFrameEx + 20182 26 0x5401efp 27 0x53bc93p PyEval_EvalFrameEx + 21779 28 0x53fc97p 29 0x5409bfp PyEval_EvalCode + 31 30 0x60cb42p 31 0x60efeap PyRun_FileExFlags + 154 32 0x60f7dcp PyRun_SimpleFileExFlags + 444 33 0x640256p Py_Main + 1110 34 0x4d0001p main + 225 35 0x7f407aa96830p libc_start_main + 240 36 0x5d6999p start + 41 at [/paddle/paddle/fluid/platform/nccl_helper.h:108] PaddlePaddle Call Stacks: 0 0x7f400bcba626p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 486 1 0x7f400bdb5961p paddle::platform::NCCLContextMap::NCCLContextMap(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, ncclUniqueId*, unsigned long, unsigned long) + 2753 2 0x7f400bdb109ep paddle::framework::ParallelExecutor::ParallelExecutor(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, std::unordered_set<std::string, std::hashstd::string, std::equal_tostd::string, std::allocatorstd::string > const&, std::unordered_set<std::string, std::hashstd::string, std::equal_tostd::string, std::allocatorstd::string > const&, paddle::framework::ProgramDesc const&, std::string const&, paddle::framework::Scope*, std::vector<paddle::framework::Scope*, std::allocatorpaddle::framework::Scope* > const&, paddle::framework::details::ExecutionStrategy const&, paddle::framework::details::BuildStrategy const&, unsigned long, unsigned long) + 3854 3 0x7f400bd1340cp 4 0x7f400bd134fep 5 0x7f400bce238ep 6 0x4ea137p PyCFunction_Call + 119 7 0x5c20e7p PyObject_Call + 71 8 0x4fbfcep 9 0x5c20e7p PyObject_Call + 71 10 0x574c19p 11 0x57f58cp 12 0x5c20e7p PyObject_Call + 71 13 0x53b656p PyEval_EvalFrameEx + 20182 14 0x540b0bp PyEval_EvalCodeEx + 315 15 0x4ec3f7p 16 0x5c20e7p PyObject_Call + 71 17 0x4fbfcep 18 0x5c20e7p PyObject_Call + 71 19 0x574c19p 20 0x57f58cp 21 0x5c20e7p PyObject_Call + 71 22 0x53b656p PyEval_EvalFrameEx + 20182 23 0x5401efp 24 0x53bc93p PyEval_EvalFrameEx + 21779 25 0x53fc97p 26 0x5409bfp PyEval_EvalCode + 31 27 0x60cb42p 28 0x60efeap PyRun_FileExFlags + 154 29 0x60f7dcp PyRun_SimpleFileExFlags + 444 30 0x640256p Py_Main + 1110 31 0x4d0001p main + 225 32 0x7f407aa96830p __libc_start_main + 240 33 0x5d6999p _start + 41