训练时出sigsegv错误
Created by: youan1
如题,训练时出了sigsegv错误:错误日志如下,但是感觉paddle这个时候还没有开始训练,请paddle同学看看是什么问题
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ PADDLE_NUM_GRADIENT_SERVERS=50
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ export PADDLE_TRAINER_ID=9
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ PADDLE_TRAINER_ID=9
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ export PADDLE_PSERVERS=10.86.104.19,10.86.103.15,10.86.103.16,10.86.103.17,10.86.103.18,10.86.103.19,10.86.103.21,10.86.103.23,10.86.103.24,10.86.103.25,10.86.103.27,10.86.103.30,10.86.103.31,10.86.103.32,10.86.103.34,10.86.103.35,10.86.103.36,10.86.103.37,10.86.103.38,10.86.103.39,10.86.103.40,10.86.103.41,10.86.103.42,10.86.103.43,10.86.104.11,10.86.104.12,10.86.104.13,10.86.104.14,10.86.104.15,10.86.104.18,10.86.103.14,10.86.104.20,10.86.104.21,10.86.104.22,10.86.104.23,10.86.104.25,10.86.104.26,10.86.104.28,10.86.104.30,10.86.104.31,10.86.104.32,10.86.104.33,10.86.104.35,10.86.104.36,10.86.104.37,10.86.104.39,10.86.104.40,10.86.104.42,10.86.104.43,10.86.105.11
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ PADDLE_PSERVERS=10.86.104.19,10.86.103.15,10.86.103.16,10.86.103.17,10.86.103.18,10.86.103.19,10.86.103.21,10.86.103.23,10.86.103.24,10.86.103.25,10.86.103.27,10.86.103.30,10.86.103.31,10.86.103.32,10.86.103.34,10.86.103.35,10.86.103.36,10.86.103.37,10.86.103.38,10.86.103.39,10.86.103.40,10.86.103.41,10.86.103.42,10.86.103.43,10.86.104.11,10.86.104.12,10.86.104.13,10.86.104.14,10.86.104.15,10.86.104.18,10.86.103.14,10.86.104.20,10.86.104.21,10.86.104.22,10.86.104.23,10.86.104.25,10.86.104.26,10.86.104.28,10.86.104.30,10.86.104.31,10.86.104.32,10.86.104.33,10.86.104.35,10.86.104.36,10.86.104.37,10.86.104.39,10.86.104.40,10.86.104.42,10.86.104.43,10.86.105.11
Wed Sep 20 19:33:32 2017[1,9]<stderr>:+ python27-gcc482/bin/python conf/trainer_config.conf
Wed Sep 20 19:37:56 2017[1,27]<stderr>:*** Aborted at 1505907476 (unix time) try "date -d @1505907476" if you are using GNU date ***
Wed Sep 20 19:37:56 2017[1,27]<stderr>:PC: @ 0x0 (unknown)
Wed Sep 20 19:37:56 2017[1,27]<stderr>:*** SIGSEGV (@0x8) received by PID 28197 (TID 0x7ff16b6c3700) from PID 8; stack trace: ***
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff16b29a160 (unknown)
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff167bd7012 paddle::ProtoClient::recv()
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff16886444e paddle::ParameterClient2::inStatus()
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff168864584 paddle::ParameterClient2::waitForStatus()
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff1683eb1a2 paddle::RemoteParameterUpdater::init()
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff167d5e1ea ParameterUpdater::init()
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff167a07f7b _wrap_ParameterUpdater_init
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b4cb9 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4b6c52 PyEval_EvalCode
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4e1c7d PyRun_FileExFlags
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4e3501 PyRun_SimpleFileExFlags
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x4159dd Py_Main
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x7ff16a7f4bd5 __libc_start_main
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x414b71 (unknown)
Wed Sep 20 19:37:56 2017[1,27]<stderr>: @ 0x0 (unknown)
Wed Sep 20 19:37:56 2017[1,3]<stderr>:*** Aborted at 1505907476 (unix time) try "date -d @1505907476" if you are using GNU date ***
Wed Sep 20 19:37:56 2017[1,3]<stderr>:PC: @ 0x0 (unknown)
Wed Sep 20 19:37:56 2017[1,3]<stderr>:*** SIGSEGV (@0x8) received by PID 14576 (TID 0x7f3913253700) from PID 8; stack trace: ***
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f3912e2a160 (unknown)
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390c925012 paddle::ProtoClient::recv()
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390d5b244e paddle::ParameterClient2::inStatus()
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390d5b2584 paddle::ParameterClient2::waitForStatus()
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390d1391a2 paddle::RemoteParameterUpdater::init()
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390caac1ea ParameterUpdater::init()
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x7f390c755f7b _wrap_ParameterUpdater_init
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b4cb9 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b5d10 PyEval_EvalFrameEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b6b28 PyEval_EvalCodeEx
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4b6c52 PyEval_EvalCode
Wed Sep 20 19:37:56 2017[1,3]<stderr>: @ 0x4e1c7d PyRun_FileExFlags