Created by: guozhiyao

原始代码运行正常，结构如下：

main_program = fluid.default_main_program()
start_program = fluid.default_startup_program()
model,loss = get_model()
val_program = main_program.clone(for_test=True)
optimizer = optimizer_program(args)
optimizer.minimize(avg_cost)

但是我希望加入梯度裁剪，修改为

main_program = fluid.default_main_program()
start_program = fluid.default_startup_program()
model,loss = get_model()
fluid.clip.set_gradient_clip(
        clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0))
val_program = main_program.clone(for_test=True)
optimizer = optimizer_program(args)
optimizer.minimize(avg_cost)

此时就报错了：

Traceback (most recent call last):
  File "trainer.py", line 418, in <module>
    main(use_cuda)
  File "trainer.py", line 407, in main
    train(use_cuda=use_cuda, params_dirname=save_path)
  File "trainer.py", line 311, in train
    train_loop()
  File "trainer.py", line 270, in train_loop
    fetch_list=[avg_cost.name, mean.name,mean_predict.name])
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/executor.py", line 666, in run
    return_numpy=return_numpy)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/executor.py", line 528, in _run_parallel
    exe.run(fetch_var_names, fetch_var_name)
paddle.fluid.core_avx.EnforceNotMet: Invoke operator elementwise_mul error.
Python Callstacks:
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/framework.py", line 1771, in append_op
    attrs=kwargs.get("attrs", None))
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
    return self.main_program.current_block().append_op(*args, **kwargs)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 9904, in _elementwise_op
    'use_mkldnn': use_mkldnn})
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/layers/nn.py", line 9965, in elementwise_mul
    return _elementwise_op(LayerHelper('elementwise_mul', **locals()))
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/clip.py", line 334, in _create_operators
    x=grad, y=self.context[group_scale_name])
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/clip.py", line 395, in append_gradient_clip_ops
    res.append(clip_attr._create_operators(param=p, grad=g))
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/optimizer.py", line 526, in apply_gradients
    params_grads = append_gradient_clip_ops(params_grads)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/optimizer.py", line 562, in apply_optimize
    optimize_ops = self.apply_gradients(params_grads)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/optimizer.py", line 601, in minimize
    loss, startup_program=startup_program, params_grads=params_grads)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/dygraph/base.py", line 87, in __impl__
    return func(*args, **kwargs)
  File "/home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/paddle/fluid/wrapped_decorator.py", line 25, in __impl__
    return wrapped_func(*args, **kwargs)
  File "</home/vis/yourenchun/anaconda3/envs/padpy36/lib/python3.6/site-packages/decorator.py:decorator-gen-20>", line 2, in minimize
  File "trainer.py", line 181, in train
    optimizer.minimize(avg_cost)
  File "trainer.py", line 407, in main
    train(use_cuda=use_cuda, params_dirname=save_path)
  File "trainer.py", line 418, in <module>
    main(use_cuda)
C++ Callstacks:
holder_ should not be null
Tensor holds no memory. Call Tensor::mutable_data first. at [/paddle/paddle/fluid/framework/tensor.cc:23]
PaddlePaddle Call Stacks:
0       0x7f1bc3069c88p void paddle::platform::EnforceNotMet::Init<std::string>(std::string, char const*, int) + 360
1       0x7f1bc3069fd7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87
2       0x7f1bc5025cd9p paddle::framework::Tensor::check_memory_size() const + 185
3       0x7f1bc3070a89p float const* paddle::framework::Tensor::data<float>() const + 25
4       0x7f1bc4823b5cp void paddle::operators::ElementwiseComputeEx<paddle::operators::MulFunctor<float>, paddle::platform::CUDADeviceContext, float, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, int, paddle::operators::MulFunctor<float>, paddle::framework::Tensor*) + 76
5       0x7f1bc48244a3p void paddle::operators::default_elementwise_mul<paddle::platform::CUDADeviceContext, float>(paddle::framework::ExecutionContext const&, paddle::framework::Tensor const*, paddle::framework::Tensor const*, paddle::framework::Tensor*) + 115
python -u trainer.py --lr 0.01 --num_epochs 100 --mode resnet101 --multiscale False --pretrained_model ./pretrain_backbone/ResNet101_pretrained/
6       0x7f1bc482480bp paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const + 811
7       0x7f1bc4824c93p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, int>, paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ElementwiseMulKernel<paddle::platform::CUDADeviceContext, paddle::platform::float16> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
8       0x7f1bc4fcfe07p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375
9       0x7f1bc4fd01e1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
10      0x7f1bc4fcd7dcp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332
11      0x7f1bc4dc8fc9p
12      0x7f1bc4dba3adp
13      0x7f1bc4dbb0e4p paddle::framework::details::OpHandleBase::RunAndRecordEvent(std::function<void ()> const&) + 116
14      0x7f1bc4dc8c5cp paddle::framework::details::ComputationOpHandle::RunImpl() + 124
15      0x7f1bc4dbb680p paddle::framework::details::OpHandleBase::Run(bool) + 160
16      0x7f1bc4d9c9f6p paddle::framework::details::FastThreadedSSAGraphExecutor::RunOpSync(paddle::framework::details::OpHandleBase*) + 310
17      0x7f1bc4d9b65fp paddle::framework::details::FastThreadedSSAGraphExecutor::RunOp(paddle::framework::details::OpHandleBase*, std::shared_ptr<paddle::framework::BlockingQueue<unsigned long> > const&, unsigned long*) + 47
18      0x7f1bc4d9ba1fp
19      0x7f1bc329c6c3p std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result<void>, std::__future_base::_Result_base::_Deleter>, void> >::_M_invoke(std::_Any_data const&) + 35
20      0x7f1bc3133477p std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&) + 39
21      0x7f1c6d6d5b23p pthread_once + 83
22      0x7f1bc4d970a2p
23      0x7f1bc31349f4p ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const + 404
24      0x7f1be7a2e421p
25      0x7f1c6d6d0851p
26      0x7f1c6d41e67dp clone + 109