Cannot find fetch variable in scope, fetch_var_name
Created by: xfcygaocan
问题：Cannot find fetch variable in scope, fetch_var_name，存在相似问题https://github.com/PaddlePaddle/Paddle/issues/19114 ，但是没有解决。
环境：paddle版本：fluid 1.5/1.5.1，单机单卡k40-12g/v100-16g，cuda 9.0，cudnn 7.0
补充说明：模型参考ernie结构写的
报错信息：
I0830 15:40:21.593181 3612 operator.cc:166] CUDAPlace(0) Op(create_py_reader), inputs:{blocking_queue[train_reader_queue:[-1]({{}})]}, outputs:{Out[train_reader_reader:[-1]({{}})]}.
I0830 15:40:21.593266 3612 operator.cc:187] CUDAPlace(0) Op(create_py_reader), inputs:{blocking_queue[train_reader_queue:[-1]({{}})]}, outputs:{Out[train_reader_reader:[-1]({{}})]}.
I0830 15:40:21.593389 3612 operator.cc:166] CUDAPlace(0) Op(create_py_reader), inputs:{blocking_queue[test_reader_queue:[-1]({{}})]}, outputs:{Out[test_reader_reader:[-1]({{}})]}.
I0830 15:40:21.593475 3612 operator.cc:187] CUDAPlace(0) Op(create_py_reader), inputs:{blocking_queue[test_reader_queue:[-1]({{}})]}, outputs:{Out[test_reader_reader:[-1]({{}})]}.
I0830 15:40:21.628964 3612 operator.cc:166] **CUDAPlace(0) Op(fetch), inputs:{X[mean_0.tmp_0[uninited]]}, outputs:{Out[fetch:[-1]({{}})]}.**
Traceback (most recent call last):
File "run_classifier.py", line 249, in <module>
main(args)
File "run_classifier.py", line 172, in main
graph_vars, "train")
File "/home/disk1/du/gaocan/6-query_att/src/nets.py", line 174, in evaluate
outputs = exe.run(fetch_list=train_fetch_list)
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/executor.py", line 650, in run
use_program_cache=use_program_cache)
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/executor.py", line 748, in _run
exe.run(program.desc, scope, 0, True, True, fetch_var_name)
**paddle.fluid.core_avx.EnforceNotMet: Invoke operator fetch error.**
Python Callstacks:
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/framework.py", line 1748, in append_op
attrs=kwargs.get("attrs", None))
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/executor.py", line 437, in _add_feed_fetch_ops
attrs={'col': i})
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/executor.py", line 744, in _run
fetch_var_name=fetch_var_name)
File "/home/disk1/du/gaocan/0-bin/8-paddle/1-fluid.1.5/paddle_release_home/python/lib/python2.7/site-packages/paddle/fluid/executor.py", line 650, in run
use_program_cache=use_program_cache)
File "/home/disk1/du/gaocan/6-query_att/src/nets.py", line 174, in evaluate
outputs = exe.run(fetch_list=train_fetch_list)
File "run_classifier.py", line 172, in main
graph_vars, "train")
File "run_classifier.py", line 249, in <module>
main(args)
C++ Callstacks:
**Cannot find fetch variable in scope, fetch_var_name is mean_0.tmp_0** at [/paddle/paddle/fluid/operators/controlflow/fetch_op.cc:37]模型代码：
pyreader = fluid.layers.py_reader(
capacity=50,
shapes=[[-1, 1], [-1, 1], [-1, 1], [-1, 1]],
dtypes=['int64', 'int64', 'int64', 'int64'],
lod_levels=[2, 1, 0, 0],
name=pyreader_name,
use_double_buffer=True)
(session_ids, att_ids, labels, qids) = fluid.layers.read_file(pyreader)
param_initializer = fluid.initializer.TruncatedNormal(scale=0.02)
emb_session = fluid.layers.embedding(
input=session_ids,
size=[args.dict_dim, args.emb_dim],
is_sparse=False,
param_attr=fluid.ParamAttr(
name="word_emb", initializer=param_initializer))
emb_att = fluid.layers.embedding(
input=att_ids,
size=[args.dict_dim, args.emb_dim],
is_sparse=False,
param_attr=fluid.ParamAttr(
name="word_emb", initializer=param_initializer))
drnn = fluid.layers.DynamicRNN()
with drnn.block():
emb_click = drnn.step_input(emb_session)
bow_click = fluid.layers.sequence_pool(
input=emb_click,
pool_type='sum')
bow_click_tanh = fluid.layers.softsign(bow_click)
drnn.output(bow_click_tanh)
bow_session = drnn()
bow_session1 = fluid.layers.sequence_pool(
input=bow_session,
pool_type='sum')
bow_session1 = fluid.layers.softsign(bow_session1)
bow_session1 = fluid.layers.dropout(
x=bow_session1,
dropout_prob=0.1,
dropout_implementation="upscale_in_train")
bow_att = fluid.layers.sequence_pool(input=emb_att,
pool_type='sum')
bow_att = fluid.layers.softsign(bow_att)
bow_feature = bow_att + bow_session1
logits = fluid.layers.fc(
input=bow_feature,
size=2,
param_attr=fluid.ParamAttr(
name="feature_out_w",
initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
bias_attr=fluid.ParamAttr(
name="feature_out_b", initializer=fluid.initializer.Constant(0.)))
if is_prediction:
probs = fluid.layers.softmax(logits)
feed_targets_name = [session_ids.name, att_ids.name]
return pyreader, probs, feed_targets_name
ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
logits=logits,
label=labels,
return_softmax=True)
loss = fluid.layers.mean(x=ce_loss)
num_seqs = fluid.layers.create_tensor(dtype='int64')
accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs)
graph_vars = {
"loss": loss,
"probs": probs,
"accuracy": accuracy,
"labels": labels,
"num_seqs": num_seqs,
"qids": qids,
}
for k, v in graph_vars.items():
v.persistable = True
return pyreader, graph_varsexe.run代码：
train_fetch_list = [
graph_vars["loss"].name,
graph_vars["accuracy"].name,
graph_vars["num_seqs"].name,
]
if eval_phase == "train":
if "learning_rate" in graph_vars:
train_fetch_list.append(graph_vars["learning_rate"].name)
outputs = exe.run(fetch_list=train_fetch_list)
print(outputs[0])
ret = {"loss": np.mean(outputs[0]), "accuracy": np.mean(outputs[1])}
#ret = {"loss": 0., "accuracy": 0.}
if "learning_rate" in graph_vars:
ret["learning_rate"] = float(outputs[3][0])
#ret["learning_rate"] = 0.
return ret