diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc
index 84b765680fdbd96f2cb78d9502dd762fac960e43..abeadb5aa14915f51b35eb71ca1a52e7b59cceac 100644
--- a/paddle/fluid/framework/new_executor/interpretercore.cc
+++ b/paddle/fluid/framework/new_executor/interpretercore.cc
@@ -98,6 +98,9 @@ void InterpreterCore::Convert() {
 
     for (auto& item : op_func_node.input_index) {
       for (auto id : item.second) {
+        if (id == kEmptyVarIndex) {
+          continue;
+        }
         input_var2op_info_.at(id).push_back(op_idx);
         // var can be gc-ed
         if (!info.IsBuilt()) {
diff --git a/paddle/fluid/framework/new_executor/interpretercore_garbage_collector.cc b/paddle/fluid/framework/new_executor/interpretercore_garbage_collector.cc
index 59dd44ab9ada6b3bc9c23c7c550e5fc24b69c983..f17f64dbcae45433592b183d36769cc991611df8 100644
--- a/paddle/fluid/framework/new_executor/interpretercore_garbage_collector.cc
+++ b/paddle/fluid/framework/new_executor/interpretercore_garbage_collector.cc
@@ -60,6 +60,10 @@ void InterpreterCoreGarbageCollector::Add(
 void InterpreterCoreGarbageCollector::Add(paddle::framework::Variable* var,
                                           paddle::platform::DeviceEvent& event,
                                           const platform::DeviceContext* ctx) {
+  if (!var) {
+    return;
+  }
+
   if (var->IsType<LoDTensor>()) {
     Add(var->GetMutable<LoDTensor>()->MoveMemoryHolder(), event, ctx);
   } else if (var->IsType<
diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.cc b/paddle/fluid/framework/new_executor/interpretercore_util.cc
index 972c8c19dd40507e76c65d525bb106910eaa7b4f..068b554a57f1976391e194f829f3eaa615906ddd 100644
--- a/paddle/fluid/framework/new_executor/interpretercore_util.cc
+++ b/paddle/fluid/framework/new_executor/interpretercore_util.cc
@@ -446,7 +446,13 @@ void build_op_func_list(const platform::Place& place,
     VariableValueMap ins_map;
     VariableIdMap ins_name2id;
     bool enforce_exist = true;
-    if (op->Type() == "recurrent_grad") enforce_exist = false;
+    if (op->Type() == "recurrent_grad" || op->Type() == "rnn_memory_helper" ||
+        op->Type() == "rnn_memory_helper_grad" ||
+        op->Type() == "conditional_block" ||
+        op->Type() == "conditional_block_grad" || op->Type() == "while" ||
+        op->Type() == "while_grad") {
+      enforce_exist = false;
+    }
     std::tie(ins_map, ins_name2id) =
         build_variable_map(inputs_names, var_scope, enforce_exist);
 
diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.cc b/paddle/fluid/framework/new_executor/new_executor_defs.cc
index 221ad2dd628b6fbaff707aa5be677541171f1db0..bd8072885e2386d54c6a05b520cfd22031ac07af 100644
--- a/paddle/fluid/framework/new_executor/new_executor_defs.cc
+++ b/paddle/fluid/framework/new_executor/new_executor_defs.cc
@@ -480,7 +480,7 @@ const std::vector<Variable*>& InterpretercoreInferShapeContext::OutputVars(
 VariableScope::VariableScope(Scope* scope) {
   // for @EMPTY@ variable
   var_list_.push_back(nullptr);
-  name2id_[kEmptyVarName] = 0;
+  name2id_[kEmptyVarName] = kEmptyVarIndex;
   vec_meta_info_.emplace_back(0, nullptr);
   scope_ = scope;
   PADDLE_ENFORCE_NE(
diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.h b/paddle/fluid/framework/new_executor/new_executor_defs.h
index 4206f2733a06b3bc76dabdddded67bdd86ee2c99..2479abb8926abe2becf0b0c529fb16effe242bb2 100644
--- a/paddle/fluid/framework/new_executor/new_executor_defs.h
+++ b/paddle/fluid/framework/new_executor/new_executor_defs.h
@@ -43,6 +43,8 @@ using OpKernelComputeFunc = std::function<void(const ExecutionContext&)>;
 using OpKernelMap =
     std::unordered_map<OpKernelType, OpKernelComputeFunc, OpKernelType::Hash>;
 
+constexpr int kEmptyVarIndex = 0;
+
 class InterpretercoreInferShapeContext : public InferShapeContext {
  public:
   InterpretercoreInferShapeContext(const OperatorBase& op,
diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py
index 527840f50718ae1fa167ae5e799ef5ded07f8ae9..393232a2061050c2e71da409c680d7f83a887b1b 100644
--- a/python/paddle/fluid/executor.py
+++ b/python/paddle/fluid/executor.py
@@ -598,13 +598,13 @@ class _ExecutorCache(object):
         assert isinstance(
             program, Program), "Required type(Program), but received {}".format(
                 type(program).__name__)
-        if program not in self._cached_executors:
+        if str(program) not in self._cached_executors:
             new_program = program.clone()
             _prune_feed_ops(new_program)
             new_exe = _StandaloneExecutor(self._place, new_program, scope)
-            self._cached_executors[program] = new_exe
+            self._cached_executors[str(program)] = new_exe
 
-        return self._cached_executors[program]
+        return self._cached_executors[str(program)]
 
 
 class Executor(object):