Add friendly dygraph trace API (#21091)

* friendly trace interface, test=develop

* refine TracedLayer, test=develop

* add some docs, test=develop

paddle/fluid/pybind/pybind.cc

@@ -528,6 +528,7 @@ PYBIND11_MODULE(core_noavx, m) {
       .def("_get_double_element", TensorGetElement<double>)
       .def("_place", [](Tensor &self) { return self.place(); })
       .def("_dtype", [](Tensor &self) { return self.type(); })
+      .def("_share_data_with", &Tensor::ShareDataWith)
       .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
       .def("__str__", [](const Tensor &self) {
         std::stringstream ostr;

python/paddle/fluid/dygraph/base.py

@@ -27,6 +27,17 @@ __all__ = [
 ]
 
 
+def _switch_to_static_graph_(func):
+    def __impl__(*args, **kwargs):
+        with framework._dygraph_guard(None):
+            return func(*args, **kwargs)
+
+    return __impl__
+
+
+switch_to_static_graph = wrap_decorator(_switch_to_static_graph_)
+
+
 @signature_safe_contextmanager
 def program_desc_tracing_guard(enable):
     tracer = framework._dygraph_tracer()

python/paddle/fluid/dygraph/jit.py

@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__all__ = ['trace']
+__all__ = ['TracedLayer']
 
-from .base import program_desc_tracing_guard
+from .base import program_desc_tracing_guard, switch_to_static_graph
 from .layers import Layer
-from paddle.fluid.framework import Program, Block, Variable, _dygraph_tracer, dygraph_only, _dygraph_guard
+from paddle.fluid import core
+from paddle.fluid.framework import Program, Block, Variable, _dygraph_tracer, dygraph_only, _dygraph_guard, _current_expected_place, in_dygraph_mode
+from paddle.fluid.executor import Executor, scope_guard
+from paddle.fluid.compiler import CompiledProgram
+import paddle.fluid.io as fluid_io
 
 
 def create_program_from_desc(program_desc):
@@ -43,7 +47,7 @@ def extract_vars(inputs):
 
 
 @dygraph_only
-def trace(layer, inputs, feed_names=None, fetch_names=None):
+def _trace(layer, inputs, feed_names=None, fetch_names=None):
     """
     Trace dygraph network into a :code:`Program`. The returned :code:`Program`
     can be run in static graph mode. This method would simply record all
@@ -52,7 +56,7 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
     and would not be changed between different batches. Otherwise, the traced
     result may be different.
 
-    Parameters:
+    Args:
         layer(Layer): the layer to be traced.
         inputs(list): the input arguments of :code:`layer.forward()` method.
         feed_names(list(str), optional): the input variable names in the 
@@ -70,13 +74,14 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
                 different between different batches. Default None.
                 
     Returns:
-        A tuple of 2 items, whose first item is the outputs of 
+        A tuple of 4 items, whose first item is the outputs of 
         :code:`layer.forward()` method, and second item is the traced 
-        :code:`Program` .
+        :code:`Program`, and the third item is names of feed variables,
+        and the fourth item is names of fetch variables.
 
     Examples:
 
-        .. code-blocks: python:
+        .. code-block:: python:
 
             import paddle.fluid as fluid
             from paddle.fluid.dygraph import FC, to_variable
@@ -95,7 +100,7 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
                 layer = ExampleLayer("example_layer")
                 in_np = np.random.random([2, 3]).astype('float32')
                 in_var = to_variable(in_np)
-                out, program = jit.trace(layer, inputs=[in_var],
+                out, program, _, _ = jit._trace(layer, inputs=[in_var],
                                          feed_names=['input'],
                                          fetch_names=['fc_out'])
 
@@ -114,6 +119,9 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
     tracer = _dygraph_tracer()._get_program_desc_tracer()
 
     var_list = extract_vars(inputs)
+    if callable(feed_names):
+        feed_names = feed_names(len(var_list))
+
     tracer.set_feed_vars(var_list, feed_names)
 
     with program_desc_tracing_guard(True):
@@ -124,6 +132,9 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
             outputs = original_outputs
         out_vars = [var._ivar for var in outputs]
 
+        if callable(fetch_names):
+            fetch_names = fetch_names(len(out_vars))
+
         tracer.set_fetch_vars(out_vars, fetch_names)
         tracer.set_name_prefix('t_')
 
@@ -133,4 +144,252 @@ def trace(layer, inputs, feed_names=None, fetch_names=None):
     with _dygraph_guard(None):
         program = create_program_from_desc(program_desc)
 
-    return original_outputs, program
+    return original_outputs, program, feed_names, fetch_names
+
+
+class TracedLayer(object):
+    """
+    TracedLayer is a callable object which is converted from dygraph model. 
+    Inside TracedLayer, the dygraph model is converted into a static graph
+    model, and it would run the static graph model using 
+    :code:`Executor` and :code:`CompiledProgram` . The static graph model 
+    would share parameters with the dygraph model. 
+    
+    All TracedLayer objects should not be created by constructor and should 
+    be created by static method :code:`TracedLayer.trace(layer, inputs)` .
+
+    The TracedLayer can only be used to convert the data-independent dygraph
+    model into the static graph model, which means the dygraph model should
+    be independent with the tensor data and shape.
+    """
+
+    def __init__(self, program, parameters, feed_names, fetch_names):
+        self._program = program
+        self._feed_names = feed_names
+        self._fetch_names = fetch_names
+
+        self._place = _current_expected_place()
+
+        self._scope = core.Scope()
+        for p in parameters:
+            src_tensor = p._ivar.value().get_tensor()
+            dst_tensor = self._scope.var(p.name).get_tensor()
+            dst_tensor._share_data_with(src_tensor)
+
+        self._exe = Executor(self._place)
+        self._compiled_program = None
+        self._build_strategy = None
+        self._exec_strategy = None
+
+    @property
+    def program(self):
+        return self._program
+
+    def _switch(self, is_test=True):
+        for block_id in range(self._program.num_blocks):
+            block = self._program.block(block_id)
+            for op in block.ops:
+                if op.has_attr("is_test"):
+                    op._set_attr("is_test", is_test)
+
+    @staticmethod
+    @dygraph_only
+    def trace(layer, inputs):
+        """
+        This method is the only allowed method to create TracedLayer object. 
+        It would call the :code:`layer(*inputs)` method to run the dygraph
+        model and convert it into a static graph model.
+
+        Args:
+            layer (paddle.fluid.dygraph.Layer): the layer object to be traced.
+            inputs (list(Variable)): the input variables of the layer object. 
+
+        Returns:
+            A tuple of 2 items, whose the first item is the output of 
+            :code:`layer(*inputs)` , and the second item is the created
+            TracedLayer object. 
+            
+        Examples:
+
+            .. code-block:: python:
+
+                import paddle.fluid as fluid
+                from paddle.fluid.dygraph import FC, to_variable, TracedLayer
+                import paddle.fluid.dygraph.jit as jit
+                import numpy as np
+
+                class ExampleLayer(fluid.dygraph.Layer):
+                    def __init__(self, name_scope):
+                        super(ExampleLayer, self).__init__(name_scope)
+                        self._fc = FC(self.full_name(), 10)
+
+                    def forward(self, input):
+                        return self._fc(input)
+
+                with fluid.dygraph.guard():
+                    layer = ExampleLayer("example_layer")
+                    in_np = np.random.random([2, 3]).astype('float32')
+                    in_var = to_variable(in_np)
+                    out_dygraph, static_layer = TracedLayer.trace(layer, inputs=[in_var])
+                    out_static_graph = static_layer([in_var]) 
+        """
+        feed_func = lambda n: ['feed_{}'.format(i) for i in range(n)]
+        fetch_func = lambda n: ['fetch_{}'.format(i) for i in range(n)]
+        outs, prog, feed, fetch = _trace(layer, inputs, feed_func, fetch_func)
+        traced = TracedLayer(prog, layer.parameters(), feed, fetch)
+        return outs, traced
+
+    def set_strategy(self, build_strategy=None, exec_strategy=None):
+        """
+        Set the strategies when running static graph model.
+
+        Args:
+            build_strategy (BuildStrategy, optional): build strategy of 
+                :code:`CompiledProgram` inside TracedLayer. Default None.
+            exec_strategy (ExecutionStrategy, optional): execution strategy of
+                :code:`CompiledProgram` inside TracedLayer. Default None.
+
+        Returns:
+            None
+
+        Examples:
+
+            .. code-block:: python:
+
+                import paddle.fluid as fluid
+                from paddle.fluid.dygraph import FC, to_variable, TracedLayer
+                import paddle.fluid.dygraph.jit as jit
+                import numpy as np
+
+                class ExampleLayer(fluid.dygraph.Layer):
+                    def __init__(self, name_scope):
+                        super(ExampleLayer, self).__init__(name_scope)
+                        self._fc = FC(self.full_name(), 10) 
+
+                    def forward(self, input):
+                        return self._fc(input)
+
+                with fluid.dygraph.guard():
+                    layer = ExampleLayer("example_layer")
+                    in_np = np.random.random([2, 3]).astype('float32')
+                    in_var = to_variable(in_np)
+
+                    out_dygraph, static_layer = TracedLayer.trace(layer, inputs=[in_var])
+
+                    build_strategy = fluid.BuildStrategy()
+                    build_strategy.enable_inplace = True
+
+                    exec_strategy = fluid.ExecutionStrategy()
+                    exec_strategy.num_threads = 2
+
+                    static_layer.set_strategy(build_strategy=build_strategy, exec_strategy=exec_strategy)
+                    out_static_graph = static_layer([in_var])
+        """
+        assert self._compiled_program is None, "Cannot set strategy after run"
+        self._build_strategy = build_strategy
+        self._exec_strategy = exec_strategy
+
+    @switch_to_static_graph
+    def _compile(self):
+        self._compiled_program = CompiledProgram(
+            self._program).with_data_parallel(
+                build_strategy=self._build_strategy,
+                exec_strategy=self._exec_strategy,
+                places=self._place)
+
+    def _build_feed(self, inputs):
+        assert isinstance(inputs, (list, tuple)), \
+            "Inputs should be a list or tuple of variables"
+        assert len(inputs) == len(self._feed_names)
+        feed_dict = {}
+        if in_dygraph_mode():
+            for x, name in zip(inputs, self._feed_names):
+                feed_dict[name] = x._ivar.value().get_tensor()
+        else:
+            for x, name in zip(inputs, self._feed_names):
+                feed_dict[name] = x
+
+        return feed_dict
+
+    @switch_to_static_graph
+    def _run(self, feed):
+        return self._exe.run(self._compiled_program,
+                             feed=feed,
+                             fetch_list=self._fetch_names)
+
+    def __call__(self, inputs):
+        with scope_guard(self._scope):
+            if self._compiled_program is None:
+                self._compile()
+
+            return self._run(self._build_feed(inputs))
+
+    @switch_to_static_graph
+    def save_inference_model(self, dirname, feed=None, fetch=None):
+        """
+        Save the TracedLayer to an model for inference. The saved
+        inference model can be loaded by C++ inference APIs. 
+
+        Args:
+            dirname (str): the directory to save the inference model.  
+            feed (list[int], optional): the input variable indices of the saved
+                inference model. If None, all input variables of the 
+                TracedLayer object would be the inputs of the saved inference
+                model. Default None.
+            fetch (list[int], optional): the output variable indices of the
+                saved inference model. If None, all output variables of the
+                TracedLayer object would be the outputs of the saved inference
+                model. Default None.
+
+        Returns:
+            The fetch variables' name list
+        
+        Return Type: 
+            list(str)
+
+        Examples:
+
+            .. code-block:: python:
+
+                import paddle.fluid as fluid
+                from paddle.fluid.dygraph import FC, to_variable, TracedLayer
+                import paddle.fluid.dygraph.jit as jit
+                import numpy as np
+
+                class ExampleLayer(fluid.dygraph.Layer):
+                    def __init__(self, name_scope):
+                        super(ExampleLayer, self).__init__(name_scope)
+                        self._fc = FC(self.full_name(), 10) 
+
+                    def forward(self, input):
+                        return self._fc(input)
+
+                with fluid.dygraph.guard():
+                    layer = ExampleLayer("example_layer")
+                    in_np = np.random.random([2, 3]).astype('float32')
+                    in_var = to_variable(in_np)
+                    out_dygraph, static_layer = TracedLayer.trace(layer, inputs=[in_var])
+                    static_layer.save_inference_model('./saved_infer_model')
+        """
+
+        def get_feed_fetch(all_vars, partial_vars):
+            if partial_vars is None:
+                return all_vars
+
+            return [all_vars[idx] for idx in feed]
+
+        with scope_guard(self._scope):
+            feeded_var_names = get_feed_fetch(self._feed_names, feed)
+            target_var_names = get_feed_fetch(self._fetch_names, fetch)
+            target_vars = []
+            for name in target_var_names:
+                target_var = self._program.global_block().vars.get(name, None)
+                assert target_var is not None, "{} cannot be found".format(name)
+                target_vars.append(target_var)
+
+            return fluid_io.save_inference_model(
+                dirname=dirname,
+                feeded_var_names=feeded_var_names,
+                target_vars=target_vars,
+                executor=self._exe,
+                main_program=self._program.clone())

python/paddle/fluid/tests/unittests/test_imperative_mnist.py

@@ -26,7 +26,8 @@ from paddle.fluid.optimizer import SGDOptimizer
 from paddle.fluid.dygraph.nn import Conv2D, Pool2D, FC
 from paddle.fluid.dygraph.base import to_variable
 from test_imperative_base import new_program_scope
-from utils import DyGraphProgramDescTracerTestHelper
+from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
+from paddle.fluid.dygraph.jit import TracedLayer
 
 
 class SimpleImgConvPool(fluid.dygraph.Layer):
@@ -119,6 +120,8 @@ class TestImperativeMnist(unittest.TestCase):
         batch_size = 128
         batch_num = 50
 
+        traced_layer = None
+
         with fluid.dygraph.guard():
             fluid.default_startup_program().random_seed = seed
             fluid.default_main_program().random_seed = seed
@@ -137,8 +140,8 @@ class TestImperativeMnist(unittest.TestCase):
             mnist.train()
             dy_param_init_value = {}
 
-            helper = DyGraphProgramDescTracerTestHelper(mnist, self)
-
+            helper = DyGraphProgramDescTracerTestHelper(self)
+            program = None
             for epoch in range(epoch_num):
                 for batch_id, data in enumerate(batch_py_reader()):
                     if batch_id >= batch_num:
@@ -149,13 +152,20 @@ class TestImperativeMnist(unittest.TestCase):
                     label.stop_gradient = True
 
                     if batch_id % 10 == 0:
-                        cost, cost_static = helper.run(inputs=img,
-                                                       feed_names=['image'],
-                                                       fetch_names=['cost'])
-                        helper.assertEachVar(cost, cost_static)
+                        cost, traced_layer = TracedLayer.trace(
+                            mnist, inputs=img)
+                        if program is not None:
+                            self.assertTrue(program, traced_layer.program)
+                        program = traced_layer.program
+                        traced_layer.save_inference_model(
+                            './infer_imperative_mnist')
                     else:
                         cost = mnist(img)
 
+                    if traced_layer is not None:
+                        cost_static = traced_layer([img])
+                        helper.assertEachVar(cost, cost_static)
+
                     loss = fluid.layers.cross_entropy(cost, label)
                     avg_loss = fluid.layers.mean(loss)
 
@@ -220,6 +230,10 @@ class TestImperativeMnist(unittest.TestCase):
 
                     fetch_list = [avg_loss.name]
                     fetch_list.extend(static_param_name_list)
+
+                    if traced_layer is not None:
+                        traced_layer([static_x_data])
+
                     out = exe.run(
                         fluid.default_main_program(),
                         feed={"pixel": static_x_data,

python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py

@@ -21,10 +21,11 @@ from paddle.fluid.dygraph.nn import Embedding
 import paddle.fluid.framework as framework
 from paddle.fluid.optimizer import SGDOptimizer
 from paddle.fluid.dygraph.base import to_variable
+from paddle.fluid.dygraph.jit import TracedLayer
 from test_imperative_base import new_program_scope
 import numpy as np
 import six
-from utils import DyGraphProgramDescTracerTestHelper
+from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
 
 
 class SimpleLSTMRNN(fluid.Layer):
@@ -221,6 +222,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
         batch_size = 4
         batch_num = 200
 
+        traced_layer = None
+
         with fluid.dygraph.guard():
             fluid.default_startup_program().random_seed = seed
             fluid.default_main_program().random_seed = seed
@@ -240,7 +243,8 @@ class TestDygraphPtbRnn(unittest.TestCase):
             last_hidden = None
             last_cell = None
 
-            helper = DyGraphProgramDescTracerTestHelper(ptb_model, self)
+            helper = DyGraphProgramDescTracerTestHelper(self)
+            program = None
 
             for i in range(batch_num):
                 x_data = np.arange(12).reshape(4, 3).astype('int64')
@@ -256,11 +260,19 @@ class TestDygraphPtbRnn(unittest.TestCase):
                 init_hidden = to_variable(init_hidden_data)
                 init_cell = to_variable(init_cell_data)
                 if i % 5 == 0:
-                    outs, outs_static = helper.run(
-                        [x, y, init_hidden, init_cell],
-                        feed_names=['x', 'y', 'init_hidden', 'init_cell'],
-                        fetch_names=['dy_loss', 'last_hidden', 'last_cell'])
+                    outs, traced_layer = TracedLayer.trace(
+                        ptb_model, [x, y, init_hidden, init_cell])
+                    outs_static = traced_layer([x, y, init_hidden, init_cell])
                     helper.assertEachVar(outs, outs_static)
+
+                    if program is not None:
+                        self.assertTrue(
+                            is_equal_program(traced_layer.program, program))
+
+                    program = traced_layer.program
+
+                    traced_layer.save_inference_model(
+                        './infe_imperative_ptb_rnn', feed=range(4))
                 else:
                     outs = ptb_model(x, y, init_hidden, init_cell)
 

python/paddle/fluid/tests/unittests/test_imperative_resnet.py

@@ -24,7 +24,8 @@ from paddle.fluid.layer_helper import LayerHelper
 from paddle.fluid import Conv2D, Pool2D, BatchNorm, FC
 from paddle.fluid.dygraph.base import to_variable
 from test_imperative_base import new_program_scope
-from utils import DyGraphProgramDescTracerTestHelper
+from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
+from paddle.fluid.dygraph.jit import TracedLayer
 
 batch_size = 8
 train_parameters = {
@@ -227,6 +228,8 @@ class TestDygraphResnet(unittest.TestCase):
         batch_size = train_parameters["batch_size"]
         batch_num = 10
 
+        traced_layer = None
+
         with fluid.dygraph.guard():
             fluid.default_startup_program().random_seed = seed
             fluid.default_main_program().random_seed = seed
@@ -250,7 +253,8 @@ class TestDygraphResnet(unittest.TestCase):
             for param in resnet.parameters():
                 dy_param_init_value[param.name] = param.numpy()
 
-            helper = DyGraphProgramDescTracerTestHelper(resnet, self)
+            helper = DyGraphProgramDescTracerTestHelper(self)
+            program = None
 
             for batch_id, data in enumerate(batch_py_reader()):
                 if batch_id >= batch_num:
@@ -260,14 +264,29 @@ class TestDygraphResnet(unittest.TestCase):
                 label = data[1]
                 label.stop_gradient = True
 
+                out = None
                 if batch_id % 5 == 0:
-                    out, out_static = helper.run(img,
-                                                 feed_names=['image'],
-                                                 fetch_names=['logits'])
-                    helper.assertEachVar(out, out_static)
+                    out, traced_layer = TracedLayer.trace(resnet, img)
+                    if program is not None:
+                        self.assertTrue(
+                            is_equal_program(program, traced_layer.program))
+
+                    traced_layer.save_inference_model(
+                        './infer_imperative_resnet')
+
+                    program = traced_layer.program
                 else:
                     out = resnet(img)
 
+                if traced_layer is not None:
+                    resnet.eval()
+                    traced_layer._switch(is_test=True)
+                    out_dygraph = resnet([img])
+                    out_static = traced_layer([img])
+                    traced_layer._switch(is_test=False)
+                    helper.assertEachVar(out_dygraph, out_static)
+                    resnet.train()
+
                 loss = fluid.layers.cross_entropy(input=out, label=label)
                 avg_loss = fluid.layers.mean(x=loss)
 
@@ -346,6 +365,9 @@ class TestDygraphResnet(unittest.TestCase):
                 y_data = np.array([x[1] for x in data]).astype('int64').reshape(
                     [batch_size, 1])
 
+                if traced_layer is not None:
+                    traced_layer([static_x_data])
+
                 fetch_list = [avg_loss.name]
                 fetch_list.extend(static_param_name_list)
                 fetch_list.extend(static_grad_name_list)

python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py

@@ -18,13 +18,14 @@ import unittest
 import paddle.fluid as fluid
 from paddle.fluid import Embedding, LayerNorm, FC, Layer
 from paddle.fluid.dygraph import to_variable, guard
+from paddle.fluid.dygraph.jit import TracedLayer
 from test_imperative_base import new_program_scope
 from paddle.fluid import core
 import numpy as np
 import six
 np.set_printoptions(suppress=True)
 
-from utils import DyGraphProgramDescTracerTestHelper
+from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
 
 
 # Copy from models
@@ -979,23 +980,24 @@ class TestDygraphTransformerSortGradient(unittest.TestCase):
             dy_param_init = dict()
             dy_param_updated = dict()
 
-            helper = DyGraphProgramDescTracerTestHelper(transformer, self)
+            helper = DyGraphProgramDescTracerTestHelper(self)
+            program = None
 
             for i in range(batch_num):
                 enc_inputs, dec_inputs, label, weights = create_data()
-                if i % 5 == 0:
-                    outs, outs_static = helper.run(
-                        inputs=[enc_inputs, dec_inputs, label, weights],
-                        feed_names=[
-                            'enc_input_0', 'enc_input_1', 'enc_input_2',
-                            'dec_input_0', 'dec_input_1', 'dec_input_2',
-                            'dec_input_3', 'label', 'weights'
-                        ],
-                        fetch_names=[
-                            'dy_sum_cost', 'dy_avg_cost', 'dy_predict',
-                            'dy_token_num'
-                        ])
+                if i % 2 == 0:
+                    outs, traced_layer = TracedLayer.trace(
+                        transformer, [enc_inputs, dec_inputs, label, weights])
+                    outs_static = traced_layer(enc_inputs + dec_inputs +
+                                               [label, weights])
                     helper.assertEachVar(outs, outs_static)
+                    if program is not None:
+                        self.assertTrue(
+                            is_equal_program(program, traced_layer.program))
+
+                    program = traced_layer.program
+                    traced_layer.save_inference_model(
+                        './infer_imperative_transformer')
                 else:
                     outs = transformer(enc_inputs, dec_inputs, label, weights)
 

python/paddle/fluid/tests/unittests/utils.py

@@ -21,7 +21,7 @@ import numpy as np
 import os
 import time
 
-__all__ = ['DyGraphProgramDescTracerTestHelper', ]
+__all__ = ['DyGraphProgramDescTracerTestHelper', 'is_equal_program']
 
 
 def is_equal_program(prog1, prog2):
@@ -107,74 +107,8 @@ def load_dygraph_vars_to_scope(model_path, scope, place):
 
 
 class DyGraphProgramDescTracerTestHelper(object):
-    def __init__(self,
-                 module,
-                 unittest_obj,
-                 model_path=None,
-                 scope=None,
-                 place=None):
-        self.module = module
+    def __init__(self, unittest_obj):
         self.unittest_obj = unittest_obj
-        self.scope = fluid.Scope() if scope is None else scope
-
-        self.model_path = model_path
-        if model_path is None:
-            millis = int(round(time.time() * 1000))
-            self.model_path = "id_{}_{}".format(id(module), millis)
-
-        self.place = place
-        if place is None:
-            self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda(
-            ) else fluid.CPUPlace()
-
-        self.program = None
-
-        self.executor = fluid.Executor(self.place)
-
-    def _remove_model_path(self):
-        if os.path.exists(self.model_path + ".pdparams"):
-            os.remove(self.model_path + ".pdparams")
-
-        if os.path.exists(self.model_path + ".pdopt"):
-            os.remove(self.model_path + ".pdopt")
-
-    def _run_static_graph(self, inputs, feed_names, fetch_names):
-        var_list = extract_vars(inputs)
-        assert len(var_list) == len(feed_names)
-
-        feed_dict = {}
-        for name, var in zip(feed_names, var_list):
-            feed_dict[name] = np.array(var.value().get_tensor())
-
-        with fluid.scope_guard(self.scope):
-            with _dygraph_guard(None):
-                return self.executor.run(self.program,
-                                         feed=feed_dict,
-                                         fetch_list=fetch_names)
-
-    def run(self, inputs, feed_names, fetch_names):
-        out_dygraph, program = jit.trace(
-            self.module, inputs, feed_names=feed_names, fetch_names=fetch_names)
-
-        if self.program is not None:
-            self.unittest_obj.assertTrue(
-                is_equal_program(self.program, program))
-
-        self.program = program
-
-        fluid.save_dygraph(self.module.state_dict(), self.model_path)
-        load_dygraph_vars_to_scope(self.model_path, self.scope, self.place)
-
-        self._remove_model_path()
-
-        out_static_graph = self._run_static_graph(inputs, feed_names,
-                                                  fetch_names)
-
-        if not isinstance(out_dygraph, (list, tuple)):
-            assert len(out_static_graph) == 1
-            out_static_graph = out_static_graph[0]
-
-        return out_dygraph, out_static_graph
 
     def assertEachVar(self, out_dygraph, out_static_graph, func=None):
         if func is None: