[dy2stat]Support save_inference_model in program_translator (#24353)

* support save_inference_model in program_translator test=develop

* fix compatibility with OrderedDict.values() in python3 test=develop

* synchronized random_seed test=develop

* Polish Error Message test=develop

python/paddle/fluid/dygraph/dygraph_to_static/__init__.py

@@ -32,9 +32,6 @@ from .program_translator import *
 from . import convert_call_func
 from .convert_call_func import *
 
-from . import list_transformer
-from .list_transformer import *
-
 __all__ = []
 __all__ += ast_transformer.__all__
 __all__ += loop_transformer.__all__
@@ -42,4 +39,3 @@ __all__ += static_analysis.__all__
 __all__ += variable_trans_func.__all__
 __all__ += program_translator.__all__
 __all__ += convert_call_func.__all__
-__all__ += list_transformer.__all__

python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py

@@ -24,8 +24,6 @@ from paddle.fluid.layers import array_length, array_read, array_write, create_ar
 from paddle.fluid.layers import assign, cast, fill_constant, slice
 from paddle.fluid.layers.control_flow import cond, while_loop, less_than, increment
 
-__all__ = ['convert_list_pop']
-
 
 def create_array_in_parent_blcok(null_array):
     # TODO(liym27): Create a null tensor_array with the same name in parent block to avoid a bug in control flow,
@@ -312,7 +310,7 @@ class ListTransformer(gast.NodeTransformer):
         else:
             idx_str = "None"
 
-        new_call_str = "fluid.dygraph.dygraph_to_static.convert_list_pop({}, {})".format(
+        new_call_str = "fluid.dygraph.dygraph_to_static.list_transformer.convert_list_pop({}, {})".format(
             target_str, idx_str)
         new_call_node = gast.parse(new_call_str).body[0].value
         return new_call_node

python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py

@@ -26,7 +26,9 @@ class PartialProgramLayer(layers.Layer):
     and execute them as a static subgraph.
 
     .. note::
-        **1. It should not be called directly and is used to train dygraph by static mode.
+        **1. This is a very low level API. Users should not use this API
+             directly. Please use `partial_program_from(concrete_program)`
+             to create it.
         **2. LoDTensorArray is not currently supported in the output.
 
     Args:
@@ -43,7 +45,13 @@ class PartialProgramLayer(layers.Layer):
         super(PartialProgramLayer, self).__init__()
         self.inputs = inputs
         self.outputs = outputs
-        self._params = parameters
+        self._params = parameters if parameters is not None else []
+        # Check all params from main program can be found in self._params:
+        # 1. parameter in self._params should be type `framework.ParamBase` which are created in dygraph.
+        # 2. parameter from transformed program shall be found in self._params.
+        #    Because they share same data with ParamBase of original dygraph.
+        self._check_params_all_inited(main_program)
+
         self._infer_program = main_program
         self._train_program = self._append_backward_desc()
         # Switch infer or train by train() and eval()
@@ -155,6 +163,38 @@ class PartialProgramLayer(layers.Layer):
                 continue
             param._set_grad_type(grad_var.type())
 
+    def _check_params_all_inited(self, main_program):
+        """
+        Check all params from main program are already initialized, see details as follows:
+            1. all parameters in self._params should be type `framework.ParamBase` which are created in dygraph.
+            2. all parameters from transformed program can be found in self._params.
+               Because they share same data with ParamBase of original dygraph.
+        """
+        if not isinstance(self._params, (list, tuple)):
+            raise TypeError(
+                "Type of self._params in PartialProgramLayer should be list or tuple, but received %s."
+                % type(self._params))
+
+        params_name_set = set()
+        for i, param in enumerate(self._params):
+            if not isinstance(param, framework.ParamBase):
+                raise TypeError(
+                    'Type of self._params[{}] in PartialProgramLayer should be framework.ParamBase, but received {}.'.
+                    format(i, type(param)))
+            params_name_set.add(param.name)
+
+        for block in main_program.blocks:
+            for name, var in block.vars.items():
+                if isinstance(var, framework.Parameter):
+                    if name not in params_name_set:
+                        raise ValueError(
+                            "\n\tWe don't support to define layer with parameters in the function "
+                            "decorated by `@declarative`.\n\tBecause that will re-defined parameters "
+                            "every time when you run the function.\n\t"
+                            "But we found parameter(%s) was created in the decorated function.\n\t"
+                            "Please define the layer with parameters in `__init__` function."
+                            % name)
+
 
 def valid_vars(vars):
     """
@@ -172,18 +212,6 @@ def valid_vars(vars):
     ]
 
 
-def append_grad_suffix(name):
-    """
-    Append grad suffix to the given variable name.
-    e.g. x ==> x@GRAD
-    """
-    suffix = core.kGradVarSuffix()
-    name = cpt.to_text(name)
-    if suffix not in name:
-        name = name + suffix
-    return name
-
-
 def partial_program_from(concrete_program):
     inputs = concrete_program.inputs
     if inputs and isinstance(inputs[0], layers.Layer):

python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py

@@ -18,9 +18,11 @@ import inspect
 import logging
 import textwrap
 import threading
+import collections
 import numpy as np
-from paddle.fluid import core
+from paddle.fluid import core, scope_guard
 from paddle.fluid import framework
+from paddle.fluid import executor
 from paddle.fluid import unique_name
 from paddle.fluid.dygraph import layers
 from paddle.fluid.dygraph.base import switch_to_static_graph
@@ -92,10 +94,12 @@ class FunctionSpec(object):
         return self._args and isinstance(self._args[0], layers.Layer)
 
     def parameters(self, include_sublayer=True):
-        params = {}
+        params = collections.OrderedDict()
         if self.is_method():
             if include_sublayer:
                 params = self._args[0].parameters()
+                names = [p.name for p in params]
+                params = collections.OrderedDict(zip(names, params))
             else:
                 params = self._args[0]._parameters
         return params
@@ -155,11 +159,11 @@ class ConcreteProgram(object):
                  parameters,
                  func,
                  main_program,
-                 start_up=None):
+                 startup_program=None):
         self.inputs = inputs
         self.outputs = outputs
         self.main_program = main_program
-        self.startup_program = start_up
+        self.startup_program = startup_program
         self.parameters = parameters
         self.func_spec = func
 
@@ -174,18 +178,20 @@ class ConcreteProgram(object):
         dygaph_function = func_spec.dyfunc
         static_func = convert_function_with_cache(dygaph_function)
 
-        main_program, start_up = framework.Program(), framework.Program()
-
-        # Synchronous random seed of program
+        main_program, startup_program = framework.Program(), framework.Program()
+        # Note: The random seed should be synchronized into cached program
+        # if set in `fluid.dygrap_guard` because some ops rely on it, such as
+        # `fluid.layers.dropout`.
         main_program.random_seed = framework.default_main_program().random_seed
-        start_up.random_seed = framework.default_startup_program().random_seed
+        startup_program.random_seed = framework.default_startup_program(
+        ).random_seed
 
-        with framework.program_guard(main_program, start_up):
+        with framework.program_guard(main_program, startup_program):
             # 1. Adds `fluid.data` layers for input if needed
             inputs = func_spec.to_static_inputs(main_program)
 
             # 2. Gets all ParamBases in the function
-            all_parameters = func_spec.parameters()
+            all_parameters = list(func_spec.parameters().values())
 
             # 3. Builds program only once and returns the output Variables.
             with param_guard(func_spec.parameters(False)):
@@ -199,7 +205,7 @@ class ConcreteProgram(object):
             parameters=all_parameters,
             func=dygaph_function,
             main_program=main_program,
-            start_up=start_up)
+            startup_program=startup_program)
 
 
 class ProgramCache(object):
@@ -208,7 +214,7 @@ class ProgramCache(object):
     """
 
     def __init__(self):
-        self._caches = {}
+        self._caches = collections.OrderedDict()
 
     def _build_once(self, func_spec):
         concrete_program = ConcreteProgram.from_func_spec(func_spec)
@@ -223,6 +229,12 @@ class ProgramCache(object):
             self._caches[item] = self._build_once(item)
         return self._caches[item]
 
+    def last(self):
+        assert len(
+            self._caches) >= 1, "No valid cached program in ProgramCache."
+        key = next(reversed(self._caches.keys()))
+        return key, self._caches[key]
+
 
 def synchronized(func):
     func.__lock__ = threading.Lock()
@@ -476,10 +488,20 @@ class ProgramTranslator(object):
 
         func_spec = FunctionSpec(dygraph_func, args, kwargs)
         concrete_program, _ = self._program_cache[func_spec]
+        # Note: concrete_program hold all input/output infos include non-Variable
+        input_vars = [
+            var for var in concrete_program.inputs
+            if isinstance(var, framework.Variable)
+        ]
+        output_vars = [
+            var for var in concrete_program.outputs
+            if isinstance(var, framework.Variable)
+        ]
+
         return concrete_program.main_program, \
                concrete_program.startup_program, \
-               concrete_program.inputs, \
-               concrete_program.outputs
+               input_vars, \
+               output_vars
 
     def get_code(self, dygraph_func):
         """
@@ -527,6 +549,96 @@ class ProgramTranslator(object):
         source_code = ast_to_source_code(root_wrapper.node)
         return source_code
 
+    def save_inference_model(self, dirname, feed=None, fetch=None):
+        """
+        Saves current model as the inference model. It will prune the main_program
+        to build a new program especially for inference, and then save it and all
+        related parameters to given `dirname` . The saved inference model can be
+        loaded by `:ref:`api_fluid_io_load_inference_model` or `C++ inference APIs.
+
+        Args:
+            dirname (str): the directory to save the inference model.
+            feed (list[int], optional): the input variable indices of the saved
+                inference model. If None, all input variables of the
+                ProgramTranslator would be the inputs of the saved inference
+                model. Default None.
+            fetch (list[int], optional): the output variable indices of the
+                saved inference model. If None, all output variables of the
+                TracedLayer object would be the outputs of the saved inference
+                model. Default None.
+        Returns:
+            None
+        Examples:
+            .. code-block:: python
+                import numpy as np
+                import paddle.fluid as fluid
+                from paddle.fluid.dygraph import Linear
+                from paddle.fluid.dygraph import ProgramTranslator
+
+                class SimpleNet(fluid.dygraph.Layer):
+                    def __init__(self, in_size, out_size):
+                        super(SimpleNet, self).__init__()
+                        self._linear = Linear(in_size, out_size)
+
+                    @declarative
+                    def forward(self, x):
+                        y = self._linear(x)
+                        z = self._linear(y)
+                        loss = fluid.layers.mean(z)
+                        return z, loss
+
+                with fluid.dygraph.guard(fluid.CPUPlace()):
+                    net = SimpleNet(8, 8)
+                    adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters())
+                    x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32'))
+                    for i in range(10):
+                        loss, out = net(x)
+                        loss.backward()
+                        adam.minimize(loss)
+                        net.clear_gradients()
+                # Save inference model.
+                # Note that fetch=[0] means we set 'y' as the inference output.
+                prog_trans = ProgramTranslator()
+                prog_trans.save_inference_model("./dy2stat_infer_model", fetch=[0])
+
+                # In this example, the inference model will be pruned based on input (x) and
+                # output (y). The pruned inference program is going to be saved in the folder
+                # "./dy2stat_infer_model" and parameters are going to be saved in separate
+                # files in the folder.
+        """
+
+        def get_feed_fetch(var_list, partial_vars, return_name=False):
+            vars = [
+                var for var in var_list if isinstance(var, framework.Variable)
+            ]
+            if partial_vars:
+                vars = [vars[idx] for idx in partial_vars]
+            if return_name:
+                vars = [var.name for var in vars]
+
+            return vars
+
+        func_spec, (concrete_program,
+                    partial_layer) = self._program_cache.last()
+        # share paramBase data with parameter
+        scope = core.Scope()
+        for param_base in concrete_program.parameters:
+            param_tensor = scope.var(param_base.name).get_tensor()
+            src_tensor = param_base.value().get_tensor()
+            param_tensor._share_data_with(src_tensor)
+
+        feed_var_names = get_feed_fetch(concrete_program.inputs, feed, True)
+        fetch_vars = get_feed_fetch(concrete_program.outputs, fetch)
+
+        from paddle.fluid.io import save_inference_model
+        with scope_guard(scope):
+            save_inference_model(
+                dirname=dirname,
+                feeded_var_names=feed_var_names,
+                target_vars=fetch_vars,
+                executor=executor.Executor(framework._current_expected_place()),
+                main_program=concrete_program.main_program.clone())
+
     def get_program_cache(self):
         """
         Returns the ProgramCache instance. This method is used by PaddlePaddle

python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py

@@ -21,6 +21,7 @@ import numpy as np
 
 import paddle
 import paddle.fluid as fluid
+from paddle.fluid.dygraph.base import switch_to_static_graph
 from paddle.fluid.dygraph import to_variable
 from paddle.fluid.dygraph.nn import Conv2D, Linear, Pool2D
 from paddle.fluid.optimizer import AdamOptimizer
@@ -193,9 +194,32 @@ class TestMNISTWithDeclarative(TestMNIST):
                         mnist.eval()
                         prediction, acc, avg_loss = mnist(img, label)
                         loss_data.append(avg_loss.numpy()[0])
+                        self.check_save_inference_model([dy_x_data, y_data],
+                                                        prog_trans, to_static,
+                                                        prediction)
                         break
         return loss_data
 
+    @switch_to_static_graph
+    def check_save_inference_model(self, inputs, prog_trans, to_static, gt_out):
+        if to_static:
+            infer_model_path = "./test_mnist_inference_model"
+            prog_trans.save_inference_model(infer_model_path)
+            infer_out = self.load_and_run_inference(infer_model_path, inputs)
+            self.assertTrue(np.allclose(gt_out.numpy(), infer_out))
+
+    def load_and_run_inference(self, model_path, inputs):
+        exe = fluid.Executor(self.place)
+        [inference_program, feed_target_names,
+         fetch_targets] = fluid.io.load_inference_model(
+             dirname=model_path, executor=exe)
+        assert len(inputs) == len(feed_target_names)
+        results = exe.run(inference_program,
+                          feed=dict(zip(feed_target_names, inputs)),
+                          fetch_list=fetch_targets)
+
+        return np.array(results[0])
+
 
 if __name__ == "__main__":
     unittest.main()

python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py

@@ -102,6 +102,14 @@ class StaticCode2():
         return x_v
 
 
+class NetWithError(fluid.dygraph.layers.Layer):
+    @declarative
+    def forward(self, x):
+        linear = fluid.dygraph.Linear(32, 64)
+        y = linear(x)
+        return y
+
+
 class TestDygraphToStaticCode(unittest.TestCase):
     def setUp(self):
         # set to print all string diff when assertEqual fails
@@ -122,73 +130,79 @@ class TestDygraphToStaticCode(unittest.TestCase):
 
 
 class TestEnableDeclarative(unittest.TestCase):
-    def test_enable_disable_get_output(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
+    def setUp(self):
+        self.x = np.random.randn(30, 10, 32).astype('float32')
+        self.weight = np.random.randn(32, 64).astype('float32')
+        self.program_translator = ProgramTranslator()
+
+    def test_raise_error(self):
+        with fluid.dygraph.guard():
+            self.program_translator.enable(True)
+            net = NetWithError()
+            with self.assertRaises(ValueError):
+                net(fluid.dygraph.to_variable(self.x))
 
+    def test_enable_disable_get_output(self):
+        self.program_translator.enable(True)
         with fluid.dygraph.guard():
-            program_translator.enable(True)
-            static_output = program_translator.get_output(simple_func, x,
-                                                          weight)
+            static_output = self.program_translator.get_output(
+                simple_func, self.x, self.weight)
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_output = program_translator.get_output(simple_func, x,
-                                                           weight)
+            dygraph_output = self.program_translator.get_output(
+                simple_func, self.x, self.weight)
             self.assertTrue(
                 np.allclose(
                     static_output.numpy(), dygraph_output.numpy(), atol=1e-4))
 
     def test_enable_disable_get_func(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
+        self.program_translator.enable(True)
         with fluid.dygraph.guard():
-            program_translator.enable(True)
-            static_func = program_translator.get_func(simple_func)
+            static_func = self.program_translator.get_func(simple_func)
             self.assertTrue(callable(static_func))
-            static_output = static_func(x, weight)
+            static_output = static_func(self.x, self.weight)
             self.assertTrue(isinstance(static_output, fluid.Variable))
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_func = program_translator.get_func(simple_func)
+            dygraph_func = self.program_translator.get_func(simple_func)
             self.assertTrue(callable(dygraph_func))
-            dygraph_output = dygraph_func(x, weight)
+            dygraph_output = dygraph_func(self.x, self.weight)
             self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase))
 
     def test_enable_disable_get_program(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
-        program_translator.enable(True)
-        static_output = program_translator.get_program(simple_func, x, weight)
+        self.program_translator.enable(True)
+        static_output = self.program_translator.get_program(simple_func, self.x,
+                                                            self.weight)
         self.assertTrue(isinstance(static_output, tuple))
         self.assertEqual(len(static_output), 4)
         self.assertTrue(isinstance(static_output[0], fluid.Program))
         self.assertTrue(isinstance(static_output[1], fluid.Program))
+        # Check all inputs and outputs are Variable
+        for var in static_output[2]:
+            self.assertTrue(isinstance(var, fluid.Variable))
+
+        for var in static_output[3]:
+            self.assertTrue(isinstance(var, fluid.Variable))
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_output = program_translator.get_program(simple_func, x,
-                                                            weight)
+            dygraph_output = self.program_translator.get_program(
+                simple_func, self.x, self.weight)
             self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase))
 
     def test_enable_disable_declarative(self):
-        x = np.random.randn(30, 10, 32).astype('float32')
-        weight = np.random.randn(32, 64).astype('float32')
-        program_translator = ProgramTranslator()
 
+        self.program_translator.enable(True)
         with fluid.dygraph.guard():
-            program_translator.enable(True)
-            static_output = decorated_simple_func(x, weight)
+            static_output = decorated_simple_func(self.x, self.weight)
 
-        program_translator.enable(False)
+        self.program_translator.enable(False)
         with fluid.dygraph.guard():
-            dygraph_output = decorated_simple_func(x, weight)
+            dygraph_output = decorated_simple_func(self.x, self.weight)
             self.assertTrue(
                 np.allclose(
                     static_output.numpy(), dygraph_output.numpy(), atol=1e-4))

python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py

@@ -22,8 +22,11 @@ import paddle.fluid as fluid
 
 from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator
 from paddle.fluid.dygraph.jit import declarative
+from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from
 
-np.random.seed(2020)
+SEED = 2020
+
+np.random.seed(SEED)
 
 place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace(
 )
@@ -36,7 +39,6 @@ class SimpleFcLayer(fluid.dygraph.Layer):
 
     @declarative
     def forward(self, x):
-        x = fluid.dygraph.to_variable(x)
         y = self._linear(x)
         z = self._linear(y)
         out = fluid.layers.mean(z)
@@ -44,40 +46,92 @@ class SimpleFcLayer(fluid.dygraph.Layer):
 
 
 class TestDyToStaticSaveInferenceModel(unittest.TestCase):
-    # TODO(Aurelius84): disable temporarily, need new save_inference interface
-    def _test_save_inference_model(self):
+    def test_save_inference_model(self):
         fc_size = 20
+        x_data = np.random.random((fc_size, fc_size)).astype('float32')
+        with fluid.dygraph.guard(place):
+            fluid.default_startup_program().random_seed = SEED
+            fluid.default_main_program().random_seed = SEED
 
-        x = np.random.random((fc_size, fc_size)).astype('float32')
+            x = fluid.dygraph.to_variable(x_data)
             layer = SimpleFcLayer(fc_size)
-
-        program_translator = ProgramTranslator.get_instance()
-        adam = fluid.optimizer.SGD(learning_rate=0.001)
-        program_translator.set_optimizer(adam, index_of_loss=0)
+            adam = fluid.optimizer.SGD(learning_rate=0.1,
+                                       parameter_list=layer.parameters())
 
             for i in range(5):
-            out = layer(x)
-
-        main_program = ProgramTranslator.get_instance().main_program
-        expected_persistable_vars = set(
-            [layer._linear.weight.name, layer._linear.bias.name])
+                loss, _ = layer(x)
+                loss.backward()
+                adam.minimize(loss)
+                layer.clear_gradients()
+            # Check the correctness of the inference
+            dygraph_out, _ = layer(x)
+        self.check_save_inference_model(layer, [x_data], dygraph_out.numpy())
+        self.check_save_inference_model(
+            layer, [x_data], dygraph_out.numpy(), fetch=[0])
+        self.check_save_inference_model(
+            layer, [x_data], dygraph_out.numpy(), feed=[0])
+
+    def check_save_inference_model(self,
+                                   model,
+                                   inputs,
+                                   gt_out,
+                                   feed=None,
+                                   fetch=None):
+        program_translator = ProgramTranslator()
+        expected_persistable_vars = set([p.name for p in model.parameters()])
 
         infer_model_dir = "./test_dy2stat_save_inference_model"
-        ProgramTranslator.get_instance().save_inference_model(infer_model_dir)
-        saved_var_names = set([
-            filename for filename in os.listdir(infer_model_dir)
-            if filename != '__model__'
-        ])
-        self.assertEqual(saved_var_names, expected_persistable_vars)
-
-        infer_model_dir = "./test_dy2stat_save_inference_model_with_fetch"
-        ProgramTranslator.get_instance().save_inference_model(
-            infer_model_dir, fetch=[0])
+        program_translator.save_inference_model(
+            infer_model_dir, feed=feed, fetch=fetch)
         saved_var_names = set([
             filename for filename in os.listdir(infer_model_dir)
             if filename != '__model__'
         ])
         self.assertEqual(saved_var_names, expected_persistable_vars)
+        # Check the correctness of the inference
+        infer_out = self.load_and_run_inference(infer_model_dir, inputs)
+        self.assertTrue(np.allclose(gt_out, infer_out))
+
+    def load_and_run_inference(self, model_path, inputs):
+        exe = fluid.Executor(place)
+        [inference_program, feed_target_names,
+         fetch_targets] = fluid.io.load_inference_model(
+             dirname=model_path, executor=exe)
+        results = exe.run(inference_program,
+                          feed=dict(zip(feed_target_names, inputs)),
+                          fetch_list=fetch_targets)
+
+        return np.array(results[0])
+
+
+class TestPartialProgramRaiseError(unittest.TestCase):
+    def test_param_type(self):
+        program_translator = ProgramTranslator()
+        program_translator.enable(True)
+        x_data = np.random.random((20, 20)).astype('float32')
+
+        with fluid.dygraph.guard(fluid.CPUPlace()):
+            net = SimpleFcLayer(20)
+            x = fluid.dygraph.to_variable(x_data)
+            out = net(x)
+
+            program_cache = program_translator.get_program_cache()
+            _, (concrete_program, _) = program_cache.last()
+
+            params = concrete_program.parameters
+
+            concrete_program.parameters = params[0]
+            # TypeError: Type of self._params should be list or tuple,
+            # but received <class 'paddle.fluid.framework.ParamBase'>.
+            with self.assertRaises(TypeError):
+                partial_program_from(concrete_program)
+
+            params[0] = "linear.w.0"
+            concrete_program.parameters = params
+            # TypeError: Type of self._params[0] should be framework.ParamBase,
+            # but received <type 'str'>.
+            with self.assertRaises(TypeError):
+                partial_program_from(concrete_program)
 
 
 if __name__ == '__main__':