diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py b/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py index c1a884f3bad28ff1b4ce963ae392d279598bf3cd..d2d03d65b1b819d1a6ea0bd308ee776ede2cfaef 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/__init__.py @@ -32,9 +32,6 @@ from .program_translator import * from . import convert_call_func from .convert_call_func import * -from . import list_transformer -from .list_transformer import * - __all__ = [] __all__ += ast_transformer.__all__ __all__ += loop_transformer.__all__ @@ -42,4 +39,3 @@ __all__ += static_analysis.__all__ __all__ += variable_trans_func.__all__ __all__ += program_translator.__all__ __all__ += convert_call_func.__all__ -__all__ += list_transformer.__all__ diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py index 4136838dec9ee63a215cd18bdb20ef72ac70b256..0af980dacd8fa8a422951186046be2f33ca1bc7d 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/list_transformer.py @@ -24,8 +24,6 @@ from paddle.fluid.layers import array_length, array_read, array_write, create_ar from paddle.fluid.layers import assign, cast, fill_constant, slice from paddle.fluid.layers.control_flow import cond, while_loop, less_than, increment -__all__ = ['convert_list_pop'] - def create_array_in_parent_blcok(null_array): # TODO(liym27): Create a null tensor_array with the same name in parent block to avoid a bug in control flow, @@ -312,7 +310,7 @@ class ListTransformer(gast.NodeTransformer): else: idx_str = "None" - new_call_str = "fluid.dygraph.dygraph_to_static.convert_list_pop({}, {})".format( + new_call_str = "fluid.dygraph.dygraph_to_static.list_transformer.convert_list_pop({}, {})".format( target_str, idx_str) new_call_node = gast.parse(new_call_str).body[0].value return new_call_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index f906d86872a7f0e21d65f3e96e1131230d4bbc90..31a37e7e2bb9110582932f7156612c684448c5ff 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -26,7 +26,9 @@ class PartialProgramLayer(layers.Layer): and execute them as a static subgraph. .. note:: - **1. It should not be called directly and is used to train dygraph by static mode. + **1. This is a very low level API. Users should not use this API + directly. Please use `partial_program_from(concrete_program)` + to create it. **2. LoDTensorArray is not currently supported in the output. Args: @@ -43,7 +45,13 @@ class PartialProgramLayer(layers.Layer): super(PartialProgramLayer, self).__init__() self.inputs = inputs self.outputs = outputs - self._params = parameters + self._params = parameters if parameters is not None else [] + # Check all params from main program can be found in self._params: + # 1. parameter in self._params should be type `framework.ParamBase` which are created in dygraph. + # 2. parameter from transformed program shall be found in self._params. + # Because they share same data with ParamBase of original dygraph. + self._check_params_all_inited(main_program) + self._infer_program = main_program self._train_program = self._append_backward_desc() # Switch infer or train by train() and eval() @@ -155,6 +163,38 @@ class PartialProgramLayer(layers.Layer): continue param._set_grad_type(grad_var.type()) + def _check_params_all_inited(self, main_program): + """ + Check all params from main program are already initialized, see details as follows: + 1. all parameters in self._params should be type `framework.ParamBase` which are created in dygraph. + 2. all parameters from transformed program can be found in self._params. + Because they share same data with ParamBase of original dygraph. + """ + if not isinstance(self._params, (list, tuple)): + raise TypeError( + "Type of self._params in PartialProgramLayer should be list or tuple, but received %s." + % type(self._params)) + + params_name_set = set() + for i, param in enumerate(self._params): + if not isinstance(param, framework.ParamBase): + raise TypeError( + 'Type of self._params[{}] in PartialProgramLayer should be framework.ParamBase, but received {}.'. + format(i, type(param))) + params_name_set.add(param.name) + + for block in main_program.blocks: + for name, var in block.vars.items(): + if isinstance(var, framework.Parameter): + if name not in params_name_set: + raise ValueError( + "\n\tWe don't support to define layer with parameters in the function " + "decorated by `@declarative`.\n\tBecause that will re-defined parameters " + "every time when you run the function.\n\t" + "But we found parameter(%s) was created in the decorated function.\n\t" + "Please define the layer with parameters in `__init__` function." + % name) + def valid_vars(vars): """ @@ -172,18 +212,6 @@ def valid_vars(vars): ] -def append_grad_suffix(name): - """ - Append grad suffix to the given variable name. - e.g. x ==> x@GRAD - """ - suffix = core.kGradVarSuffix() - name = cpt.to_text(name) - if suffix not in name: - name = name + suffix - return name - - def partial_program_from(concrete_program): inputs = concrete_program.inputs if inputs and isinstance(inputs[0], layers.Layer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index ee952babbb004fe174cea8c13d0aac1defb83393..f480df8a6fdd898b95bf2122cee6cff62b3166a8 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -18,9 +18,11 @@ import inspect import logging import textwrap import threading +import collections import numpy as np -from paddle.fluid import core +from paddle.fluid import core, scope_guard from paddle.fluid import framework +from paddle.fluid import executor from paddle.fluid import unique_name from paddle.fluid.dygraph import layers from paddle.fluid.dygraph.base import switch_to_static_graph @@ -92,10 +94,12 @@ class FunctionSpec(object): return self._args and isinstance(self._args[0], layers.Layer) def parameters(self, include_sublayer=True): - params = {} + params = collections.OrderedDict() if self.is_method(): if include_sublayer: params = self._args[0].parameters() + names = [p.name for p in params] + params = collections.OrderedDict(zip(names, params)) else: params = self._args[0]._parameters return params @@ -155,11 +159,11 @@ class ConcreteProgram(object): parameters, func, main_program, - start_up=None): + startup_program=None): self.inputs = inputs self.outputs = outputs self.main_program = main_program - self.startup_program = start_up + self.startup_program = startup_program self.parameters = parameters self.func_spec = func @@ -174,18 +178,20 @@ class ConcreteProgram(object): dygaph_function = func_spec.dyfunc static_func = convert_function_with_cache(dygaph_function) - main_program, start_up = framework.Program(), framework.Program() - - # Synchronous random seed of program + main_program, startup_program = framework.Program(), framework.Program() + # Note: The random seed should be synchronized into cached program + # if set in `fluid.dygrap_guard` because some ops rely on it, such as + # `fluid.layers.dropout`. main_program.random_seed = framework.default_main_program().random_seed - start_up.random_seed = framework.default_startup_program().random_seed + startup_program.random_seed = framework.default_startup_program( + ).random_seed - with framework.program_guard(main_program, start_up): + with framework.program_guard(main_program, startup_program): # 1. Adds `fluid.data` layers for input if needed inputs = func_spec.to_static_inputs(main_program) # 2. Gets all ParamBases in the function - all_parameters = func_spec.parameters() + all_parameters = list(func_spec.parameters().values()) # 3. Builds program only once and returns the output Variables. with param_guard(func_spec.parameters(False)): @@ -199,7 +205,7 @@ class ConcreteProgram(object): parameters=all_parameters, func=dygaph_function, main_program=main_program, - start_up=start_up) + startup_program=startup_program) class ProgramCache(object): @@ -208,7 +214,7 @@ class ProgramCache(object): """ def __init__(self): - self._caches = {} + self._caches = collections.OrderedDict() def _build_once(self, func_spec): concrete_program = ConcreteProgram.from_func_spec(func_spec) @@ -223,6 +229,12 @@ class ProgramCache(object): self._caches[item] = self._build_once(item) return self._caches[item] + def last(self): + assert len( + self._caches) >= 1, "No valid cached program in ProgramCache." + key = next(reversed(self._caches.keys())) + return key, self._caches[key] + def synchronized(func): func.__lock__ = threading.Lock() @@ -476,10 +488,20 @@ class ProgramTranslator(object): func_spec = FunctionSpec(dygraph_func, args, kwargs) concrete_program, _ = self._program_cache[func_spec] + # Note: concrete_program hold all input/output infos include non-Variable + input_vars = [ + var for var in concrete_program.inputs + if isinstance(var, framework.Variable) + ] + output_vars = [ + var for var in concrete_program.outputs + if isinstance(var, framework.Variable) + ] + return concrete_program.main_program, \ concrete_program.startup_program, \ - concrete_program.inputs, \ - concrete_program.outputs + input_vars, \ + output_vars def get_code(self, dygraph_func): """ @@ -527,6 +549,96 @@ class ProgramTranslator(object): source_code = ast_to_source_code(root_wrapper.node) return source_code + def save_inference_model(self, dirname, feed=None, fetch=None): + """ + Saves current model as the inference model. It will prune the main_program + to build a new program especially for inference, and then save it and all + related parameters to given `dirname` . The saved inference model can be + loaded by `:ref:`api_fluid_io_load_inference_model` or `C++ inference APIs. + + Args: + dirname (str): the directory to save the inference model. + feed (list[int], optional): the input variable indices of the saved + inference model. If None, all input variables of the + ProgramTranslator would be the inputs of the saved inference + model. Default None. + fetch (list[int], optional): the output variable indices of the + saved inference model. If None, all output variables of the + TracedLayer object would be the outputs of the saved inference + model. Default None. + Returns: + None + Examples: + .. code-block:: python + import numpy as np + import paddle.fluid as fluid + from paddle.fluid.dygraph import Linear + from paddle.fluid.dygraph import ProgramTranslator + + class SimpleNet(fluid.dygraph.Layer): + def __init__(self, in_size, out_size): + super(SimpleNet, self).__init__() + self._linear = Linear(in_size, out_size) + + @declarative + def forward(self, x): + y = self._linear(x) + z = self._linear(y) + loss = fluid.layers.mean(z) + return z, loss + + with fluid.dygraph.guard(fluid.CPUPlace()): + net = SimpleNet(8, 8) + adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) + x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + for i in range(10): + loss, out = net(x) + loss.backward() + adam.minimize(loss) + net.clear_gradients() + # Save inference model. + # Note that fetch=[0] means we set 'y' as the inference output. + prog_trans = ProgramTranslator() + prog_trans.save_inference_model("./dy2stat_infer_model", fetch=[0]) + + # In this example, the inference model will be pruned based on input (x) and + # output (y). The pruned inference program is going to be saved in the folder + # "./dy2stat_infer_model" and parameters are going to be saved in separate + # files in the folder. + """ + + def get_feed_fetch(var_list, partial_vars, return_name=False): + vars = [ + var for var in var_list if isinstance(var, framework.Variable) + ] + if partial_vars: + vars = [vars[idx] for idx in partial_vars] + if return_name: + vars = [var.name for var in vars] + + return vars + + func_spec, (concrete_program, + partial_layer) = self._program_cache.last() + # share paramBase data with parameter + scope = core.Scope() + for param_base in concrete_program.parameters: + param_tensor = scope.var(param_base.name).get_tensor() + src_tensor = param_base.value().get_tensor() + param_tensor._share_data_with(src_tensor) + + feed_var_names = get_feed_fetch(concrete_program.inputs, feed, True) + fetch_vars = get_feed_fetch(concrete_program.outputs, fetch) + + from paddle.fluid.io import save_inference_model + with scope_guard(scope): + save_inference_model( + dirname=dirname, + feeded_var_names=feed_var_names, + target_vars=fetch_vars, + executor=executor.Executor(framework._current_expected_place()), + main_program=concrete_program.main_program.clone()) + def get_program_cache(self): """ Returns the ProgramCache instance. This method is used by PaddlePaddle diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py index 050e701a6df9d4269889a4def3b4322877746608..2880dd00559c33aa6dc9987d715a9cf87ac8e1cb 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py @@ -21,6 +21,7 @@ import numpy as np import paddle import paddle.fluid as fluid +from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph import to_variable from paddle.fluid.dygraph.nn import Conv2D, Linear, Pool2D from paddle.fluid.optimizer import AdamOptimizer @@ -193,9 +194,32 @@ class TestMNISTWithDeclarative(TestMNIST): mnist.eval() prediction, acc, avg_loss = mnist(img, label) loss_data.append(avg_loss.numpy()[0]) + self.check_save_inference_model([dy_x_data, y_data], + prog_trans, to_static, + prediction) break return loss_data + @switch_to_static_graph + def check_save_inference_model(self, inputs, prog_trans, to_static, gt_out): + if to_static: + infer_model_path = "./test_mnist_inference_model" + prog_trans.save_inference_model(infer_model_path) + infer_out = self.load_and_run_inference(infer_model_path, inputs) + self.assertTrue(np.allclose(gt_out.numpy(), infer_out)) + + def load_and_run_inference(self, model_path, inputs): + exe = fluid.Executor(self.place) + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + dirname=model_path, executor=exe) + assert len(inputs) == len(feed_target_names) + results = exe.run(inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets) + + return np.array(results[0]) + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index ad278a848edf469d912117849f7abd75dea59d27..373a5e867c76600e86aac25a589ad1b481c4ee8d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -102,6 +102,14 @@ class StaticCode2(): return x_v +class NetWithError(fluid.dygraph.layers.Layer): + @declarative + def forward(self, x): + linear = fluid.dygraph.Linear(32, 64) + y = linear(x) + return y + + class TestDygraphToStaticCode(unittest.TestCase): def setUp(self): # set to print all string diff when assertEqual fails @@ -122,73 +130,79 @@ class TestDygraphToStaticCode(unittest.TestCase): class TestEnableDeclarative(unittest.TestCase): - def test_enable_disable_get_output(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() + def setUp(self): + self.x = np.random.randn(30, 10, 32).astype('float32') + self.weight = np.random.randn(32, 64).astype('float32') + self.program_translator = ProgramTranslator() + + def test_raise_error(self): + with fluid.dygraph.guard(): + self.program_translator.enable(True) + net = NetWithError() + with self.assertRaises(ValueError): + net(fluid.dygraph.to_variable(self.x)) + def test_enable_disable_get_output(self): + self.program_translator.enable(True) with fluid.dygraph.guard(): - program_translator.enable(True) - static_output = program_translator.get_output(simple_func, x, - weight) + static_output = self.program_translator.get_output( + simple_func, self.x, self.weight) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_output = program_translator.get_output(simple_func, x, - weight) + dygraph_output = self.program_translator.get_output( + simple_func, self.x, self.weight) self.assertTrue( np.allclose( static_output.numpy(), dygraph_output.numpy(), atol=1e-4)) def test_enable_disable_get_func(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() + self.program_translator.enable(True) with fluid.dygraph.guard(): - program_translator.enable(True) - static_func = program_translator.get_func(simple_func) + static_func = self.program_translator.get_func(simple_func) self.assertTrue(callable(static_func)) - static_output = static_func(x, weight) + static_output = static_func(self.x, self.weight) self.assertTrue(isinstance(static_output, fluid.Variable)) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_func = program_translator.get_func(simple_func) + dygraph_func = self.program_translator.get_func(simple_func) self.assertTrue(callable(dygraph_func)) - dygraph_output = dygraph_func(x, weight) + dygraph_output = dygraph_func(self.x, self.weight) self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase)) def test_enable_disable_get_program(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() - program_translator.enable(True) - static_output = program_translator.get_program(simple_func, x, weight) + self.program_translator.enable(True) + static_output = self.program_translator.get_program(simple_func, self.x, + self.weight) self.assertTrue(isinstance(static_output, tuple)) self.assertEqual(len(static_output), 4) self.assertTrue(isinstance(static_output[0], fluid.Program)) self.assertTrue(isinstance(static_output[1], fluid.Program)) + # Check all inputs and outputs are Variable + for var in static_output[2]: + self.assertTrue(isinstance(var, fluid.Variable)) + + for var in static_output[3]: + self.assertTrue(isinstance(var, fluid.Variable)) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_output = program_translator.get_program(simple_func, x, - weight) + dygraph_output = self.program_translator.get_program( + simple_func, self.x, self.weight) self.assertTrue(isinstance(dygraph_output, fluid.core.VarBase)) def test_enable_disable_declarative(self): - x = np.random.randn(30, 10, 32).astype('float32') - weight = np.random.randn(32, 64).astype('float32') - program_translator = ProgramTranslator() + self.program_translator.enable(True) with fluid.dygraph.guard(): - program_translator.enable(True) - static_output = decorated_simple_func(x, weight) + static_output = decorated_simple_func(self.x, self.weight) - program_translator.enable(False) + self.program_translator.enable(False) with fluid.dygraph.guard(): - dygraph_output = decorated_simple_func(x, weight) + dygraph_output = decorated_simple_func(self.x, self.weight) self.assertTrue( np.allclose( static_output.numpy(), dygraph_output.numpy(), atol=1e-4)) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py index d44d162615030632818634721d90efb943cd8ab2..7414d240bf5727886f0d8ae5b493194f0bdd25e3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py @@ -22,8 +22,11 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator from paddle.fluid.dygraph.jit import declarative +from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from -np.random.seed(2020) +SEED = 2020 + +np.random.seed(SEED) place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace( ) @@ -36,7 +39,6 @@ class SimpleFcLayer(fluid.dygraph.Layer): @declarative def forward(self, x): - x = fluid.dygraph.to_variable(x) y = self._linear(x) z = self._linear(y) out = fluid.layers.mean(z) @@ -44,40 +46,92 @@ class SimpleFcLayer(fluid.dygraph.Layer): class TestDyToStaticSaveInferenceModel(unittest.TestCase): - # TODO(Aurelius84): disable temporarily, need new save_inference interface - def _test_save_inference_model(self): + def test_save_inference_model(self): fc_size = 20 - - x = np.random.random((fc_size, fc_size)).astype('float32') - layer = SimpleFcLayer(fc_size) - - program_translator = ProgramTranslator.get_instance() - adam = fluid.optimizer.SGD(learning_rate=0.001) - program_translator.set_optimizer(adam, index_of_loss=0) - - for i in range(5): - out = layer(x) - - main_program = ProgramTranslator.get_instance().main_program - expected_persistable_vars = set( - [layer._linear.weight.name, layer._linear.bias.name]) + x_data = np.random.random((fc_size, fc_size)).astype('float32') + with fluid.dygraph.guard(place): + fluid.default_startup_program().random_seed = SEED + fluid.default_main_program().random_seed = SEED + + x = fluid.dygraph.to_variable(x_data) + layer = SimpleFcLayer(fc_size) + adam = fluid.optimizer.SGD(learning_rate=0.1, + parameter_list=layer.parameters()) + + for i in range(5): + loss, _ = layer(x) + loss.backward() + adam.minimize(loss) + layer.clear_gradients() + # Check the correctness of the inference + dygraph_out, _ = layer(x) + self.check_save_inference_model(layer, [x_data], dygraph_out.numpy()) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), fetch=[0]) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), feed=[0]) + + def check_save_inference_model(self, + model, + inputs, + gt_out, + feed=None, + fetch=None): + program_translator = ProgramTranslator() + expected_persistable_vars = set([p.name for p in model.parameters()]) infer_model_dir = "./test_dy2stat_save_inference_model" - ProgramTranslator.get_instance().save_inference_model(infer_model_dir) - saved_var_names = set([ - filename for filename in os.listdir(infer_model_dir) - if filename != '__model__' - ]) - self.assertEqual(saved_var_names, expected_persistable_vars) - - infer_model_dir = "./test_dy2stat_save_inference_model_with_fetch" - ProgramTranslator.get_instance().save_inference_model( - infer_model_dir, fetch=[0]) + program_translator.save_inference_model( + infer_model_dir, feed=feed, fetch=fetch) saved_var_names = set([ filename for filename in os.listdir(infer_model_dir) if filename != '__model__' ]) self.assertEqual(saved_var_names, expected_persistable_vars) + # Check the correctness of the inference + infer_out = self.load_and_run_inference(infer_model_dir, inputs) + self.assertTrue(np.allclose(gt_out, infer_out)) + + def load_and_run_inference(self, model_path, inputs): + exe = fluid.Executor(place) + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + dirname=model_path, executor=exe) + results = exe.run(inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets) + + return np.array(results[0]) + + +class TestPartialProgramRaiseError(unittest.TestCase): + def test_param_type(self): + program_translator = ProgramTranslator() + program_translator.enable(True) + x_data = np.random.random((20, 20)).astype('float32') + + with fluid.dygraph.guard(fluid.CPUPlace()): + net = SimpleFcLayer(20) + x = fluid.dygraph.to_variable(x_data) + out = net(x) + + program_cache = program_translator.get_program_cache() + _, (concrete_program, _) = program_cache.last() + + params = concrete_program.parameters + + concrete_program.parameters = params[0] + # TypeError: Type of self._params should be list or tuple, + # but received . + with self.assertRaises(TypeError): + partial_program_from(concrete_program) + + params[0] = "linear.w.0" + concrete_program.parameters = params + # TypeError: Type of self._params[0] should be framework.ParamBase, + # but received . + with self.assertRaises(TypeError): + partial_program_from(concrete_program) if __name__ == '__main__':