From 8924b03bec042c63a1dcf1cc47b1743fb047db2f Mon Sep 17 00:00:00 2001 From: Bin Li Date: Tue, 15 Jan 2019 22:49:36 +0800 Subject: [PATCH] Support specifying layers to validate --- .gitlab-ci.yml | 4 +-- docs/development/how_to_debug.rst | 2 +- .../tools/converter_tool/hexagon_converter.py | 10 +++++- .../tools/converter_tool/transformer.py | 8 ++++- mace/python/tools/layers_validate.py | 31 ++++++++++++++++++- tools/converter.py | 9 +++--- tools/device.py | 14 +++++---- 7 files changed, 62 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7730f99..76907b14 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,7 +147,7 @@ python_tools_tests: python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,armhf --validate --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1; + python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; model_tests: stage: model_tests @@ -190,7 +190,7 @@ quantization_tests: python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1; + python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; done - rm -rf mace-models diff --git a/docs/development/how_to_debug.rst b/docs/development/how_to_debug.rst index 0c04cebc..ea468858 100644 --- a/docs/development/how_to_debug.rst +++ b/docs/development/how_to_debug.rst @@ -71,7 +71,7 @@ If you want to specify input data to use, you can add an option in yaml config u If model's output is suspected to be incorrect, it might be useful to debug your model layer by layer by specifying an intermediate layer as output, or use binary search method until suspicious layer is found. -You can also specify `--validate_all_layers` to validate all the layers of the model(excluding some layers changed by MACE, e.g., BatchToSpaceND), +You can also specify `--layers` after `--validate` to validate all or some of the layers of the model(excluding some layers changed by MACE, e.g., BatchToSpaceND), it only supports TensorFlow now. You can find validation results in `builds/your_model/model/runtime_in_yaml/log.csv`. For quantized model, if you want to check one layer, you can add `check_tensors` and `check_shapes` like in the yaml above. You can only specify diff --git a/mace/python/tools/converter_tool/hexagon_converter.py b/mace/python/tools/converter_tool/hexagon_converter.py index f37ab7ba..42154b99 100644 --- a/mace/python/tools/converter_tool/hexagon_converter.py +++ b/mace/python/tools/converter_tool/hexagon_converter.py @@ -365,9 +365,17 @@ class HexagonConverter(base_converter.ConverterInterface): node_id_map[tensor_op] = tensor.node_id print("Hexagon op:") + index = 0 for op in self._model.op: op.node_id = node_id_counter - print('Op: %s (%s, %d)' % (op.name, op.type, op.node_id)) + if op.type not in [HexagonOp.QuantizeINPUT_f_to_8, + HexagonOp.DequantizeOUTPUT_8tof.name]: + index_str = str(index) + index += 1 + else: + index_str = '' + print('Op: %s (%s, node_id:%d, index:%s)' % + (op.name, op.type, op.node_id, index_str)) node_id_counter += 1 node_id_map[op.name] = op.node_id for ipt in op.input: diff --git a/mace/python/tools/converter_tool/transformer.py b/mace/python/tools/converter_tool/transformer.py index 9ea6b6d8..07ae7d8a 100644 --- a/mace/python/tools/converter_tool/transformer.py +++ b/mace/python/tools/converter_tool/transformer.py @@ -1370,8 +1370,14 @@ class Transformer(base_converter.ConverterInterface): net.op.extend(sorted_nodes) print("Final ops:") + index = 0 for op in net.op: - print("%s (%s): %s" % (op.name, op.type, [ + if op.type not in [MaceOp.Quantize.name, MaceOp.Dequantize.name]: + index_str = str(index) + index += 1 + else: + index_str = '' + print("%s (%s, index:%s): %s" % (op.name, op.type, index_str, [ out_shape.dims for out_shape in op.output_shape])) return False diff --git a/mace/python/tools/layers_validate.py b/mace/python/tools/layers_validate.py index 35238a8b..3316580b 100644 --- a/mace/python/tools/layers_validate.py +++ b/mace/python/tools/layers_validate.py @@ -31,6 +31,27 @@ def normalize_op_name(name): return name.replace('/', '_').replace(':', '_') +def handle_index(start, end, layers): + num_layers = end - start + 1 + if ':' in layers: + start_index, end_index = layers.split(':') + start_index = int(start_index) if start_index else 0 + end_index = int(end_index) if end_index else num_layers - 1 + else: + start_index = int(layers) + end_index = start_index + 1 + if start_index < 0: + start_index += num_layers + if end_index < 0: + end_index += num_layers + start_index += start + end_index += start + start_index = max(start, min(end - 1, start_index)) + end_index = max(start + 1, min(end, end_index)) + + return start_index, end_index + + def main(unused_args): mace_check(os.path.isfile(FLAGS.model_file), "Input graph file '" + FLAGS.model_file + "' does not exist!") @@ -68,6 +89,8 @@ def main(unused_args): # omit original output end_index -= 1 + index, end_index = handle_index(index, end_index, FLAGS.layers) + data_format = net_def.output_info[0].data_format output_configs = {"subgraphs": []} while index < end_index: @@ -82,7 +105,7 @@ def main(unused_args): net = copy.deepcopy(net_def) if hexagon_flag: # reuse dequantize op and it's min/max tensor's node_id - del net.op[index+1:end_index+1] + del net.op[index+1:-1] else: del net.op[index+1:] del net.output_info[:] @@ -163,6 +186,12 @@ def parse_args(): type=str, default="", help="Directory to save the output graph to.") + parser.add_argument( + "--layers", + type=str, + default="-1", + help="'start_layer:end_layer' or 'layer', similar to python slice." + " Use with --validate flag.") return parser.parse_known_args() diff --git a/tools/converter.py b/tools/converter.py index aedbd989..cda65de0 100644 --- a/tools/converter.py +++ b/tools/converter.py @@ -1194,10 +1194,11 @@ def parse_args(): help="whether to verify the results are consistent with " "the frameworks.") run.add_argument( - "--validate_all_layers", - action="store_true", - help="whether to verify the results of all layers are " - "consistent with the frameworks.") + "--layers", + type=str, + default="-1", + help="'start_layer:end_layer' or 'layer', similar to python slice." + " Use with --validate flag.") run.add_argument( "--caffe_env", type=str_to_caffe_env_type, diff --git a/tools/device.py b/tools/device.py index 679d09e8..217f179c 100644 --- a/tools/device.py +++ b/tools/device.py @@ -416,7 +416,7 @@ class DeviceWrapper: six.print_('Tuning done! \n') @staticmethod - def get_layers(model_dir, model_name): + def get_layers(model_dir, model_name, layers): sh_commands.bazel_build_common("//mace/python/tools:layers_validate") model_file = "%s/%s.pb" % (model_dir, model_name) @@ -428,6 +428,7 @@ class DeviceWrapper: "-u", "--model_file=%s" % model_file, "--output_dir=%s" % output_dir, + "--layers=%s" % layers, _fg=True) output_configs_path = output_dir + "outputs.yml" @@ -553,14 +554,15 @@ class DeviceWrapper: output_shapes = subgraphs[0][YAMLKeyword.check_shapes] output_configs = [] log_file = "" - if flags.validate_all_layers: + if flags.layers != "-1": mace_check(configs[YAMLKeyword.model_graph_format] == ModelFormat.file and configs[YAMLKeyword.model_data_format] == ModelFormat.file, "Device", - "'--validate_all_layers' only supports model format 'file'.") # noqa - output_configs = \ - self.get_layers(mace_model_dir, model_name) + "'--layers' only supports model format 'file'.") + output_configs = self.get_layers(mace_model_dir, + model_name, + flags.layers) log_dir = mace_model_dir + "/" + runtime if os.path.exists(log_dir): sh.rm('-rf', log_dir) @@ -611,7 +613,7 @@ class DeviceWrapper: layers_validate_file=output_config[ YAMLKeyword.model_file_path] ) - if flags.validate or flags.validate_all_layers: + if flags.validate: model_file_path, weight_file_path = get_model_files( model_config[YAMLKeyword.model_file_path], model_config[YAMLKeyword.model_sha256_checksum], -- GitLab