diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7730f99dbe13a7f7eb69de246ba24fe62f8feba..76907b14272e861b04ac4f156a2ed83e76df264c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,7 +147,7 @@ python_tools_tests: python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,armhf --validate --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1; + python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; model_tests: stage: model_tests @@ -190,7 +190,7 @@ quantization_tests: python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1; - python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1; + python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1; done - rm -rf mace-models diff --git a/docs/development/how_to_debug.rst b/docs/development/how_to_debug.rst index 0c04cebc94b534609fbd12969122061ee46f9158..ea4688585562d812d06cdfa4a27935f3252df66a 100644 --- a/docs/development/how_to_debug.rst +++ b/docs/development/how_to_debug.rst @@ -71,7 +71,7 @@ If you want to specify input data to use, you can add an option in yaml config u If model's output is suspected to be incorrect, it might be useful to debug your model layer by layer by specifying an intermediate layer as output, or use binary search method until suspicious layer is found. -You can also specify `--validate_all_layers` to validate all the layers of the model(excluding some layers changed by MACE, e.g., BatchToSpaceND), +You can also specify `--layers` after `--validate` to validate all or some of the layers of the model(excluding some layers changed by MACE, e.g., BatchToSpaceND), it only supports TensorFlow now. You can find validation results in `builds/your_model/model/runtime_in_yaml/log.csv`. For quantized model, if you want to check one layer, you can add `check_tensors` and `check_shapes` like in the yaml above. You can only specify diff --git a/mace/python/tools/converter_tool/hexagon_converter.py b/mace/python/tools/converter_tool/hexagon_converter.py index f37ab7ba15027cc021d55d381352c6ec436ddfb3..42154b9982de843572e00ee558f525a5759843cb 100644 --- a/mace/python/tools/converter_tool/hexagon_converter.py +++ b/mace/python/tools/converter_tool/hexagon_converter.py @@ -365,9 +365,17 @@ class HexagonConverter(base_converter.ConverterInterface): node_id_map[tensor_op] = tensor.node_id print("Hexagon op:") + index = 0 for op in self._model.op: op.node_id = node_id_counter - print('Op: %s (%s, %d)' % (op.name, op.type, op.node_id)) + if op.type not in [HexagonOp.QuantizeINPUT_f_to_8, + HexagonOp.DequantizeOUTPUT_8tof.name]: + index_str = str(index) + index += 1 + else: + index_str = '' + print('Op: %s (%s, node_id:%d, index:%s)' % + (op.name, op.type, op.node_id, index_str)) node_id_counter += 1 node_id_map[op.name] = op.node_id for ipt in op.input: diff --git a/mace/python/tools/converter_tool/transformer.py b/mace/python/tools/converter_tool/transformer.py index 9ea6b6d83628cfef70c58db0b401472d29565776..07ae7d8a9c6fd41ac50813fffff6e7479e5d4800 100644 --- a/mace/python/tools/converter_tool/transformer.py +++ b/mace/python/tools/converter_tool/transformer.py @@ -1370,8 +1370,14 @@ class Transformer(base_converter.ConverterInterface): net.op.extend(sorted_nodes) print("Final ops:") + index = 0 for op in net.op: - print("%s (%s): %s" % (op.name, op.type, [ + if op.type not in [MaceOp.Quantize.name, MaceOp.Dequantize.name]: + index_str = str(index) + index += 1 + else: + index_str = '' + print("%s (%s, index:%s): %s" % (op.name, op.type, index_str, [ out_shape.dims for out_shape in op.output_shape])) return False diff --git a/mace/python/tools/layers_validate.py b/mace/python/tools/layers_validate.py index 35238a8bcd65bd998c695fce8d2ab3277a6ba30b..3316580b2ba7a376e561d8a170af3d1913227430 100644 --- a/mace/python/tools/layers_validate.py +++ b/mace/python/tools/layers_validate.py @@ -31,6 +31,27 @@ def normalize_op_name(name): return name.replace('/', '_').replace(':', '_') +def handle_index(start, end, layers): + num_layers = end - start + 1 + if ':' in layers: + start_index, end_index = layers.split(':') + start_index = int(start_index) if start_index else 0 + end_index = int(end_index) if end_index else num_layers - 1 + else: + start_index = int(layers) + end_index = start_index + 1 + if start_index < 0: + start_index += num_layers + if end_index < 0: + end_index += num_layers + start_index += start + end_index += start + start_index = max(start, min(end - 1, start_index)) + end_index = max(start + 1, min(end, end_index)) + + return start_index, end_index + + def main(unused_args): mace_check(os.path.isfile(FLAGS.model_file), "Input graph file '" + FLAGS.model_file + "' does not exist!") @@ -68,6 +89,8 @@ def main(unused_args): # omit original output end_index -= 1 + index, end_index = handle_index(index, end_index, FLAGS.layers) + data_format = net_def.output_info[0].data_format output_configs = {"subgraphs": []} while index < end_index: @@ -82,7 +105,7 @@ def main(unused_args): net = copy.deepcopy(net_def) if hexagon_flag: # reuse dequantize op and it's min/max tensor's node_id - del net.op[index+1:end_index+1] + del net.op[index+1:-1] else: del net.op[index+1:] del net.output_info[:] @@ -163,6 +186,12 @@ def parse_args(): type=str, default="", help="Directory to save the output graph to.") + parser.add_argument( + "--layers", + type=str, + default="-1", + help="'start_layer:end_layer' or 'layer', similar to python slice." + " Use with --validate flag.") return parser.parse_known_args() diff --git a/tools/converter.py b/tools/converter.py index aedbd9894ab0358ce90371fa0017ab03e039a8ac..cda65de0b4870d840921f8525aa2dc856045f30c 100644 --- a/tools/converter.py +++ b/tools/converter.py @@ -1194,10 +1194,11 @@ def parse_args(): help="whether to verify the results are consistent with " "the frameworks.") run.add_argument( - "--validate_all_layers", - action="store_true", - help="whether to verify the results of all layers are " - "consistent with the frameworks.") + "--layers", + type=str, + default="-1", + help="'start_layer:end_layer' or 'layer', similar to python slice." + " Use with --validate flag.") run.add_argument( "--caffe_env", type=str_to_caffe_env_type, diff --git a/tools/device.py b/tools/device.py index 679d09e8fed23c4976052ef9af8aa66504178c40..217f179c683615aa64c30337e2ba9a825a6c18f1 100644 --- a/tools/device.py +++ b/tools/device.py @@ -416,7 +416,7 @@ class DeviceWrapper: six.print_('Tuning done! \n') @staticmethod - def get_layers(model_dir, model_name): + def get_layers(model_dir, model_name, layers): sh_commands.bazel_build_common("//mace/python/tools:layers_validate") model_file = "%s/%s.pb" % (model_dir, model_name) @@ -428,6 +428,7 @@ class DeviceWrapper: "-u", "--model_file=%s" % model_file, "--output_dir=%s" % output_dir, + "--layers=%s" % layers, _fg=True) output_configs_path = output_dir + "outputs.yml" @@ -553,14 +554,15 @@ class DeviceWrapper: output_shapes = subgraphs[0][YAMLKeyword.check_shapes] output_configs = [] log_file = "" - if flags.validate_all_layers: + if flags.layers != "-1": mace_check(configs[YAMLKeyword.model_graph_format] == ModelFormat.file and configs[YAMLKeyword.model_data_format] == ModelFormat.file, "Device", - "'--validate_all_layers' only supports model format 'file'.") # noqa - output_configs = \ - self.get_layers(mace_model_dir, model_name) + "'--layers' only supports model format 'file'.") + output_configs = self.get_layers(mace_model_dir, + model_name, + flags.layers) log_dir = mace_model_dir + "/" + runtime if os.path.exists(log_dir): sh.rm('-rf', log_dir) @@ -611,7 +613,7 @@ class DeviceWrapper: layers_validate_file=output_config[ YAMLKeyword.model_file_path] ) - if flags.validate or flags.validate_all_layers: + if flags.validate: model_file_path, weight_file_path = get_model_files( model_config[YAMLKeyword.model_file_path], model_config[YAMLKeyword.model_sha256_checksum],