Support layers validate

.gitlab-ci.yml

@@ -147,6 +147,7 @@ python_tools_tests:
       python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1;
       python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,armhf --validate --model_graph_format=file --model_data_format=file || exit 1;
       python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1;
 
 model_tests:
   stage: model_tests
@@ -189,6 +190,7 @@ quantization_tests:
       python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1;
       python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
       python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate_all_layers --model_graph_format=file --model_data_format=file || exit 1;
       done
     - rm -rf mace-models
 

docs/development/how_to_debug.rst

@@ -34,6 +34,14 @@ It is usually used to measure classification accuracy. The higher the better.
 where :math:`X` is expected output (from training platform) whereas :math:`X'` is actual output (from MACE) .
 
 
+You can validate it by specifying `--validate` while running the model.
+
+    .. code:: sh
+
+        # Validate the correctness by comparing the results against the
+        # original model and framework
+        python tools/converter.py run --config=/path/to/your/model_deployment_file.yml --validate
+
 MACE automatically validate these metrics by running models with synthetic inputs.
 If you want to specify input data to use, you can add an option in yaml config under 'subgraphs', e.g.,
 
@@ -53,13 +61,22 @@ If you want to specify input data to use, you can add an option in yaml config u
 	          - MobilenetV1/Predictions/Reshape_1
 	        output_shapes:
 	          - 1,1001
+	        check_tensors:
+	          - MobilenetV1/Logits/Conv2d_1c_1x1/BiasAdd:0
+	        check_shapes:
+	          - 1,1,1,1001
 	        validation_inputs_data:
 	          - https://cnbj1.fds.api.xiaomi.com/mace/inputs/dog.npy
 
-
 If model's output is suspected to be incorrect, it might be useful to debug your model layer by layer by specifying an intermediate layer as output,
 or use binary search method until suspicious layer is found.
 
+You can also specify `--validate_all_layers` to validate all the layers of the model(excluding some layers changed by MACE, e.g., BatchToSpaceND),
+it only supports TensorFlow now. You can find validation results in `builds/your_model/model/runtime_in_yaml/log.csv`.
+
+For quantized model, if you want to check one layer, you can add `check_tensors` and `check_shapes` like in the yaml above. You can only specify
+MACE op's output.
+
 
 Debug memory usage
 --------------------------

mace/python/tools/BUILD

@@ -55,3 +55,15 @@ py_binary(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
 )
+
+py_binary(
+    name = "layers_validate",
+    srcs = [
+        "layers_validate.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":converter_lib",
+        ":model_saver_lib",
+    ],
+)

mace/python/tools/converter_tool/hexagon_converter.py

@@ -14,6 +14,7 @@
 
 import copy
 import numpy as np
+from enum import Enum
 from operator import mul
 
 from mace.proto import mace_pb2
@@ -28,22 +29,44 @@ from mace.python.tools.convert_util import mace_check
 from mace.python.tools import graph_util
 
 
+HexagonSupportedOps = [
+    'BatchToSpaceND_8',
+    'DepthwiseSupernode_8x8p32to8',
+    'DequantizeOUTPUT_8tof',
+    'QuantizedAdd_8p8to8',
+    'QuantizedAvgPool_8',
+    'QuantizedConcat_8',
+    'QuantizedMaxPool_8',
+    'QuantizedResizeBilinear_8',
+    'QuantizedSoftmax_8',
+    'QuantizeINPUT_f_to_8',
+    'SpaceToBatchND_8',
+    'Supernode_8x8p32to8',
+    'Nop',
+]
+
+HexagonOp = Enum('HexagonOp', [(op, op) for op in HexagonSupportedOps],
+                 type=str)
+
+
 class HexagonOps(object):
     def __init__(self):
         self.hexagon_ops = {
-            'Quantize': 'QuantizeINPUT_f_to_8',
-            'Dequantize': 'DequantizeOUTPUT_8tof',
-            'Concat': 'QuantizedConcat_8',
-            'Conv2D': 'Supernode_8x8p32to8',
-            'DepthwiseConv2d': 'DepthwiseSupernode_8x8p32to8',
-            'ResizeBilinear': 'QuantizedResizeBilinear_8',
-            'SpaceToBatchND': 'SpaceToBatchND_8',
-            'BatchToSpaceND': 'BatchToSpaceND_8',
-            'Softmax': 'QuantizedSoftmax_8',
-            'Eltwise': 'Eltwise',
-            'Pooling': 'Pooling',
-            'Identity': 'Nop',
-            'Squeeze': 'Nop',
+            MaceOp.BatchToSpaceND.name: HexagonOp.BatchToSpaceND_8.name,
+            MaceOp.Concat.name: HexagonOp.QuantizedConcat_8.name,
+            MaceOp.Conv2D.name: HexagonOp.Supernode_8x8p32to8.name,
+            MaceOp.DepthwiseConv2d.name:
+                HexagonOp.DepthwiseSupernode_8x8p32to8.name,
+            MaceOp.Dequantize.name: HexagonOp.DequantizeOUTPUT_8tof.name,
+            MaceOp.Eltwise.name: [HexagonOp.QuantizedAdd_8p8to8],
+            MaceOp.Identity.name: HexagonOp.Nop.name,
+            MaceOp.Quantize.name: HexagonOp.QuantizeINPUT_f_to_8.name,
+            MaceOp.Pooling.name: [HexagonOp.QuantizedAvgPool_8.name,
+                                  HexagonOp.QuantizedMaxPool_8.name],
+            MaceOp.ResizeBilinear.name:
+                HexagonOp.QuantizedResizeBilinear_8.name,
+            MaceOp.SpaceToBatchND.name: HexagonOp.SpaceToBatchND_8.name,
+            MaceOp.Softmax.name: HexagonOp.QuantizedSoftmax_8.name,
         }
 
     def has_op(self, tf_op):
@@ -116,7 +139,6 @@ class HexagonConverter(base_converter.ConverterInterface):
         for op in self._model.op:
             if not self._hexagon_ops.has_op(op.type):
                 raise Exception('Unsupported op: ', op)
-            print('Op: %s (%s)' % (op.name, op.type))
             for i in range(len(op.input)):
                 if ':' not in op.input[i]:
                     node_name = op.input[i]
@@ -250,14 +272,14 @@ class HexagonConverter(base_converter.ConverterInterface):
                     and ConverterUtil.get_arg(
                         op, MaceKeyword.mace_element_type_str).i
                     == EltwiseType.SUM.value):
-                op.type = 'QuantizedAdd_8p8to8'
+                op.type = HexagonOp.QuantizedAdd_8p8to8.name
             elif op.type == MaceOp.Pooling.name:
                 pooling_type_arg = ConverterUtil.get_arg(
                     op, MaceKeyword.mace_pooling_type_str)
                 if PoolingType(pooling_type_arg.i) == PoolingType.AVG:
-                    op.type = 'QuantizedAvgPool_8'
+                    op.type = HexagonOp.QuantizedAvgPool_8.name
                 else:
-                    op.type = 'QuantizedMaxPool_8'
+                    op.type = HexagonOp.QuantizedMaxPool_8.name
             else:
                 op.type = self._hexagon_ops.map_nn_op(op.type)
 
@@ -342,8 +364,10 @@ class HexagonConverter(base_converter.ConverterInterface):
             tensor_op, port = get_op_and_port_from_tensor(tensor.name)
             node_id_map[tensor_op] = tensor.node_id
 
+        print("Hexagon op:")
         for op in self._model.op:
             op.node_id = node_id_counter
+            print('Op: %s (%s, %d)' % (op.name, op.type, op.node_id))
             node_id_counter += 1
             node_id_map[op.name] = op.node_id
             for ipt in op.input:

mace/python/tools/layers_validate.py

+# Copyright 2018 Xiaomi, Inc.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import copy
+import os
+import sys
+import yaml
+
+from mace.proto import mace_pb2
+from mace.python.tools.converter_tool.base_converter import ConverterUtil
+from mace.python.tools.converter_tool.base_converter import MaceKeyword
+from mace.python.tools.converter_tool.base_converter import MaceOp
+from mace.python.tools.converter_tool.hexagon_converter import HexagonOp
+from mace.python.tools.convert_util import mace_check
+from mace.python.tools.model_saver import save_model_to_proto
+
+
+def normalize_op_name(name):
+    return name.replace('/', '_').replace(':', '_')
+
+
+def main(unused_args):
+    mace_check(os.path.isfile(FLAGS.model_file),
+               "Input graph file '" + FLAGS.model_file + "' does not exist!")
+    mace_check(os.path.isdir(FLAGS.output_dir),
+               "Output directory '" + FLAGS.output_dir + "' does not exist!")
+    net_def = mace_pb2.NetDef()
+    with open(FLAGS.model_file, "rb") as f:
+        net_def.ParseFromString(f.read())
+
+    quantize_flag = ConverterUtil.get_arg(
+        net_def, MaceKeyword.mace_quantize_flag_arg_str)
+    quantize_flag = False if quantize_flag is None else quantize_flag.i == 1
+    hexagon_flag = False
+    index = 0
+    end_index = len(net_def.op)
+    if quantize_flag:
+        while index < end_index:
+            # omit op quantize
+            if net_def.op[index].type == MaceOp.Quantize.name or \
+                    net_def.op[index].type == \
+                    HexagonOp.QuantizeINPUT_f_to_8.name:
+                index += 1
+            # omit op dequantize
+            elif net_def.op[end_index - 1].type == MaceOp.Dequantize.name or \
+                    net_def.op[end_index - 1].type == \
+                    HexagonOp.DequantizeOUTPUT_8tof.name:
+                end_index -= 1
+            else:
+                break
+        mace_check(0 < index < end_index < len(net_def.op),
+                   "Wrong number of op quantize(%d) or dequantize(%d)." %
+                   (index, len(net_def.op) - end_index))
+        if net_def.op[-1].type == HexagonOp.DequantizeOUTPUT_8tof.name:
+            hexagon_flag = True
+    # omit original output
+    end_index -= 1
+
+    data_format = net_def.output_info[0].data_format
+    output_configs = {"subgraphs": []}
+    while index < end_index:
+        # omit BatchToSpaceND and op before that due to changed graph
+        if net_def.op[index].type == MaceOp.BatchToSpaceND.name or \
+                net_def.op[index].type == HexagonOp.BatchToSpaceND_8.name or \
+                (index + 1 < end_index and
+                 (net_def.op[index + 1].type == MaceOp.BatchToSpaceND.name or
+                  net_def.op[index + 1].type == HexagonOp.BatchToSpaceND_8.name)):  # noqa
+            index += 1
+            continue
+        net = copy.deepcopy(net_def)
+        if hexagon_flag:
+            # reuse dequantize op and it's min/max tensor's node_id
+            del net.op[index+1:end_index+1]
+        else:
+            del net.op[index+1:]
+        del net.output_info[:]
+        op = net.op[index]
+        index += 1
+
+        output_tensors = []
+        output_shapes = []
+        op_name = op.name
+        if quantize_flag:
+            op.name = MaceKeyword.mace_output_node_name + '_' + op.name
+        if hexagon_flag:
+            mace_check(len(op.output) == 1,
+                       "Only supports number of outputs of Hexagon op be 1.")
+        for i in range(len(op.output)):
+            output_tensors.append(str(op.output[i]))
+            output_shapes.append(
+                ",".join([str(dim) for dim in op.output_shape[i].dims]))
+            # modify output info
+            output_info = net.output_info.add()
+            output_info.name = op.output[i]
+            output_info.data_format = data_format
+            output_info.dims.extend(op.output_shape[i].dims)
+            output_info.data_type = mace_pb2.DT_FLOAT
+            # modify output op
+            if quantize_flag:
+                output_name = op.output[i]
+                new_output_name = \
+                    MaceKeyword.mace_output_node_name + '_' + op.output[i]
+                op.output[i] = new_output_name
+                if not hexagon_flag:
+                    dequantize_op = net.op.add()
+                    dequantize_op.name = normalize_op_name(output_name)
+                    dequantize_op.type = MaceOp.Dequantize.name
+                    dequantize_op.input.append(new_output_name)
+                    dequantize_op.output.append(output_name)
+                    output_shape = dequantize_op.output_shape.add()
+                    output_shape.dims.extend(op.output_shape[i].dims)
+                    dequantize_op.output_type.append(mace_pb2.DT_FLOAT)
+                    ConverterUtil.add_data_type_arg(dequantize_op,
+                                                    mace_pb2.DT_UINT8)
+                else:
+                    dequantize_op = net.op[-1]
+                    dequantize_op.name = normalize_op_name(output_name)
+                    del dequantize_op.input[:]
+                    del dequantize_op.output[:]
+                    dequantize_op.input.append(new_output_name)
+                    dequantize_op.output.append(output_name)
+                    input_min = new_output_name[:-1] + '1'
+                    input_max = new_output_name[:-1] + '2'
+                    dequantize_op.input.extend([input_min, input_max])
+                    dequantize_op.node_input[0].node_id = op.node_id
+                    dequantize_op.node_input[1].node_id = op.node_id
+                    dequantize_op.node_input[2].node_id = op.node_id
+
+        model_path = save_model_to_proto(net, normalize_op_name(op_name),
+                                         FLAGS.output_dir)
+        output_config = {"model_file_path": str(model_path),
+                         "output_tensors": output_tensors,
+                         "output_shapes": output_shapes}
+        output_configs["subgraphs"].append(output_config)
+
+    output_configs_path = FLAGS.output_dir + "outputs.yml"
+    with open(output_configs_path, "w") as f:
+        yaml.dump(output_configs, f, default_flow_style=False)
+
+
+def parse_args():
+    """Parses command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_file",
+        type=str,
+        default="",
+        help="pb file to load.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="",
+        help="Directory to save the output graph to.")
+    return parser.parse_known_args()
+
+
+if __name__ == '__main__':
+    FLAGS, unparsed = parse_args()
+    main(unused_args=[sys.argv[0]] + unparsed)

mace/python/tools/model_saver.py

@@ -190,6 +190,8 @@ def save_model_to_proto(net_def, model_tag, output_dir):
     with open(proto_file_path + '_txt', "w") as f:
         f.write(str(net_def))
 
+    return proto_file_path
+
 
 def save_model_to_code(net_def, model_tag, device,
                        template_dir, output_dir, embed_model_data,

tools/common.py

@@ -14,6 +14,7 @@
 
 import enum
 import hashlib
+import inspect
 import re
 import os
 
@@ -34,6 +35,12 @@ class CMDColors:
     UNDERLINE = '\033[4m'
 
 
+def get_frame_info(level=2):
+    caller_frame = inspect.stack()[level]
+    info = inspect.getframeinfo(caller_frame[0])
+    return info.filename + ':' + str(info.lineno) + ': '
+
+
 class MaceLogger:
     @staticmethod
     def header(message):
@@ -45,22 +52,25 @@ class MaceLogger:
 
     @staticmethod
     def info(message):
-        six.print_(message)
+        six.print_(get_frame_info() + message)
 
     @staticmethod
     def warning(message):
-        six.print_(CMDColors.YELLOW + 'WARNING:' + message + CMDColors.ENDC)
+        six.print_(CMDColors.YELLOW + 'WARNING:' + get_frame_info() + message
+                   + CMDColors.ENDC)
 
     @staticmethod
-    def error(module, message):
-        six.print_(CMDColors.RED + 'ERROR: [' + module + '] '
+    def error(module, message, location_info=""):
+        if not location_info:
+            location_info = get_frame_info()
+        six.print_(CMDColors.RED + 'ERROR: [' + module + '] ' + location_info
                    + message + CMDColors.ENDC)
         exit(1)
 
 
 def mace_check(condition, module, message):
     if not condition:
-        MaceLogger.error(module, message)
+        MaceLogger.error(module, message, get_frame_info())
 
 
 ################################

tools/converter.py

@@ -1177,6 +1177,11 @@ def parse_args():
         action="store_true",
         help="whether to verify the results are consistent with "
              "the frameworks.")
+    run.add_argument(
+        "--validate_all_layers",
+        action="store_true",
+        help="whether to verify the results of all layers are "
+             "consistent with the frameworks.")
     run.add_argument(
         "--caffe_env",
         type=str_to_caffe_env_type,

tools/device.py

@@ -179,6 +179,7 @@ class DeviceWrapper:
                    address_sanitizer=False,
                    link_dynamic=False,
                    quantize_stat=False,
+                   layers_validate_file="",
                    ):
         six.print_("* Run '%s' with round=%s, restart_round=%s, tuning=%s, "
                    "out_of_range_check=%s, omp_num_threads=%s, "
@@ -189,7 +190,8 @@ class DeviceWrapper:
                     cpu_affinity_policy, gpu_perf_hint, gpu_priority_hint))
         mace_model_path = ""
         if model_graph_format == ModelFormat.file:
-            mace_model_path = "%s/%s.pb" % (mace_model_dir, model_tag)
+            mace_model_path = layers_validate_file if layers_validate_file \
+                else "%s/%s.pb" % (mace_model_dir, model_tag)
         if self.system == SystemType.host:
             libmace_dynamic_lib_path = \
                 os.path.dirname(libmace_dynamic_library_path)
@@ -413,6 +415,28 @@ class DeviceWrapper:
 
         six.print_('Tuning done! \n')
 
+    @staticmethod
+    def get_layers(model_dir, model_name):
+        sh_commands.bazel_build_common("//mace/python/tools:layers_validate")
+
+        model_file = "%s/%s.pb" % (model_dir, model_name)
+        output_dir = "%s/output_models/" % model_dir
+        if os.path.exists(output_dir):
+            sh.rm('-rf', output_dir)
+        os.makedirs(output_dir)
+        sh.python("bazel-bin/mace/python/tools/layers_validate",
+                  "-u",
+                  "--model_file=%s" % model_file,
+                  "--output_dir=%s" % output_dir,
+                  _fg=True)
+
+        output_configs_path = output_dir + "outputs.yml"
+        with open(output_configs_path) as f:
+            output_configs = yaml.load(f)
+        output_configs = output_configs[YAMLKeyword.subgraphs]
+
+        return output_configs
+
     def run_specify_abi(self, flags, configs, target_abi):
         if target_abi not in self.target_abis:
             six.print_('There is no device with soc: %s abi: %s' %
@@ -527,81 +551,111 @@ class DeviceWrapper:
                 else:
                     output_nodes = subgraphs[0][YAMLKeyword.check_tensors]
                     output_shapes = subgraphs[0][YAMLKeyword.check_shapes]
-                run_output = self.tuning_run(
-                    abi=target_abi,
-                    target_dir=build_tmp_binary_dir,
-                    target_name=target_name,
-                    vlog_level=flags.vlog_level,
-                    embed_model_data=embed_model_data,
-                    model_output_dir=model_output_dir,
-                    input_nodes=subgraphs[0][YAMLKeyword.input_tensors],
-                    output_nodes=output_nodes,
-                    input_shapes=subgraphs[0][YAMLKeyword.input_shapes],
-                    output_shapes=output_shapes,
-                    mace_model_dir=mace_model_dir,
-                    model_tag=model_name,
-                    device_type=device_type,
-                    running_round=flags.round,
-                    restart_round=flags.restart_round,
-                    limit_opencl_kernel_time=model_config[
-                        YAMLKeyword.limit_opencl_kernel_time],
-                    tuning=False,
-                    out_of_range_check=flags.gpu_out_of_range_check,
-                    model_graph_format=configs[YAMLKeyword.model_graph_format],
-                    omp_num_threads=flags.omp_num_threads,
-                    cpu_affinity_policy=flags.cpu_affinity_policy,
-                    gpu_perf_hint=flags.gpu_perf_hint,
-                    gpu_priority_hint=flags.gpu_priority_hint,
-                    runtime_failure_ratio=flags.runtime_failure_ratio,
-                    address_sanitizer=flags.address_sanitizer,
-                    opencl_binary_file=model_opencl_output_bin_path,
-                    opencl_parameter_file=model_opencl_parameter_path,
-                    libmace_dynamic_library_path=LIBMACE_DYNAMIC_PATH,
-                    link_dynamic=link_dynamic,
-                    quantize_stat=flags.quantize_stat,
-                    input_dir=flags.input_dir,
-                    output_dir=flags.output_dir,
-                )
-                if flags.validate:
-                    model_file_path, weight_file_path = get_model_files(
-                        model_config[YAMLKeyword.model_file_path],
-                        model_config[YAMLKeyword.model_sha256_checksum],
-                        BUILD_DOWNLOADS_DIR,
-                        model_config[YAMLKeyword.weight_file_path],
-                        model_config[YAMLKeyword.weight_sha256_checksum]
-                    )
-
-                    validate_type = device_type
-                    if model_config[YAMLKeyword.quantize] == 1:
-                        validate_type = device_type + '_QUANTIZE'
-                    sh_commands.validate_model(
+                output_configs = []
+                log_file = ""
+                if flags.validate_all_layers:
+                    mace_check(configs[YAMLKeyword.model_graph_format] ==
+                               ModelFormat.file and
+                               configs[YAMLKeyword.model_data_format] ==
+                               ModelFormat.file, "Device",
+                               "'--validate_all_layers' only supports model format 'file'.")  # noqa
+                    output_configs = \
+                        self.get_layers(mace_model_dir, model_name)
+                    log_dir = mace_model_dir + "/" + runtime
+                    if os.path.exists(log_dir):
+                        sh.rm('-rf', log_dir)
+                    os.makedirs(log_dir)
+                    log_file = log_dir + "/log.csv"
+                model_path = "%s/%s.pb" % (mace_model_dir, model_name)
+                output_config = {YAMLKeyword.model_file_path: model_path,
+                                 YAMLKeyword.output_tensors: output_nodes,
+                                 YAMLKeyword.output_shapes: output_shapes}
+                output_configs.append(output_config)
+                for output_config in output_configs:
+                    run_output = self.tuning_run(
                         abi=target_abi,
-                        device=self,
-                        model_file_path=model_file_path,
-                        weight_file_path=weight_file_path,
-                        platform=model_config[YAMLKeyword.platform],
-                        device_type=device_type,
+                        target_dir=build_tmp_binary_dir,
+                        target_name=target_name,
+                        vlog_level=flags.vlog_level,
+                        embed_model_data=embed_model_data,
+                        model_output_dir=model_output_dir,
                         input_nodes=subgraphs[0][YAMLKeyword.input_tensors],
-                        output_nodes=output_nodes,
+                        output_nodes=output_config[
+                            YAMLKeyword.output_tensors],
                         input_shapes=subgraphs[0][YAMLKeyword.input_shapes],
-                        output_shapes=output_shapes,
-                        model_output_dir=model_output_dir,
-                        input_data_types=subgraphs[0][
-                            YAMLKeyword.input_data_types],
-                        caffe_env=flags.caffe_env,
-                        validation_threshold=subgraphs[0][
-                            YAMLKeyword.validation_threshold][validate_type],
-                        backend=subgraphs[0][YAMLKeyword.backend]
-                    )
-                if flags.report and flags.round > 0:
-                    tuned = is_tuned and device_type == DeviceType.GPU
-                    self.report_run_statistics(
-                        target_abi=target_abi,
-                        model_name=model_name,
+                        output_shapes=output_config[YAMLKeyword.output_shapes],
+                        mace_model_dir=mace_model_dir,
+                        model_tag=model_name,
                         device_type=device_type,
-                        output_dir=flags.report_dir,
-                        tuned=tuned
+                        running_round=flags.round,
+                        restart_round=flags.restart_round,
+                        limit_opencl_kernel_time=model_config[
+                            YAMLKeyword.limit_opencl_kernel_time],
+                        tuning=False,
+                        out_of_range_check=flags.gpu_out_of_range_check,
+                        model_graph_format=configs[
+                            YAMLKeyword.model_graph_format],
+                        omp_num_threads=flags.omp_num_threads,
+                        cpu_affinity_policy=flags.cpu_affinity_policy,
+                        gpu_perf_hint=flags.gpu_perf_hint,
+                        gpu_priority_hint=flags.gpu_priority_hint,
+                        runtime_failure_ratio=flags.runtime_failure_ratio,
+                        address_sanitizer=flags.address_sanitizer,
+                        opencl_binary_file=model_opencl_output_bin_path,
+                        opencl_parameter_file=model_opencl_parameter_path,
+                        libmace_dynamic_library_path=LIBMACE_DYNAMIC_PATH,
+                        link_dynamic=link_dynamic,
+                        quantize_stat=flags.quantize_stat,
+                        input_dir=flags.input_dir,
+                        output_dir=flags.output_dir,
+                        layers_validate_file=output_config[
+                            YAMLKeyword.model_file_path]
                     )
+                    if flags.validate or flags.validate_all_layers:
+                        model_file_path, weight_file_path = get_model_files(
+                            model_config[YAMLKeyword.model_file_path],
+                            model_config[YAMLKeyword.model_sha256_checksum],
+                            BUILD_DOWNLOADS_DIR,
+                            model_config[YAMLKeyword.weight_file_path],
+                            model_config[YAMLKeyword.weight_sha256_checksum]
+                        )
+                        validate_type = device_type
+                        if model_config[YAMLKeyword.quantize] == 1:
+                            validate_type = device_type + '_QUANTIZE'
+                        sh_commands.validate_model(
+                            abi=target_abi,
+                            device=self,
+                            model_file_path=model_file_path,
+                            weight_file_path=weight_file_path,
+                            platform=model_config[YAMLKeyword.platform],
+                            device_type=device_type,
+                            input_nodes=subgraphs[0][
+                                YAMLKeyword.input_tensors],
+                            output_nodes=output_config[
+                                YAMLKeyword.output_tensors],
+                            input_shapes=subgraphs[0][
+                                YAMLKeyword.input_shapes],
+                            output_shapes=output_config[
+                                YAMLKeyword.output_shapes],
+                            model_output_dir=model_output_dir,
+                            input_data_types=subgraphs[0][
+                                YAMLKeyword.input_data_types],
+                            caffe_env=flags.caffe_env,
+                            validation_threshold=subgraphs[0][
+                                YAMLKeyword.validation_threshold][
+                                validate_type],
+                            backend=subgraphs[0][YAMLKeyword.backend],
+                            log_file=log_file,
+                        )
+                    if flags.report and flags.round > 0:
+                        tuned = is_tuned and device_type == DeviceType.GPU
+                        self.report_run_statistics(
+                            target_abi=target_abi,
+                            model_name=model_name,
+                            device_type=device_type,
+                            output_dir=flags.report_dir,
+                            tuned=tuned
+                        )
         if model_output_dirs:
             opencl_output_bin_path = get_opencl_binary_output_path(
                 library_name, target_abi, self

tools/sh_commands.py

@@ -622,7 +622,9 @@ def validate_model(abi,
                    input_file_name="model_input",
                    output_file_name="model_out",
                    validation_threshold=0.9,
-                   backend="tensorflow"):
+                   backend="tensorflow",
+                   log_file="",
+                   ):
     six.print_("* Validate with %s" % platform)
     if abi != "host":
         for output_name in output_nodes:
@@ -639,14 +641,16 @@ def validate_model(abi,
                  "%s/%s" % (model_output_dir, output_file_name), device_type,
                  ":".join(input_shapes), ":".join(output_shapes),
                  ",".join(input_nodes), ",".join(output_nodes),
-                 validation_threshold, ",".join(input_data_types), backend)
+                 validation_threshold, ",".join(input_data_types), backend,
+                 log_file)
     elif platform == "onnx":
         validate(platform, model_file_path, "",
                  "%s/%s" % (model_output_dir, input_file_name),
                  "%s/%s" % (model_output_dir, output_file_name), device_type,
                  ":".join(input_shapes), ":".join(output_shapes),
                  ",".join(input_nodes), ",".join(output_nodes),
-                 validation_threshold, ",".join(input_data_types), backend)
+                 validation_threshold, ",".join(input_data_types), backend,
+                 log_file)
     elif platform == "caffe":
         image_name = "mace-caffe:latest"
         container_name = "mace_caffe_validator"
@@ -662,7 +666,8 @@ def validate_model(abi,
                      device_type,
                      ":".join(input_shapes), ":".join(output_shapes),
                      ",".join(input_nodes), ",".join(output_nodes),
-                     validation_threshold, ",".join(input_data_types), backend)
+                     validation_threshold, ",".join(input_data_types), backend,
+                     log_file)
         elif caffe_env == common.CaffeEnvType.DOCKER:
             docker_image_id = sh.docker("images", "-q", image_name)
             if not docker_image_id:

tools/validate.py

@@ -79,7 +79,7 @@ def calculate_pixel_accuracy(out_value, mace_out_value):
 
 
 def compare_output(platform, device_type, output_name, mace_out_value,
-                   out_value, validation_threshold):
+                   out_value, validation_threshold, log_file):
     if mace_out_value.size != 0:
         pixel_accuracy = calculate_pixel_accuracy(out_value, mace_out_value)
         out_value = out_value.reshape(-1)
@@ -91,7 +91,18 @@ def compare_output(platform, device_type, output_name, mace_out_value,
             output_name + ' MACE VS ' + platform.upper()
             + ' similarity: ' + str(similarity) + ' , sqnr: ' + str(sqnr)
             + ' , pixel_accuracy: ' + str(pixel_accuracy))
-        if similarity > validation_threshold:
+        if log_file:
+            if not os.path.exists(log_file):
+                with open(log_file, 'w') as f:
+                    f.write('output_name,similarity,sqnr,pixel_accuracy\n')
+            summary = '{output_name},{similarity},{sqnr},{pixel_accuracy}\n'\
+                .format(output_name=output_name,
+                        similarity=similarity,
+                        sqnr=sqnr,
+                        pixel_accuracy=pixel_accuracy)
+            with open(log_file, "a") as f:
+                f.write(summary)
+        elif similarity > validation_threshold:
             common.MaceLogger.summary(
                 common.StringFormatter.block("Similarity Test Passed"))
         else:
@@ -112,7 +123,8 @@ def normalize_tf_tensor_name(name):
 
 def validate_tf_model(platform, device_type, model_file, input_file,
                       mace_out_file, input_names, input_shapes,
-                      output_names, validation_threshold, input_data_types):
+                      output_names, validation_threshold, input_data_types,
+                      log_file):
     import tensorflow as tf
     if not os.path.isfile(model_file):
         common.MaceLogger.error(
@@ -151,12 +163,13 @@ def validate_tf_model(platform, device_type, model_file, input_file,
                     mace_out_value = load_data(output_file_name)
                     compare_output(platform, device_type, output_names[i],
                                    mace_out_value, output_values[i],
-                                   validation_threshold)
+                                   validation_threshold, log_file)
 
 
 def validate_caffe_model(platform, device_type, model_file, input_file,
                          mace_out_file, weight_file, input_names, input_shapes,
-                         output_names, output_shapes, validation_threshold):
+                         output_names, output_shapes, validation_threshold,
+                         log_file):
     os.environ['GLOG_minloglevel'] = '1'  # suprress Caffe verbose prints
     import caffe
     if not os.path.isfile(model_file):
@@ -201,13 +214,13 @@ def validate_caffe_model(platform, device_type, model_file, input_file,
             mace_out_file, output_names[i])
         mace_out_value = load_data(output_file_name)
         compare_output(platform, device_type, output_names[i], mace_out_value,
-                       value, validation_threshold)
+                       value, validation_threshold, log_file)
 
 
 def validate_onnx_model(platform, device_type, model_file, input_file,
                         mace_out_file, input_names, input_shapes,
                         output_names, output_shapes, validation_threshold,
-                        input_data_types, backend):
+                        input_data_types, backend, log_file):
     import onnx
     if backend == "tensorflow":
         from onnx_tf.backend import prepare
@@ -257,12 +270,12 @@ def validate_onnx_model(platform, device_type, model_file, input_file,
         mace_out_value = load_data(output_file_name)
         compare_output(platform, device_type, output_names[i],
                        mace_out_value, value,
-                       validation_threshold)
+                       validation_threshold, log_file)
 
 
 def validate(platform, model_file, weight_file, input_file, mace_out_file,
              device_type, input_shape, output_shape, input_node, output_node,
-             validation_threshold, input_data_type, backend):
+             validation_threshold, input_data_type, backend, log_file):
     input_names = [name for name in input_node.split(',')]
     input_shape_strs = [shape for shape in input_shape.split(':')]
     input_shapes = [[int(x) for x in shape.split(',')]
@@ -278,7 +291,8 @@ def validate(platform, model_file, weight_file, input_file, mace_out_file,
     if platform == 'tensorflow':
         validate_tf_model(platform, device_type, model_file, input_file,
                           mace_out_file, input_names, input_shapes,
-                          output_names, validation_threshold, input_data_types)
+                          output_names, validation_threshold, input_data_types,
+                          log_file)
     elif platform == 'caffe':
         output_shape_strs = [shape for shape in output_shape.split(':')]
         output_shapes = [[int(x) for x in shape.split(',')]
@@ -286,7 +300,7 @@ def validate(platform, model_file, weight_file, input_file, mace_out_file,
         validate_caffe_model(platform, device_type, model_file, input_file,
                              mace_out_file, weight_file, input_names,
                              input_shapes, output_names, output_shapes,
-                             validation_threshold)
+                             validation_threshold, log_file)
     elif platform == 'onnx':
         output_shape_strs = [shape for shape in output_shape.split(':')]
         output_shapes = [[int(x) for x in shape.split(',')]
@@ -295,7 +309,7 @@ def validate(platform, model_file, weight_file, input_file, mace_out_file,
                             mace_out_file, input_names, input_shapes,
                             output_names, output_shapes,
                             validation_threshold,
-                            input_data_types, backend)
+                            input_data_types, backend, log_file)
 
 
 def parse_args():