Merge branch 'cmake-tools' into 'master'

Support cmake based tools for internal use first

See merge request !1144

CMakeLists.txt

@@ -44,7 +44,11 @@ endif(MACE_ENABLE_OPT_SIZE)
 # flags apply only to mace code (third_party excluded)
 # -Wno-error=unused-command-line-argument: official Android toolchain contains
 # unsupported argument and will break ccache preprocessor
-set(MACE_CODE_CC_FLAGS "${MACE_CODE_CC_FLAGS} -Wall -Werror -Wno-error=unused-command-line-argument")
+if(ANDROID)
+  set(MACE_CODE_CC_FLAGS "${MACE_CODE_CC_FLAGS} -Wall -Werror -Wno-error=unused-command-line-argument")
+else(ANDROID)
+  set(MACE_CODE_CC_FLAGS "${MACE_CODE_CC_FLAGS} -Wall -Werror")
+endif(ANDROID)
 set(MACE_CODE_CC_FLAGS "${MACE_CODE_CC_FLAGS} -std=c++11 -D_GLIBCXX_USE_C99_MATH_TR1")
 
 if(IOS)

tools/experimental/config/model/model-template.yaml

+models:
+  mobilenet_v2:
+    platform: tensorflow
+    model_file_path: https://cnbj1.fds.api.xiaomi.com/mace/miai-models/mobilenet-v2/mobilenet-v2-1.0.pb
+    model_sha256_checksum: 369f9a5f38f3c15b4311c1c84c032ce868da9f371b5f78c13d3ea3c537389bb4
+    subgraphs:
+      - input_tensors:
+          - input
+        input_shapes:
+          - 1,224,224,3
+        output_tensors:
+          - MobilenetV2/Predictions/Reshape_1
+        output_shapes:
+          - 1,1001
+        validation_inputs_data:
+          - https://cnbj1.fds.api.xiaomi.com/mace/inputs/dog.npy
+

tools/experimental/run.py

+# Copyright 2019 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import filelock
+import random
+import argparse
+import os
+
+from utils import device
+from utils import target
+from utils import config_parser
+
+
+def device_lock(device_id, timeout=7200):
+    return filelock.FileLock("/tmp/device-lock-%s" % device_id,
+                             timeout=timeout)
+
+
+def is_device_locked(device_id):
+    try:
+        with device_lock(device_id, timeout=0.000001):
+            return False
+    except filelock.Timeout:
+        return True
+
+
+def run_target(target_abi, install_dir, target_obj, device_ids="all"):
+    if not install_dir:
+        install_dir = default_install_dir(target_abi)
+
+    device_class = device.device_class(target_abi)
+    devices = device_class.list_devices()
+
+    if device_ids == "all":
+        run_devices = devices
+    elif device_ids == "random":
+        unlocked_devices = [dev for dev in devices if
+                            not is_device_locked(dev)]
+        if unlocked_devices:
+            run_devices = [random.choice(unlocked_devices)]
+        else:
+            run_devices = [random.choice(devices)]
+    else:
+        device_id_list = [dev.strip() for dev in device_ids.split(",")]
+        run_devices = [dev for dev in device_id_list if dev in devices]
+
+    print("Run on devices: %s" % run_devices)
+
+    for device_id in run_devices:
+        # initiate device
+        dev = device.crete_device(target_abi, device_id)
+
+        # reinstall target
+        print("Install target from %s to %s" % (target_obj.path, install_dir))
+        device_target = dev.install(target_obj, install_dir)
+        print(device_target)
+
+        # run on device
+        print("Runing ...")
+        with device_lock(device_id):
+            dev.run(device_target)
+
+
+def default_install_dir(target_abi):
+    install_dir = "/tmp/mace_run"
+    if target_abi == "armeabi-v7a" or target_abi == "arm64-v8a":
+        install_dir = "/data/local/tmp/mace_run"
+
+    return install_dir
+
+
+"""
+Internal tool for mace_cc_benchmark, mace_cc_test, mace_run:
+
+python tools/experimental/run.py \
+    --target_abi=armeabi-v7a --target_socs=all --target_name=mace_cc_test \
+    --args="--gtest_filter=EnvTest.*"  --envs="MACE_CPP_MIN_VLOG_LEVEL=5"
+"""
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--target_abi",
+        type=str,
+        default="armeabi-v7a",
+        help="Target ABI: host, armeabi-v7a, arm64-v8a,"
+             " arm-linux-gnueabihf, aarch64-linux-gnu"
+    )
+    parser.add_argument(
+        "--target_socs",
+        type=str,
+        default="",
+        help="serialno for adb connection,"
+             " username@ip for arm linux,"
+             " host for host"
+             " | all | random"
+    )
+    parser.add_argument(
+        "--device_conf",
+        type=str,
+        default="",
+        help="device yaml config path"
+    )
+    parser.add_argument(
+        "--target_name",
+        type=str,
+        default="mace_cc_benchmark",
+        help="Target name: mace_cc_benchmark, mace_cc_test, mace_run"
+    )
+    parser.add_argument(
+        "--build_dir",
+        type=str,
+        default="cmake-build-debug-tools",
+        help="cmake build dir"
+    )
+    parser.add_argument(
+        "--build",
+        action="store_true",
+        help="if build before run"
+    )
+
+    parser.add_argument("--args", type=str, default="",
+                        help="Command args: --gtest_filter=*, --filter=*")
+    parser.add_argument("--envs", type=str, default="",
+                        help="Environment vars: "
+                             " MACE_CPP_MIN_VLOG_LEVEL=2,"
+                             " MACE_OUT_OF_RANGE_CHECK=1, "
+                             " MACE_OPENCL_PROFILING=1,"
+                             " MACE_INTERNAL_STORAGE_PATH=/path/to,"
+                             " LD_PRELOAD=/path/to")
+
+    flgs, _ = parser.parse_known_args()
+    return flgs
+
+
+if __name__ == "__main__":
+    flags = parse_args()
+    if flags.device_conf:
+        device_conf = config_parser.parse_device_info(flags.device_conf)
+        device.ArmLinuxDevice.set_devices(device_conf)
+
+    target_abi = flags.target_abi.strip()
+    target_name = flags.target_name.strip()
+    opts = flags.args.split(" ")
+    envs = flags.envs.split(" ")
+
+    # build
+    build_dir = flags.build_dir + "/" + target_abi
+    if flags.build:
+        cmake_shell = os.path.abspath(
+            os.path.dirname(
+                __file__)) + "/config/build/cmake-build-%s.sh" % target_abi
+        os.environ["BUILD_DIR"] = build_dir
+        device.execute(cmake_shell)
+
+    # run
+    target = target.Target(build_dir + "/install/bin/" + target_name,
+                           opts=opts, envs=envs)
+    run_target(target_abi, None, target, device_ids=flags.target_socs)

tools/experimental/utils/config_parser.py

+# Copyright 2019 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import os
+import yaml
+
+
+def sanitize_load(s):
+    # do not let yaml parse ON/OFF to boolean
+    for w in ["ON", "OFF", "on", "off"]:
+        s = re.sub(r":\s+" + w, r": '" + w + "'", s)
+
+    # sub ${} to env value
+    s = re.sub(r"\${(\w+)}", lambda x: os.environ[x.group(1)], s)
+    return yaml.load(s)
+
+
+def parse(path):
+    with open(path) as f:
+        config = sanitize_load(f.read())
+
+    return config
+
+
+def parse_device_info(path):
+    conf = parse(path)
+    return conf["devices"]

tools/experimental/utils/device.py

+# Copyright 2019 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import subprocess
+
+
+MACE_TOOL_QUIET_ENV = "MACE_TOOL_QUIET"
+
+
+def execute(cmd):
+    print("CMD> %s" % cmd)
+    p = subprocess.Popen([cmd],
+                         shell=True,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT,
+                         stdin=subprocess.PIPE,
+                         universal_newlines=True)
+    returncode = p.poll()
+    buf = []
+    while returncode is None:
+        line = p.stdout.readline()
+        returncode = p.poll()
+        line = line.strip()
+        if MACE_TOOL_QUIET_ENV not in os.environ:
+            print(line)
+        buf.append(line)
+
+    p.wait()
+
+    if returncode != 0:
+        raise Exception("errorcode: %s" % returncode)
+
+    return "\n".join(buf)
+
+
+class Device(object):
+    def __init__(self, device_id):
+        self._device_id = device_id
+
+    def install(self, target, install_dir):
+        pass
+
+    def run(self, target):
+        pass
+
+    def pull(self, target, out_dir):
+        pass
+
+
+class HostDevice(Device):
+    def __init__(self, device_id):
+        super(HostDevice, self).__init__(device_id)
+
+    @staticmethod
+    def list_devices():
+        return ["host"]
+
+    def install(self, target, install_dir):
+        install_dir = os.path.abspath(install_dir)
+
+        if install_dir.strip() and install_dir != os.path.dirname(target.path):
+            execute("mkdir -p %s" % install_dir)
+            execute("cp %s %s" % (target.path, install_dir))
+            for lib in target.libs:
+                execute("cp %s %s" % (lib, install_dir))
+
+            target.path = "%s/%s" % (install_dir,
+                                     os.path.basename(target.path))
+            target.libs = ["%s/%s" % (install_dir, os.path.basename(lib))
+                           for lib in target.libs]
+
+        target.envs.append("LD_LIBRARY_PATH=%s" % install_dir)
+
+        return target
+
+    def run(self, target):
+        execute(str(target))
+
+    def pull(self, target, out_dir):
+        out_dir = os.path.abspath(out_dir)
+
+        if out_dir.strip() and out_dir != os.path.dirname(target.path):
+            execute("cp -r %s %s" % (target.path, out_dir))
+
+
+class AndroidDevice(Device):
+    def __init__(self, device_id):
+        super(AndroidDevice, self).__init__(device_id)
+
+    @staticmethod
+    def list_devices():
+        out = execute("adb devices")
+        serialno_list = out.strip().split('\n')[1:]
+        serialno_list = [tuple(pair.split('\t')) for pair in serialno_list]
+        devices = []
+        for serialno in serialno_list:
+            if not serialno[1].startswith("no permissions"):
+                devices.append(serialno[0])
+
+        return devices
+
+    def install(self, target, install_dir):
+        install_dir = os.path.abspath(install_dir)
+        sn = self._device_id
+
+        execute("adb -s %s shell mkdir -p %s" % (sn, install_dir))
+        execute("adb -s %s push %s %s" % (sn, target.path, install_dir))
+        for lib in target.libs:
+            execute("adb -s %s push %s %s" % (sn, lib, install_dir))
+
+        target.path = "%s/%s" % (install_dir, os.path.basename(target.path))
+        target.libs = ["%s/%s" % (install_dir, os.path.basename(lib))
+                       for lib in target.libs]
+        target.envs.append("LD_LIBRARY_PATH=%s" % install_dir)
+
+        return target
+
+    def run(self, target):
+        out = execute("adb -s %s shell %s" % (self._device_id, target))
+        # May have false positive using the following error word
+        for line in out.split("\n")[:-10]:
+            if ("Aborted" in line
+                    or "FAILED" in line or "Segmentation fault" in line):
+                raise Exception(line)
+
+    def pull(self, target, out_dir):
+        sn = self._device_id
+        execute("adb -s %s pull %s %s" % (sn, target.path, out_dir))
+
+
+class ArmLinuxDevice(Device):
+    devices = {}
+
+    def __init__(self, device_id):
+        super(ArmLinuxDevice, self).__init__(device_id)
+
+    @staticmethod
+    def list_devices():
+        device_ids = []
+        for dev_name, dev_info in ArmLinuxDevice.devices:
+            address = dev_info["address"]
+            username = dev_info["username"]
+            device_ids.append("%s@%s" % (username, address))
+
+    @staticmethod
+    def set_devices(devices):
+        ArmLinuxDevice.devices = devices
+
+    def install(self, target, install_dir):
+        install_dir = os.path.abspath(install_dir)
+        ip = self._device_id
+
+        execute("ssh %s mkdir -p %s" % install_dir)
+        execute("scp %s %s:%s" % (target.path, ip, install_dir))
+        for lib in target.libs:
+            execute("scp %s:%s" % (lib, install_dir))
+
+        target.path = "%s/%s" % (install_dir, os.path.basename(target.path))
+        target.libs = ["%s/%s" % (install_dir, os.path.basename(lib))
+                       for lib in target.libs]
+        target.envs.append("LD_LIBRARY_PATH=%s" % install_dir)
+
+        return target
+
+    def run(self, target):
+        execute("ssh %s shell %s" % (self._device_id, target))
+
+    def pull(self, target, out_dir):
+        sn = self._device_id
+        execute("scp %s:%s %s" % (sn, target.path, out_dir))
+
+
+def device_class(target_abi):
+    device_dispatch = {
+        "host": "HostDevice",
+        "armeabi-v7a": "AndroidDevice",
+        "arm64-v8a": "AndroidDevice",
+        "arm-linux-gnueabihf": "ArmLinuxDevice",
+        "aarch64-linux-gnu": "ArmLinuxDevice"
+    }
+
+    if target_abi not in device_dispatch:
+        raise ValueError(
+            "target_abi should be one of %s" % device_dispatch.keys())
+
+    return globals()[device_dispatch[target_abi]]
+
+
+def crete_device(target_abi, device_id=None):
+    return device_class(target_abi)(device_id)

tools/experimental/utils/target.py

+# Copyright 2019 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+class Target(object):
+    def __init__(self, path, libs=None, opts=None, envs=None):
+        if libs is None:
+            libs = []
+        if opts is None:
+            opts = []
+        if envs is None:
+            envs = []
+
+        self.path = path
+        self.libs = libs
+        self.opts = opts
+        self.envs = envs
+
+    def __str__(self):
+        # env, path, run_opts
+        cmd_list = []
+        cmd_list += self.envs
+        cmd_list += [self.path]
+        cmd_list += self.opts
+        print(cmd_list)
+        return " ".join(cmd_list)

tools/experimental/utils/util.py

+# Copyright 2019 The MACE Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function