Bug: Fix random device chosen bug of CI.

1. Fix random device chosen bug. 2. Add random device chosen strategy for running model.

Bug: Fix random device chosen bug of CI.
1. Fix random device chosen bug. 2. Add random device chosen strategy for running model.
3d283d2a · liuqi · ed1f237a · 3d283d2a · 3d283d2a · 3d283d2a
8 changed file
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,7 @@ stages:
  - platform_compatible_tests
  - build_libraries
  - ndk_versions_compatible_tests
+  - ops_test_disable_neon
  - ops_test
  - api_test
  - python_tools_tests
@@ -44,51 +45,6 @@ docs:
    paths:
      - docs/_build

-ops_test:
-  stage: ops_test
-  script:
-    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
-    - >
-      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
-        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
-        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
-      fi
-    - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64,armhf --target_socs=$TARGET_SOCS
-    - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64,armhf --target_socs=$TARGET_SOCS --enable_neon=false
-
-api_test:
-  stage: api_test
-  script:
-    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
-    - >
-      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
-        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
-        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
-      fi
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
-    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
-
-ops_benchmark:
-  stage: ops_benchmark
-  script:
-    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
-    - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS --args="--filter=.*CONV.*"
-
-extra_tests:
-  stage: extra_tests
-  script:
-    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
-    - >
-      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
-        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
-        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
-      fi
-    - python tools/bazel_adb_run.py --target="//mace/utils:tuner_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS
-
 platform_compatible_tests:
  stage: platform_compatible_tests
  script:
@@ -106,6 +62,7 @@ ndk_versions_compatible_tests:
  script:
    - DEFAULT_NDK_PATH=$ANDROID_NDK_HOME
    - prefix_path=${DEFAULT_NDK_PATH%android-ndk-*}
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
    - >
      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
@@ -119,7 +76,6 @@ ndk_versions_compatible_tests:
        export ANDROID_NDK_HOME=$new_ndk_path;
        export PATH=$ANDROID_NDK_HOME:$PATH;
        echo "ndk path: $ANDROID_NDK_HOME";
-        if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
        python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*";
        python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a --target_socs=$TARGET_SOCS --args="--gtest_filter=ActivationOpTest*";
        python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64-v8a --target_socs=$TARGET_SOCS --enable_neon=false --args="--gtest_filter=ActivationOpTest*";
@@ -131,6 +87,44 @@ ndk_versions_compatible_tests:
    - export ANDROID_NDK_HOME=$DEFAULT_NDK_PATH
    - export PATH=$ANDROID_NDK_HOME:$PATH

+ops_test_disable_neon:
+  stage: ops_test_disable_neon
+  script:
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
+    - >
+      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
+        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
+        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
+      fi
+    - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS --enable_neon=false
+
+ops_test:
+  stage: ops_test
+  script:
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
+    - >
+      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
+        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
+        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
+      fi
+    - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS
+
+api_test:
+  stage: api_test
+  script:
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS
+    - >
+      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
+        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
+        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
+      fi
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_mt_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
+    - python tools/bazel_adb_run.py --target="//mace/test:mace_api_exception_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=arm64 --target_socs=$TARGET_SOCS
+
 python_tools_tests:
  stage: python_tools_tests
  script:
@@ -143,11 +137,12 @@ python_tools_tests:
        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
      fi
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
    - >
-      python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,armhf --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,armhf --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,arm64 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1;

 model_tests:
  stage: model_tests
@@ -162,15 +157,16 @@ model_tests:
        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
      fi
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
    - >
-      python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file --cl_mem_type=buffer || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,arm64 --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py convert --config=${CONF_FILE}  --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file --cl_mem_type=buffer || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --round=1 --target_abis=armeabi-v7a,arm64 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --target_abis=armeabi-v7a,arm64 --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
    - CONF_FILE=mace-models/mobilenet-v2/mobilenet-v2-host.yml
    - >
-      python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
    - rm -rf mace-models

 quantization_tests:
@@ -184,13 +180,14 @@ quantization_tests:
        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
      fi
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
    - >
      for CONF_FILE in mace-models/mobilenet-v1/mobilenet-v1-quantize-retrain.yml mace-models/mobilenet-v1/mobilenet-v1-quantize-retrain-for-check-only.yml mace-models/mobilenet-v1/mobilenet-v1-quantize-retrain-dsp.yml;
      do
-      python tools/converter.py convert --config=${CONF_FILE} --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
-      python tools/converter.py run --config=${CONF_FILE} --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py convert --config=${CONF_FILE} --target_socs=$TARGET_SOCS --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --model_graph_format=file --model_data_format=file || exit 1;
+      python tools/converter.py run --config=${CONF_FILE} --target_socs=$TARGET_SOCS --device_yml=${DEVICE_CONF_FILE} --example --round=1 --validate --layers=0 --model_graph_format=file --model_data_format=file || exit 1;
      done
    - rm -rf mace-models

@@ -199,6 +196,23 @@ build_android_demo:
  script:
    - pushd mace/examples/android/ && bash build.sh static && bash build.sh dynamic && popd

+ops_benchmark:
+  stage: ops_benchmark
+  script:
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
+    - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --target_abis=armeabi-v7a,arm64-v8a --target_socs=$TARGET_SOCS --args="--filter=.*CONV.*"
+
+extra_tests:
+  stage: extra_tests
+  script:
+    - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi
+    - >
+      if ping -c 1 v9.git.n.xiaomi.com 1>/dev/null 2>&1; then
+        GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git clone git@v9.git.n.xiaomi.com:deep-computing/generic-mobile-devices.git
+        DEVICE_CONF_FILE=generic-mobile-devices/devices.yml
+      fi
+    - python tools/bazel_adb_run.py --target="//mace/utils:tuner_test" --device_yml=${DEVICE_CONF_FILE} --run_target=True --stdout_processor=unittest_stdout_processor --target_abis=armeabi-v7a,arm64-v8a,arm64 --target_socs=$TARGET_SOCS
+
 so_size_check:
  stage: so_size_check
  script:

--- a/mace/ops/conv_2d_test.cc
+++ b/mace/ops/conv_2d_test.cc
@@ -575,7 +575,7 @@ void TestComplexConvNxNS12(const std::vector<index_t> &shape,
    // Run on device
    net.RunOp(D);
    ExpectTensorNear<float>(*expected, *net.GetOutput("OPENCLOutput"), 1e-4,
-                            1e-4);
+                            1e-3);
  };

  for (int kernel_size : {1, 3, 5, 7}) {

--- a/mace/utils/tuner.h
+++ b/mace/utils/tuner.h
@@ -172,6 +172,10 @@ class Tuner {
                       << " failed, error code: " << strerror(errno);
        }
        return;
+      } else if (!S_ISREG(st.st_mode)) {
+        VLOG(1) << "The path " << tuned_param_file_path_
+                << " is not a file";
+        return;
      }
      int fd = open(tuned_param_file_path_.c_str(), O_RDONLY);
      if (fd < 0) {

--- a/tools/bazel_adb_run.py
+++ b/tools/bazel_adb_run.py
@@ -20,7 +20,6 @@
 #     --stdout_processor=stdout_processor

 import argparse
-import random
 import re
 import sys

@@ -137,19 +136,15 @@ def main(unused_args):
                                address_sanitizer=FLAGS.address_sanitizer)
        if FLAGS.run_target:
            target_devices = DeviceManager.list_devices(FLAGS.device_yml)
-            if FLAGS.target_socs != "all" and FLAGS.target_socs != "random":
+            if FLAGS.target_socs != TargetSOCTag.all and\
+                    FLAGS.target_socs != TargetSOCTag.random:
                target_socs = set(FLAGS.target_socs.split(','))
                target_devices = \
                    [dev for dev in target_devices
                     if dev[YAMLKeyword.target_socs] in target_socs]
-            if FLAGS.target_socs == "random":
-                unlocked_devices = \
-                    [d for d in target_devices if
-                     not sh_commands.is_device_locked(d)]
-                if len(unlocked_devices) > 0:
-                    target_devices = [random.choice(unlocked_devices)]
-                else:
-                    target_devices = [random.choice(target_devices)]
+            if FLAGS.target_socs == TargetSOCTag.random:
+                target_devices = sh_commands.choose_a_random_device(
+                    target_devices, target_abi)

            for dev in target_devices:
                if target_abi not in dev[YAMLKeyword.target_abis]:

--- a/tools/common.py
+++ b/tools/common.py
@@ -428,7 +428,6 @@ BM_MODEL_STATIC_NAME = "benchmark_model_static"
 BM_MODEL_DYNAMIC_NAME = "benchmark_model_dynamic"
 BM_MODEL_STATIC_TARGET = "//mace/benchmark:" + BM_MODEL_STATIC_NAME
 BM_MODEL_DYNAMIC_TARGET = "//mace/benchmark:" + BM_MODEL_DYNAMIC_NAME
-ALL_SOC_TAG = 'all'


 ################################
@@ -486,3 +485,11 @@ class ToolchainType:
    android = 'android'
    arm_linux_gnueabihf = 'arm_linux_gnueabihf'
    aarch64_linux_gnu = 'aarch64_linux_gnu'
+
+
+#################################
+# SOC tag
+#################################
+class TargetSOCTag:
+    all = 'all'
+    random = 'random'
--- a/tools/converter.py
+++ b/tools/converter.py
@@ -229,7 +229,8 @@ def format_model_config(flags):
                   "target_abis must be in " + str(ABITypeStrs))

    target_socs = configs.get(YAMLKeyword.target_socs, "")
-    if flags.target_socs:
+    if flags.target_socs and flags.target_socs != TargetSOCTag.random \
+            and flags.target_socs != TargetSOCTag.all:
        configs[YAMLKeyword.target_socs] = \
            [soc.lower() for soc in flags.target_socs.split(',')]
    elif not target_socs:
@@ -244,7 +245,7 @@ def format_model_config(flags):
            or ABIType.arm64_v8a in target_abis:
        available_socs = sh_commands.adb_get_all_socs()
        target_socs = configs[YAMLKeyword.target_socs]
-        if ALL_SOC_TAG in target_socs:
+        if TargetSOCTag.all in target_socs:
            mace_check(available_socs,
                       ModuleName.YAML_CONFIG,
                       "Android abi is listed in config file and "
@@ -947,12 +948,17 @@ def run_mace(flags):

    target_socs = configs[YAMLKeyword.target_socs]
    device_list = DeviceManager.list_devices(flags.device_yml)
-    if target_socs and ALL_SOC_TAG not in target_socs:
+    if target_socs and TargetSOCTag.all not in target_socs:
        device_list = [dev for dev in device_list
                       if dev[YAMLKeyword.target_socs].lower() in target_socs]
    for target_abi in configs[YAMLKeyword.target_abis]:
+        if flags.target_socs == TargetSOCTag.random:
+            target_devices = sh_commands.choose_a_random_device(
+                device_list, target_abi)
+        else:
+            target_devices = device_list
        # build target
-        for dev in device_list:
+        for dev in target_devices:
            if target_abi in dev[YAMLKeyword.target_abis]:
                # get toolchain
                toolchain = infer_toolchain(target_abi)
@@ -1038,13 +1044,17 @@ def benchmark_model(flags):

    target_socs = configs[YAMLKeyword.target_socs]
    device_list = DeviceManager.list_devices(flags.device_yml)
-    if target_socs and ALL_SOC_TAG not in target_socs:
+    if target_socs and TargetSOCTag.all not in target_socs:
        device_list = [dev for dev in device_list
                       if dev[YAMLKeyword.target_socs].lower() in target_socs]
-
    for target_abi in configs[YAMLKeyword.target_abis]:
+        if flags.target_socs == TargetSOCTag.random:
+            target_devices = sh_commands.choose_a_random_device(
+                device_list, target_abi)
+        else:
+            target_devices = device_list
        # build benchmark_model binary
-        for dev in device_list:
+        for dev in target_devices:
            if target_abi in dev[YAMLKeyword.target_abis]:
                toolchain = infer_toolchain(target_abi)
                build_benchmark_model(configs,

--- a/tools/device.py
+++ b/tools/device.py
@@ -440,8 +440,8 @@ class DeviceWrapper:

    def run_specify_abi(self, flags, configs, target_abi):
        if target_abi not in self.target_abis:
-            six.print_('There is no device with soc: %s abi: %s' %
-                       (self.target_socs, target_abi))
+            six.print_('The device %s with soc %s do not support the abi %s' %
+                       (self.device_name, self.target_socs, target_abi))
            return
        library_name = configs[YAMLKeyword.library_name]
        mace_lib_type = flags.mace_lib_type
@@ -478,17 +478,10 @@ class DeviceWrapper:
            model_runtime = model_config[YAMLKeyword.runtime]
            subgraphs = model_config[YAMLKeyword.subgraphs]

-            if not configs[YAMLKeyword.target_socs] \
-                    or target_abi == ABIType.host:
-                model_output_base_dir, model_output_dir, mace_model_dir = \
-                    get_build_model_dirs(
-                        library_name, model_name, target_abi, self,
-                        model_config[YAMLKeyword.model_file_path])
-            else:
-                model_output_base_dir, model_output_dir, mace_model_dir = \
-                    get_build_model_dirs(
-                        library_name, model_name, target_abi, self,
-                        model_config[YAMLKeyword.model_file_path])
+            model_output_base_dir, model_output_dir, mace_model_dir = \
+                get_build_model_dirs(
+                    library_name, model_name, target_abi, self,
+                    model_config[YAMLKeyword.model_file_path])

            # clear temp model output dir
            if os.path.exists(model_output_dir):
@@ -501,7 +494,8 @@ class DeviceWrapper:
            if not flags.address_sanitizer \
                    and not flags.example \
                    and target_abi != ABIType.host \
-                    and configs[YAMLKeyword.target_socs] \
+                    and (configs[YAMLKeyword.target_socs]
+                         or flags.target_socs) \
                    and self.target_socs \
                    and model_runtime in [RuntimeType.gpu,
                                          RuntimeType.cpu_gpu] \
@@ -867,7 +861,8 @@ class DeviceWrapper:
        else:
            bm_model_binary_name = BM_MODEL_STATIC_NAME
        build_tmp_binary_dir = get_build_binary_dir(library_name, target_abi)
-        if configs[YAMLKeyword.target_socs] and target_abi != ABIType.host:
+        if (configs[YAMLKeyword.target_socs] or flags.target_socs)\
+                and target_abi != ABIType.host:
            opencl_output_bin_path = get_opencl_binary_output_path(
                library_name, target_abi, self
            )

--- a/tools/sh_commands.py
+++ b/tools/sh_commands.py
@@ -17,6 +17,7 @@ import glob
 import logging
 import numpy as np
 import os
+import random
 import re
 import sh
 import struct
@@ -96,6 +97,19 @@ def stdout_success(stdout):
    return True


+# select a random unlocked device support the ABI
+def choose_a_random_device(target_devices, target_abi):
+    eligible_devices = [dev for dev in target_devices
+                        if target_abi in dev[common.YAMLKeyword.target_abis]]
+    unlocked_devices = \
+        [dev for dev in eligible_devices if not is_device_locked(dev)]
+    if len(unlocked_devices) > 0:
+        chosen_devices = [random.choice(unlocked_devices)]
+    else:
+        chosen_devices = [random.choice(eligible_devices)]
+    return chosen_devices
+
+
 ################################
 # clear data
 ################################