Build: improved CPU/GPU build support

1) Introduce USE_GPU_TARGET (inside docker) to determine if GPU capable
2) Use of USE_GPU_HOST now only in cyber|dev_start.sh

For build,

1) Mixed used of '--config=cpu' and '--config=gpu' not allowed
2) Make CPU build (i.e., '--config=cpu') on GPU capable machine possible
3) Error and exit attempting to run GPU build (`--config=gpu`) on CPU-only machines
4) Default to USE_GPU_TARGET if none of cpu|gpu build specified

apollo.sh

@@ -15,7 +15,7 @@ USE_ESD_CAN=false
 function check_architecture_support() {
     if [[ "${SUPPORTED_ARCHS}" != *" ${ARCH} "* ]]; then
         error "Unsupported CPU arch: ${ARCH}. Currently, Apollo only" \
-              "supports running on the following CPU archs:"
+            "supports running on the following CPU archs:"
         error "${TAB}${SUPPORTED_ARCHS}"
         exit 1
     fi
@@ -33,17 +33,17 @@ function check_platform_support() {
 function check_minimal_memory_requirement() {
     local minimal_mem_gb="2.0"
     local actual_mem_gb="$(free -m | awk '/Mem:/ {printf("%0.2f", $2 / 1024.0)}')"
-    if (( $(echo "$actual_mem_gb < $minimal_mem_gb" | bc -l) )); then
+    if (($(echo "$actual_mem_gb < $minimal_mem_gb" | bc -l))); then
         warning "System memory [${actual_mem_gb}G] is lower than the minimum required" \
-                "[${minimal_mem_gb}G]. Apollo build could fail."
+            "[${minimal_mem_gb}G]. Apollo build could fail."
     fi
 }
 
 function determine_esdcan_use() {
     local esdcan_dir="${APOLLO_ROOT_DIR}/third_party/can_card_library/esd_can"
     local use_esd=false
-    if [ -f "${esdcan_dir}/include/ntcan.h" ] && \
-       [ -f "${esdcan_dir}/lib/libntcan.so.4" ]; then
+    if [ -f "${esdcan_dir}/include/ntcan.h" ] &&
+        [ -f "${esdcan_dir}/lib/libntcan.so.4" ]; then
         use_esd=true
     fi
     USE_ESD_CAN="${use_esd}"
@@ -65,7 +65,7 @@ function apollo_env_setup() {
     check_architecture_support
     check_platform_support
     check_minimal_memory_requirement
-    # determine_gpu_use # work done by scripts/apollo.bashrc
+    determine_gpu_use_target
     determine_esdcan_use
 
     APOLLO_ENV="${APOLLO_ENV} STAGE=${STAGE}"
@@ -77,10 +77,11 @@ function apollo_env_setup() {
     info "${TAB}APOLLO_CACHE_DIR: ${APOLLO_CACHE_DIR}"
     info "${TAB}APOLLO_IN_DOCKER: ${APOLLO_IN_DOCKER}"
     info "${TAB}APOLLO_VERSION: ${APOLLO_VERSION}"
-    if "${APOLLO_IN_DOCKER}" ; then
+    if "${APOLLO_IN_DOCKER}"; then
         info "${TAB}DOCKER_IMG: ${DOCKER_IMG##*:}"
     fi
-    info "${TAB}APOLLO_ENV: ${APOLLO_ENV} USE_GPU=${USE_GPU}"
+    info "${TAB}APOLLO_ENV: ${APOLLO_ENV}"
+    info "${TAB}USE_GPU: USE_GPU_HOST=${USE_GPU_HOST} USE_GPU_TARGET=${USE_GPU_TARGET}"
 
     if [ ! -f "${APOLLO_ROOT_DIR}/.apollo.bazelrc" ]; then
         env ${APOLLO_ENV} bash "${APOLLO_ROOT_DIR}/scripts/apollo_config.sh" --noninteractive
@@ -90,7 +91,7 @@ function apollo_env_setup() {
 #TODO(all): Update node modules
 function build_dreamview_frontend() {
     pushd "${APOLLO_ROOT_DIR}/modules/dreamview/frontend" >/dev/null
-        yarn build
+    yarn build
     popd >/dev/null
 }
 
@@ -102,41 +103,44 @@ function build_test_and_lint() {
 }
 
 function _usage() {
-  echo -e "\n${RED}Usage${NO_COLOR}:
-  .${BOLD}/apollo.sh${NO_COLOR} [OPTION]"
-  echo -e "\n${RED}Options${NO_COLOR}:
-  ${BLUE}config [options]${NO_COLOR}: config bazel build environment either non-interactively (default) or interactively.
-  ${BLUE}build [module]${NO_COLOR}: run build for cyber (<module> = cyber) or modules/<module>.  If <module> unspecified, build all.
-  ${BLUE}build_dbg [module]${NO_COLOR}: run debug build (default).
-  ${BLUE}build_opt [module]${NO_COLOR}: run optimized build.
-  ${BLUE}build_cpu [module]${NO_COLOR}: build in CPU mode. Equivalent to 'bazel build --config=cpu'
-  ${BLUE}build_gpu [module]${NO_COLOR}: run build in GPU mode. Equivalent to 'bazel build --config=gpu'
-  ${BLUE}build_opt_gpu [module]${NO_COLOR}: optimized build in GPU mode. Equivalent to 'bazel build --config=opt --config=gpu'
-  ${BLUE}test [module]${NO_COLOR}: run unit tests for cyber (<module> = cyber) or modules/<module>. If <module> unspecified, unit test all.
-  ${BLUE}coverage [module]${NO_COLOR}: run bazel coverage test for cyber (<module> = cyber) or modules/<module>. If <module> unspecified, coverage test all.
-  ${BLUE}lint${NO_COLOR}: run code style check
-  ${BLUE}buildify${NO_COLOR}: run 'buildifier' to fix style of bazel files
-  ${BLUE}check${NO_COLOR}: run build, test and lint on all modules. Make sure check pass before checking in new code
-  ${BLUE}build_fe${NO_COLOR}: compile dreamland frontend javascript. Requires all the node_modules installed already
-  ${BLUE}build_teleop${NO_COLOR}: run build with teleop enabled.
-  ${BLUE}build_prof [module]${NO_COLOR}: build with perf profiling support. Not implemented yet.
-  ${BLUE}doc${NO_COLOR}: generate doxygen document
-  ${BLUE}clean${NO_COLOR}: cleanup bazel output and log/coredump files
-  ${BLUE}format${NO_COLOR}: format C++/Python/Bazel/Shell files
-  ${BLUE}usage${NO_COLOR}: show this message
-  "
+    echo -e "\n${RED}Usage${NO_COLOR}:
+    .${BOLD}/apollo.sh${NO_COLOR} [OPTION]"
+    echo -e "\n${RED}Options${NO_COLOR}:
+    ${BLUE}config [options]${NO_COLOR}: config bazel build environment either non-interactively (default) or interactively.
+    ${BLUE}build [module]${NO_COLOR}: run build for cyber (<module> = cyber) or modules/<module>.  If <module> unspecified, build all.
+    ${BLUE}build_dbg [module]${NO_COLOR}: run debug build (default).
+    ${BLUE}build_opt [module]${NO_COLOR}: run optimized build.
+    ${BLUE}build_cpu [module]${NO_COLOR}: build in CPU mode. Equivalent to 'bazel build --config=cpu'
+    ${BLUE}build_gpu [module]${NO_COLOR}: run build in GPU mode. Equivalent to 'bazel build --config=gpu'
+    ${BLUE}build_opt_gpu [module]${NO_COLOR}: optimized build in GPU mode. Equivalent to 'bazel build --config=opt --config=gpu'
+    ${BLUE}test [module]${NO_COLOR}: run unittest for cyber (module='cyber') or modules/<module>. If unspecified, test all.
+    ${BLUE}coverage [module]${NO_COLOR}: run coverage test for cyber (module='cyber') or modules/<module>. If unspecified, coverage all.
+    ${BLUE}lint${NO_COLOR}: run code style check
+    ${BLUE}buildify${NO_COLOR}: run 'buildifier' to fix style of bazel files.
+    ${BLUE}check${NO_COLOR}: run build, test and lint on all modules. Recommmened before checking in new code.
+    ${BLUE}build_fe${NO_COLOR}: compile frontend JS code for Dreamview. Requires all node_modules pre-installed.
+    ${BLUE}build_teleop${NO_COLOR}: run build with teleop enabled.
+    ${BLUE}build_prof [module]${NO_COLOR}: build with perf profiling support. Not implemented yet.
+    ${BLUE}doc${NO_COLOR}: generate doxygen document
+    ${BLUE}clean${NO_COLOR}: cleanup bazel output and log/coredump files
+    ${BLUE}format${NO_COLOR}: format C++/Python/Bazel/Shell files
+    ${BLUE}usage${NO_COLOR}: show this message
+    "
 }
 
 function main() {
-    apollo_env_setup
     if [ "$#" -eq 0 ]; then
         _usage
         exit 0
     fi
+
+    apollo_env_setup
+
     local build_sh="${APOLLO_ROOT_DIR}/scripts/apollo_build.sh"
     local test_sh="${APOLLO_ROOT_DIR}/scripts/apollo_test.sh"
     local coverage_sh="${APOLLO_ROOT_DIR}/scripts/apollo_coverage.sh"
     local ci_sh="${APOLLO_ROOT_DIR}/scripts/apollo_ci.sh"
+
     local cmd="$1"; shift
     case "${cmd}" in
         config)
@@ -158,10 +162,10 @@ function main() {
             env ${APOLLO_ENV} bash "${build_sh}" --config=gpu "$@"
             ;;
         build_opt_gpu)
-            env ${APOLLO_ENV} bash "${build_sh}" --config=opt_gpu "$@"
+            env ${APOLLO_ENV} bash "${build_sh}" --config=opt --config=gpu "$@"
             ;;
         build_prof)
-            env ${APOLLO_ENV} bash "${build_sh}" --config=cpu_prof "$@"
+            env ${APOLLO_ENV} bash "${build_sh}" --config=prof "$@"
             ;;
         build_teleop)
             env ${APOLLO_ENV} bash "${build_sh}" --config=teleop "$@"
@@ -216,4 +220,3 @@ function main() {
 }
 
 main "$@"
-

scripts/apollo.bashrc

@@ -84,7 +84,7 @@ function fail() {
   exit 1
 }
 
-function determine_gpu_use() {
+function determine_gpu_use_target() {
   local arch="$(uname -m)"
   local use_gpu=0
 
@@ -95,7 +95,6 @@ function determine_gpu_use() {
       fi
     fi
   else ## x86_64 mode
-    # TODO(all): remove USE_GPU env var in {cyber,dev}_start.sh"
     # Check nvidia-driver and GPU device
     local nv_driver="nvidia-smi"
     if [ ! -x "$(command -v ${nv_driver})" ]; then
@@ -106,7 +105,7 @@ function determine_gpu_use() {
       use_gpu=1
     fi
   fi
-  export USE_GPU="${use_gpu}"
+  export USE_GPU_TARGET="${use_gpu}"
 }
 
 function file_ext() {
@@ -173,8 +172,8 @@ function run() {
     "${@}" || exit $?
   else
     local errfile="${APOLLO_ROOT_DIR}/.errors.log"
-    echo "${@}" > "${errfile}"
-    if ! "${@}" >> "${errfile}" 2>&1; then
+    echo "${@}" >"${errfile}"
+    if ! "${@}" >>"${errfile}" 2>&1; then
       local exitcode=$?
       cat "${errfile}" 1>&2
       exit $exitcode
@@ -184,22 +183,22 @@ function run() {
 
 #commit_id=$(git log -1 --pretty=%H)
 function git_sha1() {
-  if [ -x "$(which git 2> /dev/null)" ] \
-    && [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
-    git rev-parse --short HEAD 2> /dev/null || true
+  if [ -x "$(which git 2>/dev/null)" ] &&
+    [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
+    git rev-parse --short HEAD 2>/dev/null || true
   fi
 }
 
 function git_date() {
-  if [ -x "$(which git 2> /dev/null)" ] \
-    && [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
+  if [ -x "$(which git 2>/dev/null)" ] &&
+    [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
     git log -1 --pretty=%ai | cut -d " " -f 1 || true
   fi
 }
 
 function git_branch() {
-  if [ -x "$(which git 2> /dev/null)" ] \
-    && [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
+  if [ -x "$(which git 2>/dev/null)" ] &&
+    [ -d "${APOLLO_ROOT_DIR}/.git" ]; then
     git rev-parse --abbrev-ref HEAD
   else
     echo "@non-git"
@@ -224,10 +223,10 @@ function setup_gpu_support() {
     add_to_path "/usr/local/cuda/bin"
   fi
 
-  determine_gpu_use
+  determine_gpu_use_target
 
   local dev=
-  if [ "${USE_GPU}" -eq 0 ]; then
+  if [ "${USE_GPU_TARGET}" -eq 0 ]; then
     dev="cpu"
   else
     dev="gpu"
@@ -235,7 +234,7 @@ function setup_gpu_support() {
 
   local torch_path="/usr/local/libtorch_${dev}/lib"
   if [ -d "${torch_path}" ]; then
-    # FIXME(all): --config=cpu/gpu
+    # Runtime default: for ./bazel-bin/xxx/yyy to work as expected
     export LD_LIBRARY_PATH="${torch_path}:$LD_LIBRARY_PATH"
   fi
 }

scripts/apollo_build.sh

@@ -24,6 +24,7 @@ source "${TOP_DIR}/scripts/apollo_base.sh"
 ARCH="$(uname -m)"
 
 : ${USE_ESD_CAN:=false}
+USE_GPU=-1
 
 CMDLINE_OPTIONS=
 SHORTHAND_TARGETS=
@@ -40,7 +41,7 @@ function _determine_drivers_disabled() {
 
 function _determine_perception_disabled() {
   if [ "${USE_GPU}" -eq 0 ]; then
-    warning "Perception module can not work without GPU, all targets skipped"
+    warning "Perception can't work without GPU, targets under 'modules/perception' skipped"
     DISABLED_TARGETS="${DISABLED_TARGETS} except //modules/perception/..."
   fi
 }
@@ -54,8 +55,9 @@ function _determine_localization_disabled() {
 
 function _determine_planning_disabled() {
   if [ "${USE_GPU}" -eq 0 ]; then
-    DISABLED_TARGETS="${DISABLED_TARGETS} except //modules/planning/open_space/trajectory_smoother:planning_block \
-                      except //modules/planning/learning_based/..."
+    DISABLED_TARGETS="${DISABLED_TARGETS} \
+        except //modules/planning/open_space/trajectory_smoother:planning_block \
+        except //modules/planning/learning_based/..."
   fi
 }
 
@@ -115,13 +117,13 @@ function determine_build_targets() {
   for component in $@; do
     local build_targets
     local exceptions
-    if [[ "${component}" == "cyber" ]]; then
+    if [ "${component}" = "cyber" ]; then
       build_targets="//cyber/... union //modules/tools/visualizer/..."
     elif [[ -d "${APOLLO_ROOT_DIR}/modules/${component}" ]]; then
       exceptions="$(determine_disabled_build_targets ${component})"
       build_targets="//modules/${component}/... ${exceptions}"
     else
-      error "Oops, no such component '${component}' under <APOLLO_ROOT_DIR>/modules/ . Exiting ..."
+      error "Directory <APOLLO_ROOT_DIR>/modules/${component} not found. Exiting ..."
       exit 1
     fi
     if [ -z "${targets_all}" ]; then
@@ -133,7 +135,29 @@ function determine_build_targets() {
   echo "${targets_all}"
 }
 
-function _parse_cmdline_arguments() {
+function _chk_n_set_gpu_arg() {
+  local arg="$1"
+  local use_gpu=-1
+  if [ "${arg}" = "cpu" ]; then
+    use_gpu=0
+  elif [ "${arg}" = "gpu" ]; then
+    use_gpu=1
+  else
+    # Do nothing
+    return 0
+  fi
+
+  if [[ "${USE_GPU}" -lt 0 || "${USE_GPU}" = "${use_gpu}" ]]; then
+    USE_GPU="${use_gpu}"
+    return 0
+  fi
+
+  error "Mixed use of '--config=cpu' and '--config=gpu' may" \
+    "lead to unexpected behavior. Exiting..."
+  exit 1
+}
+
+function parse_cmdline_arguments() {
   local known_options=""
   local remained_args=""
 
@@ -145,11 +169,13 @@ function _parse_cmdline_arguments() {
       --config=*)
         optarg="${opt#*=}"
         known_options="${known_options} ${opt}"
+        _chk_n_set_gpu_arg "${optarg}"
         ;;
       --config)
         ((++pos))
         optarg="${!pos}"
         known_options="${known_options} ${opt} ${optarg}"
+        _chk_n_set_gpu_arg "${optarg}"
         ;;
       -c)
         ((++pos))
@@ -169,30 +195,45 @@ function _parse_cmdline_arguments() {
   SHORTHAND_TARGETS="${remained_args}"
 }
 
-function _run_bazel_build_impl() {
-  local job_args="--jobs=$(nproc) --local_ram_resources=HOST_RAM*0.7"
-  bazel build ${job_args} $@
-}
-
-function bazel_build() {
-  if ! "${APOLLO_IN_DOCKER}"; then
-    error "The build operation must be run from within docker container"
+function determine_cpu_or_gpu_build() {
+  if [ "${USE_GPU}" -lt 0 ]; then
+    if [ "${USE_GPU_TARGET}" -eq 0 ]; then
+      CMDLINE_OPTIONS="--config=cpu ${CMDLINE_OPTIONS}"
+    else
+      CMDLINE_OPTIONS="--config=gpu ${CMDLINE_OPTIONS}"
+    fi
+    # USE_GPU unset, defaults to USE_GPU_TARGET
+    USE_GPU="${USE_GPU_TARGET}"
+  elif [ "${USE_GPU}" -gt "${USE_GPU_TARGET}" ]; then
+    warning "USE_GPU=${USE_GPU} without GPU can't compile. Exiting ..."
     exit 1
   fi
 
-  _parse_cmdline_arguments $@
+  if [ "${USE_GPU}" -eq 1 ]; then
+    ok "Running GPU build on ${ARCH} platform."
+  else
+    ok "Running CPU build on ${ARCH} platform."
+  fi
+}
 
-  CMDLINE_OPTIONS="${CMDLINE_OPTIONS} --define USE_ESD_CAN=${USE_ESD_CAN}"
+function run_bazel_build() {
+  if ${USE_ESD_CAN}; then
+    CMDLINE_OPTIONS="${CMDLINE_OPTIONS} --define USE_ESD_CAN=${USE_ESD_CAN}"
+  fi
+
+  CMDLINE_OPTIONS="$(echo ${CMDLINE_OPTIONS} | xargs)"
 
   local build_targets
   build_targets="$(determine_build_targets ${SHORTHAND_TARGETS})"
   build_targets="$(echo ${build_targets} | xargs)"
 
   info "Build Overview: "
+  info "${TAB}USE_GPU: ${USE_GPU}  [ 0 for CPU, 1 for GPU ]"
   info "${TAB}Bazel Options: ${GREEN}${CMDLINE_OPTIONS}${NO_COLOR}"
   info "${TAB}Build Targets: ${GREEN}${build_targets}${NO_COLOR}"
 
-  _run_bazel_build_impl "${CMDLINE_OPTIONS}" "$(bazel query ${build_targets})"
+  local job_args="--jobs=$(nproc) --local_ram_resources=HOST_RAM*0.7"
+  bazel build ${CMDLINE_OPTIONS} ${job_args} $(bazel query ${build_targets})
 }
 
 function build_simulator() {
@@ -209,12 +250,15 @@ function build_simulator() {
 }
 
 function main() {
-  if [ "${USE_GPU}" -eq 1 ]; then
-    info "Your GPU is enabled to run the build on ${ARCH} platform."
-  else
-    info "Running build under CPU mode on ${ARCH} platform."
+  if ! "${APOLLO_IN_DOCKER}"; then
+    error "The build operation must be run from within docker container"
+    exit 1
   fi
-  bazel_build $@
+  parse_cmdline_arguments "$@"
+  determine_cpu_or_gpu_build
+
+  run_bazel_build
+
   if [ -z "${SHORTHAND_TARGETS}" ]; then
     SHORTHAND_TARGETS="apollo"
     build_simulator

tools/apollo.bazelrc.sample

@@ -9,7 +9,6 @@ build:gpu --config=cuda
 build:gpu --config=tensorrt
 
 #build:gpu --action_env USE_GPU=1
-#build --config=tensorrt
 
 ## The following was adapted from tensorflow/.bazelrc
 

tools/bazel.rc

@@ -55,15 +55,12 @@ build:optimize --copt=-march=native
 build:optimize --host_copt=-march=native
 build:optimize --define with_default_optimizations=true
 
-build:cpu --define USE_GPU=false
-build:cpu --cxxopt=-DCPU_ONLY
+build:cpu --cxxopt="-DCPU_ONLY=1"
 build:gpu --define USE_GPU=true
-build:opt_gpu --config=opt --config=gpu
 
 # Build with profiling
-build:cpu_prof --config=cpu
-build:cpu_prof --linkopt=-lprofiler
-build:cpu_prof --cxxopt="-DENABLE_PERF"
+build:prof --linkopt=-lprofiler
+build:prof --cxxopt="-DENABLE_PERF=1"
 
 # Build Apollo with C++ 17 features.
 build:c++17 --cxxopt=-std=c++1z