Bazel: add cuda support

.bazelrc

 # load bazelrc from the legacy location
 # as recommended in https://github.com/bazelbuild/bazel/issues/6319
-import %workspace%/tools/bazel.rc
+try-import %workspace%/tools/bazel.rc
+try-import %workspace%/.apollo.bazelrc
+

.gitignore

@@ -73,3 +73,6 @@ docs/demo_guide/*.record
 
 # bazel cache and others
 .cache/
+
+tools/python_bin_path.sh
+.apollo.bazelrc

WORKSPACE.in

@@ -3,6 +3,9 @@ workspace(name = "apollo")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
+load("//tools/gpus:cuda_configure.bzl", "cuda_configure")
+cuda_configure(name = "local_config_cuda")
+
 maybe(
     http_archive,
     name = "bazel_skylib",
@@ -80,9 +83,9 @@ new_local_repository(
 #    strip_prefix = "glog-0.4.0",
 #    url = "https://github.com/google/glog/archive/v0.4.0.tar.gz",
 #)
-
 # See https://github.com/bazelbuild/bazel/issues/11406
-http_archive(
+maybe(
+    http_archive,
     name = "boringssl",
     sha256 = "fb236ae74676dba515e1230aef4cc69ab265af72fc08784a6755a319dd013ca6",
     urls = ["http://182.92.10.148:8310/archive/6.0/boringssl-83da28a68f32023fd3b95a8ae94991a07b1f6c62.tar.gz"],

apollo.sh

@@ -479,16 +479,12 @@ function clean() {
 }
 
 function buildify() {
-  local buildifier_url=https://github.com/bazelbuild/buildtools/releases/download/0.4.5/buildifier
-  wget $buildifier_url -O ~/.buildifier
-  chmod +x ~/.buildifier
-  find . -name '*BUILD' -type f -exec ~/.buildifier -showlog -mode=fix {} +
+  find . -name '*BUILD' -or -name '*.bzl' -type f -exec buildifier -showlog -mode=fix {} +
   if [ $? -eq 0 ]; then
     success 'Buildify worked!'
   else
     fail 'Buildify failed!'
   fi
-  rm ~/.buildifier
 }
 
 function build_fe() {
@@ -585,10 +581,27 @@ function print_usage() {
   "
 }
 
+function bootstrap() {
+  if [ -z "$PYTHON_BIN_PATH" ]; then
+    PYTHON_BIN_PATH=$(which python3 || true)
+  fi
+  if [[ -f "${APOLLO_ROOT_DIR}/.apollo.bazelrc" ]]; then
+    return
+  fi
+  cp -f "${TOP_DIR}/tools/sample.bazelrc" "${APOLLO_ROOT_DIR}/.apollo.bazelrc"
+  # Set all env variables
+  # TODO(storypku): enable bootstrap.py inside docker
+  # $PYTHON_BIN_PATH ${APOLLO_ROOT_DIR}/tools/bootstrap.py $@
+  echo "bootstrap done"
+}
+
 function main() {
 
   check_machine_arch
   apollo_check_system_config
+
+  bootstrap
+
   check_esd_files
 
   DEFINES="--define ARCH=${MACHINE_ARCH} --define CAN_CARD=${CAN_CARD} --cxxopt=-DUSE_ESD_CAN=${USE_ESD_CAN}"

bootstrap.sh

+#!/usr/bin/env bash
+set -eo pipefail
+
+TOP_DIR=$(dirname "$0")
+
+if [[ "$1" == "--noninteractive" ]]; then
+    cp -f "${TOP_DIR}/tools/sample.bazelrc" "${TOP_DIR}/.storydev.bazelrc"
+    exit 0
+fi
+
+if [ -z "$PYTHON_BIN_PATH" ]; then
+    PYTHON_BIN_PATH=$(which python3 || true)
+fi
+
+# Set all env variables
+"$PYTHON_BIN_PATH" "${TOP_DIR}/tools/bootstrap.py" "$@"
+
+echo "Done bootstrap.sh"

modules/map/pnc_map/BUILD

 load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+load("@local_config_cuda//cuda:build_defs.bzl", "cuda_library")
 load("//tools:cpplint.bzl", "cpplint")
-load("//tools:cuda_library.bzl", "cuda_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -10,7 +10,8 @@ cuda_library(
     hdrs = ["cuda_util.h"],
     deps = [
         "//cyber/common:log",
-        "@cuda",
+        "//modules/common/math:geometry",
+        "@local_config_cuda//cuda:cudart",
     ],
 )
 
@@ -87,6 +88,7 @@ cc_test(
     deps = [
         ":path",
         "//modules/common/util",
+        "//modules/routing/proto:routing_cc_proto",
         "@com_google_googletest//:gtest_main",
     ],
 )

tools/bazel.rc

-# bazelrc file
-# https://docs.bazel.build/versions/master/user-manual.html
-
-# bazel >= 0.18 looks for %workspace%/.bazelrc (which redirects here)
-# Older bazel versions look for %workspace%/tools/bazel.rc (this file)
-# See https://github.com/bazelbuild/bazel/issues/6319
-
+# Apollo Bazel configuration file.
+# This file tries to group and simplify build options for Apollo
 # +------------------------------------------------------------+
 # | Startup Options                                            |
 # +------------------------------------------------------------+
 startup --batch_cpu_scheduling
-
 startup --host_jvm_args="-XX:-UseParallelGC"
+startup --output_user_root="/apollo/.cache/bazel"
 
-startup --output_user_root=/apollo/.cache/bazel
 # +------------------------------------------------------------+
-# | Test Configurations                                        |
+# | Common Options                                             |
 # +------------------------------------------------------------+
-
-# By default prints output only from failed tests.
-test --test_output=errors
-
-# +------------------------------------------------------------+
-# | CPP Lint Tests & Unit Tests                                |
-# +------------------------------------------------------------+
-# By default, cpplint tests are run as part of `bazel test` alongside all of
-# the other compilation and test targets.  This is a convenience shortcut to
-# only do the cpplint testing and nothing else.
-# Do bazel test --config=cpplint <target> to enable this configuration.
-# To enable the lint test, the BUILD *must* load the cpplint.bzl by having
-# 'load("//tools:cpplint.bzl", "cpplint")' at the beginning and 'cpplint()'
-# at the end.
-
-test:cpplint --test_tag_filters=cpplint
-test:cpplint --build_tests_only
-
-# Regular unit tests.
-test:unit_test --test_tag_filters=-cpplint
-
-# Coverage tests
-test:coverage --test_tag_filters=-cpplint
-test:coverage --copt=--coverage
-test:coverage --cxxopt=--coverage
-test:coverage --cxxopt=-fprofile-arcs
-test:coverage --cxxopt=-ftest-coverage
-test:coverage --linkopt=-coverage
-test:coverage --linkopt=-lgcov
-test:coverage --linkopt=-lgcc
-test:coverage --linkopt=-lc
+# Force bazel output to use colors (good for jenkins) and print useful errors.
+common --color=yes
 
 # +------------------------------------------------------------+
 # | Build Configurations                                       |
 # +------------------------------------------------------------+
-
-# build with profiling
-build:cpu_prof --linkopt=-lprofiler
-
-# Specify protobuf cc toolchain
-build --proto_toolchain_for_cc="@com_google_protobuf//:cc_toolchain"
-
+# Make Bazel print out all options from rc files.
+build --announce_rc
 build --show_timestamps
 
 # Work around the sandbox issue.
 build --spawn_strategy=standalone
 
+# Enable colorful output of GCC
+build --cxxopt="-fdiagnostics-color=always"
+
 # Do not show warnings from external dependencies.
 # build --output_filter="^//"
 
-build --show_timestamps
-
-# build --copt="-I/usr/include/python3.6m"
 # TODO(storypku): disable the following line temporarily as
 # external/upb/upb/decode.c:164 can't compile
 #build --copt="-Werror=sign-compare"
+build --copt="-Werror=sign-compare"
 build --copt="-Werror=return-type"
 build --copt="-Werror=unused-variable"
 build --copt="-Werror=unused-but-set-variable"
 build --copt="-Werror=switch"
-build --cxxopt="-Werror=sign-compare"
 build --cxxopt="-Werror=reorder"
 
-# Enable C++14
-build --cxxopt="-std=c++1y"
+# Default paths for SYSTEM LIBRARIES
+build --define=PREFIX=/usr
+build --define=LIBDIR=$(PREFIX)/lib
+build --define=INCLUDEDIR=$(PREFIX)/include
 
-# Enable colorful output of GCC
-build --cxxopt="-fdiagnostics-color=always"
+# build --enable_platform_specific_config
+
+# dbg config, as a shorthand for '--config=opt -c dbg'
+build:dbg --config=opt -c dbg
+
+## build -c opt
+build:opt --copt=-march=native
+build:opt --host_copt=-march=native
+build:opt --define with_default_optimizations=true
+
+# Instruction set optimizations
+build:native_arch_linux --copt=-march=native
+
+# Build Apollo with C++ 17 features.
+build:c++17 --cxxopt=-std=c++1z
+build:c++17 --cxxopt=-stdlib=libc++
+build:c++1z --config=c++17
+
+# Enable C++14 (aka c++1y) by default
+build --cxxopt="-std=c++14"
+build --host_cxxopt="-std=c++14"
+
+# build with profiling
+build:cpu_prof --linkopt=-lprofiler
+
+# +------------------------------------------------------------+
+# | Test Configurations                                        |
+# +------------------------------------------------------------+
+test --flaky_test_attempts=3
+test --test_size_filters=small,medium
+test --test_env=LD_LIBRARY_PATH
+
+# By default prints output only from failed tests.
+test --test_output=errors
+
+test:coverage --copt=--coverage
+test:coverage --cxxopt=--coverage
+test:coverage --cxxopt=-fprofile-arcs
+test:coverage --cxxopt=-ftest-coverage
+test:coverage --linkopt=-coverage
+test:coverage --linkopt=-lgcov
+test:coverage --linkopt=-lgcc
+test:coverage --linkopt=-lc
+
+# +------------------------------------------------------------+
+# | CPP Lint Tests & Unit Tests                                |
+# +------------------------------------------------------------+
+# By default, cpplint tests are run as part of `bazel test` alongside all of
+# the other compilation and test targets.  This is a convenience shortcut to
+# only do the cpplint testing and nothing else.
+# Do bazel test --config=cpplint <target> to enable this configuration.
+# To enable the lint test, the BUILD *must* load the cpplint.bzl by having
+# 'load("//tools:cpplint.bzl", "cpplint")' at the beginning and 'cpplint()'
+# at the end.
+test:cpplint --test_tag_filters=cpplint
+test:cpplint --build_tests_only
+# Regular unit tests.
+test:unit_test --test_tag_filters=-cpplint
+# Coverage tests
+test:coverage --test_tag_filters=-cpplint
 
 # +------------------------------------------------------------+
 # | Python Configurations                                      |
 # +------------------------------------------------------------+
-run --python_path=/usr/bin/python3
+# build --copt="-I/usr/include/python3.6m"

tools/bootstrap.py

+#! /usr/bin/env python3
+# Adapted from tensorflow.git/configure.py
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""configure script to get build parameters from user."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import errno
+import os
+import platform
+import re
+import subprocess
+import sys
+
+# pylint: disable=g-import-not-at-top
+try:
+    from shutil import which
+except ImportError:
+    from distutils.spawn import find_executable as which
+# pylint: enable=g-import-not-at-top
+
+_DEFAULT_CUDA_VERSION = '10'
+_DEFAULT_CUDNN_VERSION = '7'
+_DEFAULT_TENSORRT_VERSION = '7'
+_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '6.0,6.1,7.0,7.2,7.5'
+
+_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
+
+_TF_WORKSPACE_ROOT = ''
+_TF_BAZELRC = ''
+_TF_CURRENT_BAZEL_VERSION = None
+_TF_MIN_BAZEL_VERSION = '2.0.0'
+_TF_MAX_BAZEL_VERSION = '3.99.0'
+
+class UserInputError(Exception):
+    pass
+
+
+def is_linux():
+    return platform.system() == 'Linux'
+
+def get_input(question):
+    try:
+        try:
+            answer = raw_input(question)
+        except NameError:
+            answer = input(question)  # pylint: disable=bad-builtin
+    except EOFError:
+        answer = ''
+    return answer
+
+
+def symlink_force(target, link_name):
+    """Force symlink, equivalent of 'ln -sf'.
+
+  Args:
+    target: items to link to.
+    link_name: name of the link.
+  """
+    try:
+        os.symlink(target, link_name)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            os.remove(link_name)
+            os.symlink(target, link_name)
+        else:
+            raise e
+
+
+def sed_in_place(filename, old, new):
+    """Replace old string with new string in file.
+
+  Args:
+    filename: string for filename.
+    old: string to replace.
+    new: new string to replace to.
+  """
+    with open(filename, 'r') as f:
+        filedata = f.read()
+    newdata = filedata.replace(old, new)
+    with open(filename, 'w') as f:
+        f.write(newdata)
+
+
+def write_to_bazelrc(line):
+    with open(_TF_BAZELRC, 'a') as f:
+        f.write(line + '\n')
+
+def write_action_env_to_bazelrc(var_name, var):
+    write_to_bazelrc('build --action_env {}="{}"'.format(var_name, str(var)))
+
+def run_shell(cmd, allow_non_zero=False, stderr=None):
+    if stderr is None:
+        stderr = sys.stdout
+    if allow_non_zero:
+        try:
+            output = subprocess.check_output(cmd, stderr=stderr)
+        except subprocess.CalledProcessError as e:
+            output = e.output
+    else:
+        output = subprocess.check_output(cmd, stderr=stderr)
+    return output.decode('UTF-8').strip()
+
+def get_python_path(environ_cp, python_bin_path):
+    """Get the python site package paths."""
+    python_paths = []
+    if environ_cp.get('PYTHONPATH'):
+        python_paths = environ_cp.get('PYTHONPATH').split(':')
+    try:
+        stderr = open(os.devnull, 'wb')
+        library_paths = run_shell(
+            [
+                python_bin_path, '-c',
+                'import site; print("\\n".join(site.getsitepackages()))'
+            ],
+            stderr=stderr).split('\n')
+    except subprocess.CalledProcessError:
+        library_paths = [
+            run_shell([
+                python_bin_path, '-c',
+                'from distutils.sysconfig import get_python_lib;'
+                'print(get_python_lib())'
+            ])
+        ]
+
+    all_paths = set(python_paths + library_paths)
+
+    paths = []
+    for path in all_paths:
+        if os.path.isdir(path):
+            paths.append(path)
+    return paths
+
+def get_python_major_version(python_bin_path):
+    """Get the python major version."""
+    return run_shell(
+        [python_bin_path, '-c', 'import sys; print(sys.version[0])'])
+
+def setup_python(environ_cp):
+    """Setup python related env variables."""
+    # Get PYTHON_BIN_PATH, default is the current running python.
+    default_python_bin_path = sys.executable
+    ask_python_bin_path = (
+        'Please specify the location of python. [Default is '
+        '{}]: ').format(default_python_bin_path)
+    while True:
+        python_bin_path = get_from_env_or_user_or_default(
+            environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path,
+            default_python_bin_path)
+        # Check if the path is valid
+        if os.path.isfile(python_bin_path) and os.access(
+                python_bin_path, os.X_OK):
+            break
+        elif not os.path.exists(python_bin_path):
+            print('Invalid python path: {} cannot be found.'.format(
+                python_bin_path))
+        else:
+            print('{} is not executable.  Is it the python binary?'.format(
+                python_bin_path))
+        environ_cp['PYTHON_BIN_PATH'] = ''
+
+    # Get PYTHON_LIB_PATH
+    python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
+    if not python_lib_path:
+        python_lib_paths = get_python_path(environ_cp, python_bin_path)
+        if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
+            python_lib_path = python_lib_paths[0]
+        else:
+            print('Found possible Python library paths:\n  %s' %
+                  '\n  '.join(python_lib_paths))
+            default_python_lib_path = python_lib_paths[0]
+            python_lib_path = get_input(
+                'Please input the desired Python library path to use.  '
+                'Default is [{}]\n'.format(python_lib_paths[0]))
+            if not python_lib_path:
+                python_lib_path = default_python_lib_path
+        environ_cp['PYTHON_LIB_PATH'] = python_lib_path
+
+    python_major_version = get_python_major_version(python_bin_path)
+    if python_major_version != '3':
+        print('Python 2 was RETIRED on April 2020. Use Python 3 instead.')
+        sys.exit(1)
+
+    # Set-up env variables used by python_configure.bzl
+    write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
+    write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
+    write_to_bazelrc('build --python_path=\"{}"'.format(python_bin_path))
+    environ_cp['PYTHON_BIN_PATH'] = python_bin_path
+
+    # If choosen python_lib_path is from a path specified in the PYTHONPATH
+    # variable, need to tell bazel to include PYTHONPATH
+    if environ_cp.get('PYTHONPATH'):
+        python_paths = environ_cp.get('PYTHONPATH').split(':')
+        if python_lib_path in python_paths:
+            write_action_env_to_bazelrc('PYTHONPATH',
+                                        environ_cp.get('PYTHONPATH'))
+    # Write tools/python_bin_path.sh
+    with open(
+            os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
+            'w') as f:
+        f.write('export PYTHON_BIN_PATH="{}"'.format(python_bin_path))
+
+def reset_tf_configure_bazelrc():
+    """Reset file that contains customized config settings."""
+    open(_TF_BAZELRC, 'w').close()
+
+def get_var(environ_cp,
+            var_name,
+            query_item,
+            enabled_by_default,
+            question=None,
+            yes_reply=None,
+            no_reply=None):
+    """Get boolean input from user.
+
+  If var_name is not set in env, ask user to enable query_item or not. If the
+  response is empty, use the default.
+
+  Args:
+    environ_cp: copy of the os.environ.
+    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
+    query_item: string for feature related to the variable, e.g. "CUDA for
+      Nvidia GPUs".
+    enabled_by_default: boolean for default behavior.
+    question: optional string for how to ask for user input.
+    yes_reply: optional string for reply when feature is enabled.
+    no_reply: optional string for reply when feature is disabled.
+
+  Returns:
+    boolean value of the variable.
+
+  Raises:
+    UserInputError: if an environment variable is set, but it cannot be
+      interpreted as a boolean indicator, assume that the user has made a
+      scripting error, and will continue to provide invalid input.
+      Raise the error to avoid infinitely looping.
+  """
+    if not question:
+        question = 'Do you wish to build your project with {} support?'.format(
+            query_item)
+    if not yes_reply:
+        yes_reply = '{} support will be enabled for your project.'.format(
+            query_item)
+    if not no_reply:
+        no_reply = 'No {}'.format(yes_reply)
+
+    yes_reply += '\n'
+    no_reply += '\n'
+
+    if enabled_by_default:
+        question += ' [Y/n]: '
+    else:
+        question += ' [y/N]: '
+
+    var = environ_cp.get(var_name)
+    if var is not None:
+        var_content = var.strip().lower()
+        true_strings = ('1', 't', 'true', 'y', 'yes')
+        false_strings = ('0', 'f', 'false', 'n', 'no')
+        if var_content in true_strings:
+            var = True
+        elif var_content in false_strings:
+            var = False
+        else:
+            raise UserInputError(
+                'Environment variable %s must be set as a boolean indicator.\n'
+                'The following are accepted as TRUE : %s.\n'
+                'The following are accepted as FALSE: %s.\n'
+                'Current value is %s.' % (var_name, ', '.join(true_strings),
+                                          ', '.join(false_strings), var))
+
+    while var is None:
+        user_input_origin = get_input(question)
+        user_input = user_input_origin.strip().lower()
+        if user_input == 'y':
+            print(yes_reply)
+            var = True
+        elif user_input == 'n':
+            print(no_reply)
+            var = False
+        elif not user_input:
+            if enabled_by_default:
+                print(yes_reply)
+                var = True
+            else:
+                print(no_reply)
+                var = False
+        else:
+            print('Invalid selection: {}'.format(user_input_origin))
+    return var
+
+
+def set_build_var(environ_cp,
+                  var_name,
+                  query_item,
+                  option_name,
+                  enabled_by_default,
+                  bazel_config_name=None):
+    """Set if query_item will be enabled for the build.
+
+  Ask user if query_item will be enabled. Default is used if no input is given.
+  Set subprocess environment variable and write to .bazelrc if enabled.
+
+  Args:
+    environ_cp: copy of the os.environ.
+    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
+    query_item: string for feature related to the variable, e.g. "CUDA for
+      Nvidia GPUs".
+    option_name: string for option to define in .bazelrc.
+    enabled_by_default: boolean for default behavior.
+    bazel_config_name: Name for Bazel --config argument to enable build feature.
+  """
+
+    var = str(
+        int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
+    environ_cp[var_name] = var
+    if var == '1':
+        write_to_bazelrc('build:%s --define %s=true' % (bazel_config_name,
+                                                        option_name))
+        write_to_bazelrc('build --config=%s' % bazel_config_name)
+    elif bazel_config_name is not None:
+        # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
+        # options and not to set build configs through environment variables.
+        write_to_bazelrc('build:%s --define %s=true' % (bazel_config_name,
+                                                        option_name))
+
+
+def set_action_env_var(environ_cp,
+                       var_name,
+                       query_item,
+                       enabled_by_default,
+                       question=None,
+                       yes_reply=None,
+                       no_reply=None,
+                       bazel_config_name=None):
+    """Set boolean action_env variable.
+
+  Ask user if query_item will be enabled. Default is used if no input is given.
+  Set environment variable and write to .bazelrc.
+
+  Args:
+    environ_cp: copy of the os.environ.
+    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
+    query_item: string for feature related to the variable, e.g. "CUDA for
+      Nvidia GPUs".
+    enabled_by_default: boolean for default behavior.
+    question: optional string for how to ask for user input.
+    yes_reply: optional string for reply when feature is enabled.
+    no_reply: optional string for reply when feature is disabled.
+    bazel_config_name: adding config to .bazelrc instead of action_env.
+  """
+    var = int(
+        get_var(environ_cp, var_name, query_item, enabled_by_default, question,
+                yes_reply, no_reply))
+
+    if not bazel_config_name:
+        write_action_env_to_bazelrc(var_name, var)
+    elif var:
+        write_to_bazelrc('build --config=%s' % bazel_config_name)
+    environ_cp[var_name] = str(var)
+
+
+def convert_version_to_int(version):
+    """Convert a version number to a integer that can be used to compare.
+
+  Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
+  'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
+
+  Args:
+    version: a version to be converted
+
+  Returns:
+    An integer if converted successfully, otherwise return None.
+  """
+    version = version.split('-')[0]
+    version_segments = version.split('.')
+    # Treat "0.24" as "0.24.0"
+    if len(version_segments) == 2:
+        version_segments.append('0')
+    for seg in version_segments:
+        if not seg.isdigit():
+            return None
+
+    version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
+    return int(version_str)
+
+
+def check_bazel_version(min_version, max_version):
+    """Check installed bazel version is between min_version and max_version.
+
+    Args:
+        min_version: string for minimum bazel version (must exist!).
+        max_version: string for maximum bazel version (must exist!).
+
+    Returns:
+        The bazel version detected.
+    """
+    if which('bazel') is None:
+        print('Cannot find bazel. Please install bazel.')
+        sys.exit(0)
+
+    stderr = open(os.devnull, 'wb')
+    curr_version = run_shell(
+        ['bazel', '--version'], allow_non_zero=True, stderr=stderr)
+    if curr_version.startswith('bazel '):
+        curr_version = curr_version.split('bazel ')[1]
+
+    min_version_int = convert_version_to_int(min_version)
+    curr_version_int = convert_version_to_int(curr_version)
+    max_version_int = convert_version_to_int(max_version)
+
+    # Check if current bazel version can be detected properly.
+    if not curr_version_int:
+        print('WARNING: current bazel installation is not a release version.')
+        print('Make sure you are running at least bazel %s' % min_version)
+        return curr_version
+
+    print('You have bazel %s installed.' % curr_version)
+
+    if curr_version_int < min_version_int:
+        print(
+            'Please upgrade your bazel installation to version %s or higher' % min_version)
+        sys.exit(1)
+    if (curr_version_int > max_version_int
+            and 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
+        print(
+            'Please downgrade your bazel installation to version %s or lower'
+            '! To downgrade: download the installer for the old version (from'
+            'https://github.com/bazelbuild/bazel/releases) then run the '
+            'installer.' % max_version)
+        sys.exit(1)
+    return curr_version
+
+
+def set_cc_opt_flags(environ_cp):
+    """Set up architecture-dependent optimization flags.
+
+  Also append CC optimization flags to bazel.rc..
+
+  Args:
+    environ_cp: copy of the os.environ.
+  """
+    default_cc_opt_flags = '-march=native'
+    question = (
+        'Please specify optimization flags to use during compilation when'
+        ' bazel option "--config=opt" is specified [Default is %s]: '
+    ) % default_cc_opt_flags
+    cc_opt_flags = get_from_env_or_user_or_default(
+        environ_cp, 'CC_OPT_FLAGS', question, default_cc_opt_flags)
+    for opt in cc_opt_flags.split():
+        write_to_bazelrc('build:opt --copt=%s' % opt)
+    # It should be safe on the same build host.
+    write_to_bazelrc('build:opt --host_copt=-march=native')
+    write_to_bazelrc('build:opt --define with_default_optimizations=true')
+
+
+def set_tf_cuda_clang(environ_cp):
+    """set TF_CUDA_CLANG action_env.
+
+  Args:
+    environ_cp: copy of the os.environ.
+  """
+    question = 'Do you want to use clang as CUDA compiler?'
+    yes_reply = 'Clang will be used as CUDA compiler.'
+    no_reply = 'nvcc will be used as CUDA compiler.'
+    set_action_env_var(
+        environ_cp,
+        'TF_CUDA_CLANG',
+        None,
+        False,
+        question=question,
+        yes_reply=yes_reply,
+        no_reply=no_reply,
+        bazel_config_name='cuda_clang')
+
+def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
+                                    var_default):
+    """Get var_name either from env, or user or default.
+
+  If var_name has been set as environment variable, use the preset value, else
+  ask for user input. If no input is provided, the default is used.
+
+  Args:
+    environ_cp: copy of the os.environ.
+    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
+    ask_for_var: string for how to ask for user input.
+    var_default: default value string.
+
+  Returns:
+    string value for var_name
+  """
+    var = environ_cp.get(var_name)
+    if not var:
+        var = get_input(ask_for_var)
+        print('\n')
+    if not var:
+        var = var_default
+    return var
+
+
+def set_clang_cuda_compiler_path(environ_cp):
+    """Set CLANG_CUDA_COMPILER_PATH."""
+    default_clang_path = which('clang') or ''
+    ask_clang_path = (
+        'Please specify which clang should be used as device and '
+        'host compiler. [Default is %s]: ') % default_clang_path
+
+    while True:
+        clang_cuda_compiler_path = get_from_env_or_user_or_default(
+            environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
+            default_clang_path)
+        if os.path.exists(clang_cuda_compiler_path):
+            break
+
+        # Reset and retry
+        print('Invalid clang path: %s cannot be found.' %
+              clang_cuda_compiler_path)
+        environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
+
+    # Set CLANG_CUDA_COMPILER_PATH
+    environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
+    write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
+                                clang_cuda_compiler_path)
+
+def prompt_loop_or_load_from_env(environ_cp,
+                                 var_name,
+                                 var_default,
+                                 ask_for_var,
+                                 check_success,
+                                 error_msg,
+                                 suppress_default_error=False,
+                                 resolve_symlinks=False,
+                                 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
+    """Loop over user prompts for an ENV param until receiving a valid response.
+
+  For the env param var_name, read from the environment or verify user input
+  until receiving valid input. When done, set var_name in the environ_cp to its
+  new value.
+
+  Args:
+    environ_cp: (Dict) copy of the os.environ.
+    var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
+    var_default: (String) default value string.
+    ask_for_var: (String) string for how to ask for user input.
+    check_success: (Function) function that takes one argument and returns a
+      boolean. Should return True if the value provided is considered valid. May
+      contain a complex error message if error_msg does not provide enough
+      information. In that case, set suppress_default_error to True.
+    error_msg: (String) String with one and only one '%s'. Formatted with each
+      invalid response upon check_success(input) failure.
+    suppress_default_error: (Bool) Suppress the above error message in favor of
+      one from the check_success function.
+    resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
+    n_ask_attempts: (Integer) Number of times to query for valid input before
+      raising an error and quitting.
+
+  Returns:
+    [String] The value of var_name after querying for input.
+
+  Raises:
+    UserInputError: if a query has been attempted n_ask_attempts times without
+      success, assume that the user has made a scripting error, and will
+      continue to provide invalid input. Raise the error to avoid infinitely
+      looping.
+  """
+    default = environ_cp.get(var_name) or var_default
+    full_query = '%s [Default is %s]: ' % (
+        ask_for_var,
+        default,
+    )
+
+    for _ in range(n_ask_attempts):
+        val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
+                                              default)
+        if check_success(val):
+            break
+        if not suppress_default_error:
+            print(error_msg % val)
+        environ_cp[var_name] = ''
+    else:
+        raise UserInputError(
+            'Invalid %s setting was provided %d times in a row. '
+            'Assuming to be a scripting mistake.' % (var_name, n_ask_attempts))
+
+    if resolve_symlinks and os.path.islink(val):
+        val = os.path.realpath(val)
+    environ_cp[var_name] = val
+    return val
+
+
+def set_gcc_host_compiler_path(environ_cp):
+    """Set GCC_HOST_COMPILER_PATH."""
+    default_gcc_host_compiler_path = which('gcc') or ''
+    cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
+
+    if os.path.islink(cuda_bin_symlink):
+        # os.readlink is only available in linux
+        default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
+
+    gcc_host_compiler_path = prompt_loop_or_load_from_env(
+        environ_cp,
+        var_name='GCC_HOST_COMPILER_PATH',
+        var_default=default_gcc_host_compiler_path,
+        ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.',
+        check_success=os.path.exists,
+        resolve_symlinks=True,
+        error_msg='Invalid gcc path. %s cannot be found.',
+    )
+
+    write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH',
+                                gcc_host_compiler_path)
+
+
+def reformat_version_sequence(version_str, sequence_count):
+    """Reformat the version string to have the given number of sequences.
+
+  For example:
+  Given (7, 2) -> 7.0
+        (7.0.1, 2) -> 7.0
+        (5, 1) -> 5
+        (5.0.3.2, 1) -> 5
+
+  Args:
+      version_str: String, the version string.
+      sequence_count: int, an integer.
+
+  Returns:
+      string, reformatted version string.
+  """
+    v = version_str.split('.')
+    if len(v) < sequence_count:
+        v = v + (['0'] * (sequence_count - len(v)))
+
+    return '.'.join(v[:sequence_count])
+
+
+def set_tf_cuda_paths(environ_cp):
+    """Set TF_CUDA_PATHS."""
+    ask_cuda_paths = (
+        'Please specify the comma-separated list of base paths to look for CUDA '
+        'libraries and headers. [Leave empty to use the default]: ')
+    tf_cuda_paths = get_from_env_or_user_or_default(
+        environ_cp, 'TF_CUDA_PATHS', ask_cuda_paths, '')
+    if tf_cuda_paths:
+        environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
+
+
+def set_tf_cuda_version(environ_cp):
+    """Set TF_CUDA_VERSION."""
+    ask_cuda_version = (
+        'Please specify the CUDA SDK version you want to use. '
+        '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
+    tf_cuda_version = get_from_env_or_user_or_default(
+        environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION)
+    environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
+
+
+def set_tf_cudnn_version(environ_cp):
+    """Set TF_CUDNN_VERSION."""
+    ask_cudnn_version = (
+        'Please specify the cuDNN version you want to use. '
+        '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
+    tf_cudnn_version = get_from_env_or_user_or_default(
+        environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version,
+        _DEFAULT_CUDNN_VERSION)
+    environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
+
+
+def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
+    """Check compatibility between given library and cudnn/cudart libraries."""
+    ldd_bin = which('ldd') or '/usr/bin/ldd'
+    ldd_out = run_shell([ldd_bin, lib], True)
+    ldd_out = ldd_out.split(os.linesep)
+    cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
+    cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
+    cudnn = None
+    cudart = None
+    cudnn_ok = True  # assume no cudnn dependency by default
+    cuda_ok = True  # assume no cuda dependency by default
+    for line in ldd_out:
+        if 'libcudnn.so' in line:
+            cudnn = cudnn_pattern.search(line)
+            cudnn_ok = False
+        elif 'libcudart.so' in line:
+            cudart = cuda_pattern.search(line)
+            cuda_ok = False
+    if cudnn and len(cudnn.group(1)):
+        cudnn = convert_version_to_int(cudnn.group(1))
+    if cudart and len(cudart.group(1)):
+        cudart = convert_version_to_int(cudart.group(1))
+    if cudnn is not None:
+        cudnn_ok = (cudnn == cudnn_ver)
+    if cudart is not None:
+        cuda_ok = (cudart == cuda_ver)
+    return cudnn_ok and cuda_ok
+
+def set_tf_tensorrt_version(environ_cp):
+    """Set TF_TENSORRT_VERSION."""
+    if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
+        return
+
+    ask_tensorrt_version = (
+        'Please specify the TensorRT version you want to use. '
+        '[Leave empty to default to TensorRT %s]: '
+    ) % _DEFAULT_TENSORRT_VERSION
+    tf_tensorrt_version = get_from_env_or_user_or_default(
+        environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
+        _DEFAULT_TENSORRT_VERSION)
+    environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
+
+def set_tf_nccl_version(environ_cp):
+    """Set TF_NCCL_VERSION."""
+    if 'TF_NCCL_VERSION' in environ_cp:
+        return
+
+    ask_nccl_version = (
+        'Please specify the locally installed NCCL version you want to use. '
+        '[Leave empty to use http://github.com/nvidia/nccl]: ')
+    tf_nccl_version = get_from_env_or_user_or_default(
+        environ_cp, 'TF_NCCL_VERSION', ask_nccl_version, '')
+    environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
+
+
+def get_native_cuda_compute_capabilities(environ_cp):
+    """Get native cuda compute capabilities.
+
+  Args:
+    environ_cp: copy of the os.environ.
+
+  Returns:
+    string of native cuda compute capabilities, separated by comma.
+  """
+    device_query_bin = os.path.join(
+        environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
+    if os.path.isfile(device_query_bin) and os.access(device_query_bin,
+                                                      os.X_OK):
+        try:
+            output = run_shell(device_query_bin).split('\n')
+            pattern = re.compile('[0-9]*\\.[0-9]*')
+            output = [pattern.search(x) for x in output if 'Capability' in x]
+            output = ','.join(x.group() for x in output if x is not None)
+        except subprocess.CalledProcessError:
+            output = ''
+    else:
+        output = ''
+    return output
+
+
+def set_tf_cuda_compute_capabilities(environ_cp):
+    """Set TF_CUDA_COMPUTE_CAPABILITIES."""
+    while True:
+        native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
+            environ_cp)
+        if not native_cuda_compute_capabilities:
+            default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
+        else:
+            default_cuda_compute_capabilities = native_cuda_compute_capabilities
+
+        ask_cuda_compute_capabilities = (
+            'Please specify a list of comma-separated '
+            'CUDA compute capabilities you want to '
+            'build with.\nYou can find the compute '
+            'capability of your device at: '
+            'https://developer.nvidia.com/cuda-gpus.\nPlease'
+            ' note that each additional compute '
+            'capability significantly increases your '
+            'build time and binary size, and that '
+            'we only supports compute '
+            'capabilities >= 5.0 [Default is: %s]: ' %
+            default_cuda_compute_capabilities)
+        tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
+            environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
+            ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
+        # Check whether all capabilities from the input is valid
+        all_valid = True
+        # Remove all whitespace characters before splitting the string
+        # that users may insert by accident, as this will result in error
+        tf_cuda_compute_capabilities = ''.join(
+            tf_cuda_compute_capabilities.split())
+        for compute_capability in tf_cuda_compute_capabilities.split(','):
+            m = re.match('[0-9]+.[0-9]+', compute_capability)
+            if not m:
+                print('Invalid compute capability: %s' % compute_capability)
+                all_valid = False
+            else:
+                ver = float(m.group(0))
+                if ver < 5.0:
+                    print(
+                        'ERROR: We only supports CUDA compute capabilities 5.0 '
+                        'and higher. Please re-specify the list of compute '
+                        'capabilities excluding version %s.' % ver)
+                    all_valid = False
+                if ver < 3.5:
+                    print(
+                        'WARNING: XLA does not support CUDA compute capabilities '
+                        'lower than 3.5. Disable XLA when running on older GPUs.'
+                    )
+
+        if all_valid:
+            break
+
+        # Reset and Retry
+        environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
+
+    # Set TF_CUDA_COMPUTE_CAPABILITIES
+    environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
+    write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
+                                tf_cuda_compute_capabilities)
+
+
+def set_other_cuda_vars(environ_cp):
+    """Set other CUDA related variables."""
+    # If CUDA is enabled, always use GPU during build and test.
+    if environ_cp.get('TF_CUDA_CLANG') == '1':
+        write_to_bazelrc('build --config=cuda_clang')
+    else:
+        write_to_bazelrc('build --config=cuda')
+
+
+def set_host_cxx_compiler(environ_cp):
+    """Set HOST_CXX_COMPILER."""
+    default_cxx_host_compiler = which('g++') or ''
+
+    host_cxx_compiler = prompt_loop_or_load_from_env(
+        environ_cp,
+        var_name='HOST_CXX_COMPILER',
+        var_default=default_cxx_host_compiler,
+        ask_for_var=('Please specify which C++ compiler should be used as the '
+                     'host C++ compiler.'),
+        check_success=os.path.exists,
+        error_msg='Invalid C++ compiler path. %s cannot be found.',
+    )
+
+    write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
+
+
+def set_host_c_compiler(environ_cp):
+    """Set HOST_C_COMPILER."""
+    default_c_host_compiler = which('gcc') or ''
+
+    host_c_compiler = prompt_loop_or_load_from_env(
+        environ_cp,
+        var_name='HOST_C_COMPILER',
+        var_default=default_c_host_compiler,
+        ask_for_var=(
+            'Please specify which C compiler should be used as the host '
+            'C compiler.'),
+        check_success=os.path.exists,
+        error_msg='Invalid C compiler path. %s cannot be found.',
+    )
+
+    write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
+
+
+def system_specific_test_config(environ_cp):
+    """Add default build and test flags required for TF tests to bazelrc."""
+    write_to_bazelrc('test --flaky_test_attempts=3')
+    write_to_bazelrc('test --test_size_filters=small,medium')
+
+    # Each instance of --test_tag_filters or --build_tag_filters overrides all
+    # previous instances, so we need to build up a complete list and write a
+    # single list of filters for the .bazelrc file.
+
+    if environ_cp.get('TF_NEED_CUDA', None) == '1':
+        write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
+
+def set_system_libs_flag(environ_cp):
+    syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
+    if syslibs:
+        if ',' in syslibs:
+            syslibs = ','.join(sorted(syslibs.split(',')))
+        else:
+            syslibs = ','.join(sorted(syslibs.split()))
+        write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
+
+    if 'PREFIX' in environ_cp:
+        write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
+    if 'LIBDIR' in environ_cp:
+        write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
+    if 'INCLUDEDIR' in environ_cp:
+        write_to_bazelrc(
+            'build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
+
+def config_info_line(name, help_text):
+    """Helper function to print formatted help text for Bazel config options."""
+    print('\t--config=%-12s\t# %s' % (name, help_text))
+
+
+def validate_cuda_config(environ_cp):
+    """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
+
+    def maybe_encode_env(env):
+        """Encodes unicode in env to str on Windows python 2.x."""
+        if sys.version_info[0] != 2:
+            return env
+        for k, v in env.items():
+            if isinstance(k, unicode):
+                k = k.encode('ascii')
+            if isinstance(v, unicode):
+                v = v.encode('ascii')
+            env[k] = v
+        return env
+
+    cuda_libraries = ['cuda', 'cudnn']
+    if int(environ_cp.get('TF_NEED_TENSORRT', False)):
+        cuda_libraries.append('tensorrt')
+    if environ_cp.get('TF_NCCL_VERSION', None):
+        cuda_libraries.append('nccl')
+
+    proc = subprocess.Popen(
+        [environ_cp['PYTHON_BIN_PATH'], 'tools/gpus/find_cuda_config.py'] +
+        cuda_libraries,
+        stdout=subprocess.PIPE,
+        env=maybe_encode_env(environ_cp))
+
+    if proc.wait():
+        # Errors from find_cuda_config.py were sent to stderr.
+        print('Asking for detailed CUDA configuration...\n')
+        return False
+
+    config = dict(
+        tuple(line.decode('ascii').rstrip().split(': '))
+        for line in proc.stdout)
+
+    print('Found CUDA %s in:' % config['cuda_version'])
+    print('    %s' % config['cuda_library_dir'])
+    print('    %s' % config['cuda_include_dir'])
+
+    print('Found cuDNN %s in:' % config['cudnn_version'])
+    print('    %s' % config['cudnn_library_dir'])
+    print('    %s' % config['cudnn_include_dir'])
+
+    if 'tensorrt_version' in config:
+        print('Found TensorRT %s in:' % config['tensorrt_version'])
+        print('    %s' % config['tensorrt_library_dir'])
+        print('    %s' % config['tensorrt_include_dir'])
+
+    if config.get('nccl_version', None):
+        print('Found NCCL %s in:' % config['nccl_version'])
+        print('    %s' % config['nccl_library_dir'])
+        print('    %s' % config['nccl_include_dir'])
+
+    print('\n')
+
+    environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
+    return True
+
+def set_overall_build_config():
+    overall_text = """
+## The following was adapted from tensorflow/.bazelrc
+
+# This config refers to building with CUDA available. It does not necessarily
+# mean that we build CUDA op kernels.
+build:using_cuda --define=using_cuda=true
+build:using_cuda --action_env TF_NEED_CUDA=1
+build:using_cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
+
+# This config refers to building CUDA op kernels with nvcc.
+build:cuda --config=using_cuda
+build:cuda --define=using_cuda_nvcc=true
+
+# This config refers to building CUDA op kernels with clang.
+build:cuda_clang --config=using_cuda
+build:cuda_clang --define=using_cuda_clang=true
+build:cuda_clang --define=using_clang=true
+build:cuda_clang --action_env TF_CUDA_CLANG=1
+
+build:tensorrt --action_env TF_NEED_TENSORRT=1
+build:nonccl --define=no_nccl_support=true
+"""
+    with open(_TF_BAZELRC, 'a') as f:
+        f.write(overall_text)
+
+def default_workspace_directory():
+    current_dir = os.path.dirname(__file__)
+    if len(current_dir) == 0:
+        current_dir = "."
+    return os.path.abspath(current_dir + "/..")
+
+def main():
+    if not is_linux():
+        raise ValueError("Currently ONLY Linux platform is supported.")
+
+    global _TF_WORKSPACE_ROOT
+    global _TF_BAZELRC
+    global _TF_CURRENT_BAZEL_VERSION
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--workspace',
+        type=str,
+        default=default_workspace_directory(),
+        help='The absolute path to your active Bazel workspace.')
+    parser.add_argument(
+        '--output_file',
+        type=str,
+        default='.apollo.bazelrc',
+        help='Path of the bazelrc file to write to (relative to workspace directory)')
+
+    args = parser.parse_args()
+
+    _TF_WORKSPACE_ROOT = args.workspace
+    _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, args.output_file)
+
+    # Make a copy of os.environ to be clear when functions and getting and setting
+    # environment variables.
+    environ_cp = dict(os.environ)
+
+    try:
+        current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
+                                                    _TF_MAX_BAZEL_VERSION)
+    except subprocess.CalledProcessError as e:
+        print("Error checking bazel version: ",
+              e.output.decode('UTF-8').strip())
+        raise e
+
+    _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
+
+    reset_tf_configure_bazelrc()
+    setup_python(environ_cp)
+
+    environ_cp['TF_NEED_CUDA'] = str(
+        int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
+    if environ_cp.get('TF_NEED_CUDA') == '1':
+        set_action_env_var(
+            environ_cp,
+            'TF_NEED_TENSORRT',
+            'TensorRT',
+            False,
+            bazel_config_name='tensorrt')
+
+        environ_save = dict(environ_cp)
+        for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
+
+            if validate_cuda_config(environ_cp):
+                cuda_env_names = [
+                    'TF_CUDA_VERSION',
+                    'TF_CUBLAS_VERSION',
+                    'TF_CUDNN_VERSION',
+                    'TF_TENSORRT_VERSION',
+                    'TF_NCCL_VERSION',
+                    'TF_CUDA_PATHS',
+                    # Items below are for backwards compatibility when not using
+                    # TF_CUDA_PATHS.
+                    'CUDA_TOOLKIT_PATH',
+                    'CUDNN_INSTALL_PATH',
+                    'NCCL_INSTALL_PATH',
+                    'NCCL_HDR_PATH',
+                    'TENSORRT_INSTALL_PATH'
+                ]
+                # Note: set_action_env_var above already writes to bazelrc.
+                for name in cuda_env_names:
+                    if name in environ_cp:
+                        write_action_env_to_bazelrc(name, environ_cp[name])
+                break
+
+            # Restore settings changed below if CUDA config could not be validated.
+            environ_cp = dict(environ_save)
+
+            set_tf_cuda_version(environ_cp)
+            set_tf_cudnn_version(environ_cp)
+            set_tf_tensorrt_version(environ_cp)
+            set_tf_nccl_version(environ_cp)
+
+            set_tf_cuda_paths(environ_cp)
+
+        else:
+            raise UserInputError(
+                'Invalid CUDA setting were provided %d '
+                'times in a row. Assuming to be a scripting mistake.' %
+                _DEFAULT_PROMPT_ASK_ATTEMPTS)
+
+        set_tf_cuda_compute_capabilities(environ_cp)
+        if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
+                'LD_LIBRARY_PATH') != '1':
+            write_action_env_to_bazelrc('LD_LIBRARY_PATH',
+                                        environ_cp.get('LD_LIBRARY_PATH'))
+
+        set_tf_cuda_clang(environ_cp)
+        if environ_cp.get('TF_CUDA_CLANG') == '1':
+            # Set up which clang we should use as the cuda / host compiler.
+            set_clang_cuda_compiler_path(environ_cp)
+        else:
+            # Set up which gcc nvcc should use as the host compiler
+            set_gcc_host_compiler_path(environ_cp)
+        set_other_cuda_vars(environ_cp)
+    else:
+        # CUDA not required. Ask whether we should download the clang toolchain and
+        # use it for the CPU build.
+        pass
+        # set_tf_download_clang(environ_cp)
+
+#    set_system_libs_flag(environ_cp)
+#    set_cc_opt_flags(environ_cp)
+#    system_specific_test_config(environ_cp)
+    set_overall_build_config()
+
+#    print('Preconfigured Bazel build configs. You can use any of the below by '
+#          'adding "--config=<>" to your build command. See .bazelrc for more '
+#          'details.')
+#    config_info_line('mkl', 'Build with MKL support.')
+#    config_info_line('ngraph', 'Build with Intel nGraph support.')
+#    config_info_line('numa', 'Build with NUMA support.')
+#    config_info_line(
+#        'dynamic_kernels',
+#        '(Experimental) Build kernels into separate shared objects.')
+    print('Preconfigured Bazel build configs to DISABLE default on features:')
+    config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
+#    config_info_line('noaws', 'Disable AWS S3 filesystem support.')
+#    config_info_line('nogcp', 'Disable GCP support.')
+#    config_info_line('nohdfs', 'Disable HDFS support.')
+
+if __name__ == '__main__':
+    main()

tools/cuda_library.bzl

-cuda_srcs = [".cu", ".cc", ".cpp"]
-cuda_headers = [".h", ".hpp"]
-
-cuda_arch = " ".join([
-    "-arch=sm_30",
-    "-gencode=arch=compute_30,code=sm_30",
-    "-gencode=arch=compute_50,code=sm_50",
-    "-gencode=arch=compute_52,code=sm_52",
-    "-gencode=arch=compute_60,code=sm_60",
-    "-gencode=arch=compute_61,code=sm_61",
-    "-gencode=arch=compute_61,code=compute_61",
-])
-
-def cuda_library_impl(ctx):
-    flags = " ".join(ctx.attr.flags)
-    output = ctx.outputs.out
-    lib_flags = ["-std=c++11", "--shared", "--compiler-options -fPIC"]
-    args = [f.path for f in ctx.files.srcs]
-    deps_flags = []
-    for f in ctx.attr.deps:
-        deps_flags += f.cc.link_flags
-        deps_flags += ["-I" + d for d in f.cc.include_directories]
-        deps_flags += ["-I" + d for d in f.cc.quote_include_directories]
-        deps_flags += ["-I" + d for d in f.cc.system_include_directories]
-
-    ctx.actions.run_shell(
-        inputs = ctx.files.srcs + ctx.files.hdrs + ctx.files.deps,
-        outputs = [ctx.outputs.out],
-        arguments = args,
-        env = {"PATH": "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"},
-        command = "nvcc %s %s %s -I. %s -o %s" % (cuda_arch, " ".join(lib_flags), " ".join(args), " ".join(deps_flags), output.path),
-    )
-
-def cuda_binary_impl(ctx):
-    flags = " ".join(ctx.attr.flags)
-    args = ctx.attr.flags + [f.path for f in ctx.files.srcs] + [f.path for f in ctx.files.hdrs]
-    deps_flags = []
-    for f in ctx.attr.deps:
-        deps_flags += f.cc.link_flags
-        deps_flags += ["-I" + d for d in f.cc.include_directories]
-        deps_flags += ["-I" + d for d in f.cc.quote_include_directories]
-        deps_flags += ["-I" + d for d in f.cc.system_include_directories]
-    output = ctx.outputs.out
-    ctx.actions.run_shell(
-        inputs = ctx.files.srcs + ctx.files.hdrs + ctx.files.deps,
-        outputs = [ctx.outputs.out],
-        arguments = args,
-        env = {"PATH": "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"},
-        command = "nvcc %s %s %s -o %s" % (" ".join(cuda_arch), " ".join(args), " ".join(deps_flags), output.path),
-    )
-
-cuda_library = rule(
-    attrs = {
-        "hdrs": attr.label_list(allow_files = cuda_headers),
-        "srcs": attr.label_list(allow_files = cuda_srcs),
-        "deps": attr.label_list(allow_files = False),
-        "flags": attr.label_list(allow_files = False),
-    },
-    outputs = {"out": "lib%{name}.so"},
-    implementation = cuda_library_impl,
-)
-
-cuda_binary = rule(
-    attrs = {
-        "hdrs": attr.label_list(allow_files = cuda_headers),
-        "srcs": attr.label_list(allow_files = cuda_srcs),
-        "deps": attr.label_list(allow_files = False),
-        "flags": attr.label_list(allow_files = False),
-    },
-    executable = True,
-    outputs = {"out": "%{name}"},
-    implementation = cuda_binary_impl,
-)

tools/gpus/BUILD

+package(
+    default_visibility = ["//visibility:public"],
+)

tools/gpus/README.md

+## How to generate "find_cuda_config.py.gz.base64"
+
+```
+$ # In tools/gpus/ dir
+$ python3 compress_find_cuda_config.py
+```

tools/gpus/check_cuda_libs.py

+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Verifies that a list of libraries is installed on the system.
+
+Takes a a list of arguments with every two subsequent arguments being a logical
+tuple of (path, check_soname). The path to the library and either True or False
+to indicate whether to check the soname field on the shared library.
+
+Example Usage:
+./check_cuda_libs.py /path/to/lib1.so True /path/to/lib2.so False
+"""
+import os
+import os.path
+import platform
+import subprocess
+import sys
+
+# pylint: disable=g-import-not-at-top,g-importing-member
+try:
+  from shutil import which
+except ImportError:
+  from distutils.spawn import find_executable as which
+# pylint: enable=g-import-not-at-top,g-importing-member
+
+
+class ConfigError(Exception):
+  pass
+
+
+def _is_windows():
+  return platform.system() == "Windows"
+
+
+def check_cuda_lib(path, check_soname=True):
+  """Tests if a library exists on disk and whether its soname matches the filename.
+
+  Args:
+    path: the path to the library.
+    check_soname: whether to check the soname as well.
+
+  Raises:
+    ConfigError: If the library does not exist or if its soname does not match
+    the filename.
+  """
+  if not os.path.isfile(path):
+    raise ConfigError("No library found under: " + path)
+  objdump = which("objdump")
+  if check_soname and objdump is not None and not _is_windows():
+    # Decode is necessary as in py3 the return type changed from str to bytes
+    output = subprocess.check_output([objdump, "-p", path]).decode("utf-8")
+    output = [line for line in output.splitlines() if "SONAME" in line]
+    sonames = [line.strip().split(" ")[-1] for line in output]
+    if not any([soname == os.path.basename(path) for soname in sonames]):
+      raise ConfigError("None of the libraries match their SONAME: " + path)
+
+
+def main():
+  try:
+    args = [argv for argv in sys.argv[1:]]
+    if len(args) % 2 == 1:
+      raise ConfigError("Expected even number of arguments")
+    checked_paths = []
+    for i in range(0, len(args), 2):
+      path = args[i]
+      check_cuda_lib(path, check_soname=args[i + 1] == "True")
+      checked_paths.append(path)
+    # pylint: disable=superfluous-parens
+    print(os.linesep.join(checked_paths))
+    # pylint: enable=superfluous-parens
+  except ConfigError as e:
+    sys.stderr.write(str(e))
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+  main()

tools/gpus/compress_find_cuda_config.py

+#! /usr/bin/env python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Compresses the contents of find_cuda_config.py
+
+The compressed file is what is actually being used. It works around remote
+config not being able to upload files yet.
+"""
+import base64
+import zlib
+
+
+def main():
+    with open('find_cuda_config.py', 'rb') as f:
+        data = f.read()
+
+    compressed = zlib.compress(data)
+    b64encoded = base64.b64encode(compressed)
+
+    with open('find_cuda_config.py.gz.base64', 'wb') as f:
+        f.write(b64encoded)
+
+
+if __name__ == '__main__':
+    main()

tools/gpus/crosstool/BUILD.tpl

+# This file is expanded from a template by cuda_configure.bzl
+# Update cuda_configure.bzl#verify_build_defines when adding new variables.
+
+load(":cc_toolchain_config.bzl", "cc_toolchain_config")
+
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "toolchain-linux-x86_64",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    toolchain = ":cc-compiler-local",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "local|compiler": ":cc-compiler-local",
+        "aarch64": ":cc-compiler-local",
+        "k8": ":cc-compiler-local",
+
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-local",
+    all_files = "%{compiler_deps}",
+    compiler_files = "%{compiler_deps}",
+    ar_files = "%{compiler_deps}",
+    as_files = "%{compiler_deps}",
+    dwp_files = ":empty",
+    linker_files = "%{compiler_deps}",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    # To support linker flags that need to go to the start of command line
+    # we need the toolchain to support parameter files. Parameter files are
+    # last on the command line and contain all shared libraries to link, so all
+    # regular options will be left of them.
+    supports_param_files = 1,
+    toolchain_identifier = "local_linux",
+    toolchain_config = ":cc-compiler-local-config",
+)
+
+cc_toolchain_config(
+    name = "cc-compiler-local-config",
+    cpu = "local",
+    builtin_include_directories = [%{cxx_builtin_include_directories}],
+    extra_no_canonical_prefixes_flags = [%{extra_no_canonical_prefixes_flags}],
+    host_compiler_path = "%{host_compiler_path}",
+    host_compiler_prefix = "%{host_compiler_prefix}",
+    host_compiler_warnings = [%{host_compiler_warnings}],
+    host_unfiltered_compile_flags = [%{unfiltered_compile_flags}],
+    linker_bin_path = "%{linker_bin_path}",
+    builtin_sysroot = "%{builtin_sysroot}",
+    cuda_path = "%{cuda_toolkit_path}",
+    compiler = "%{compiler}",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "crosstool_wrapper_driver_is_not_gcc",
+    srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+)
+

tools/gpus/crosstool/LICENSE

+Copyright 2015 The TensorFlow Authors.  All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2015, The TensorFlow Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tools/gpus/crosstool/cc_toolchain_config.bzl.tpl

+"""cc_toolchain_config rule for configuring CUDA toolchains on Linux ONLY"""
+
+load(
+    "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl",
+    "action_config",
+    "env_entry",
+    "env_set",
+    "feature",
+    "feature_set",
+    "flag_group",
+    "flag_set",
+    "tool",
+    "tool_path",
+    "variable_with_value",
+    "with_feature_set",
+)
+load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
+
+def all_assembly_actions():
+    return [
+        ACTION_NAMES.assemble,
+        ACTION_NAMES.preprocess_assemble,
+    ]
+
+def all_compile_actions():
+    return [
+        ACTION_NAMES.assemble,
+        ACTION_NAMES.c_compile,
+        ACTION_NAMES.cpp_compile,
+        ACTION_NAMES.cpp_header_parsing,
+        ACTION_NAMES.cpp_module_codegen,
+        ACTION_NAMES.cpp_module_compile,
+        ACTION_NAMES.linkstamp_compile,
+        ACTION_NAMES.preprocess_assemble,
+    ]
+
+def all_c_compile_actions():
+    return [
+        ACTION_NAMES.c_compile,
+    ]
+
+def all_cpp_compile_actions():
+    return [
+        ACTION_NAMES.cpp_compile,
+        ACTION_NAMES.cpp_header_parsing,
+        ACTION_NAMES.cpp_module_codegen,
+        ACTION_NAMES.cpp_module_compile,
+        ACTION_NAMES.linkstamp_compile,
+    ]
+
+def all_preprocessed_actions():
+    return [
+        ACTION_NAMES.c_compile,
+        ACTION_NAMES.cpp_compile,
+        ACTION_NAMES.cpp_header_parsing,
+        ACTION_NAMES.cpp_module_codegen,
+        ACTION_NAMES.cpp_module_compile,
+        ACTION_NAMES.linkstamp_compile,
+        ACTION_NAMES.preprocess_assemble,
+    ]
+
+def all_link_actions():
+    return [
+        ACTION_NAMES.cpp_link_executable,
+        ACTION_NAMES.cpp_link_dynamic_library,
+        ACTION_NAMES.cpp_link_nodeps_dynamic_library,
+    ]
+
+def all_executable_link_actions():
+    return [
+        ACTION_NAMES.cpp_link_executable,
+    ]
+
+def all_shared_library_link_actions():
+    return [
+        ACTION_NAMES.cpp_link_dynamic_library,
+        ACTION_NAMES.cpp_link_nodeps_dynamic_library,
+    ]
+
+def all_archive_actions():
+    return [ACTION_NAMES.cpp_link_static_library]
+
+def all_strip_actions():
+    return [ACTION_NAMES.strip]
+
+def _library_to_link(flag_prefix, value, iterate = None):
+    return flag_group(
+        flags = [
+            "{}%{{libraries_to_link.{}}}".format(
+                flag_prefix,
+                iterate if iterate else "name",
+            ),
+        ],
+        iterate_over = ("libraries_to_link." + iterate if iterate else None),
+        expand_if_equal = variable_with_value(
+            name = "libraries_to_link.type",
+            value = value,
+        ),
+    )
+
+def _surround_static_library(prefix, suffix):
+    return [
+        flag_group(
+            flags = [prefix, "%{libraries_to_link.name}", suffix],
+            expand_if_true = "libraries_to_link.is_whole_archive",
+        ),
+        flag_group(
+            flags = ["%{libraries_to_link.name}"],
+            expand_if_false = "libraries_to_link.is_whole_archive",
+        ),
+    ]
+
+def _prefix_static_library(prefix):
+    return [
+        flag_group(
+            flags = ["%{libraries_to_link.name}"],
+            expand_if_false = "libraries_to_link.is_whole_archive",
+        ),
+        flag_group(
+            flags = [prefix + "%{libraries_to_link.name}"],
+            expand_if_true = "libraries_to_link.is_whole_archive",
+        ),
+    ]
+
+def _static_library_to_link(alwayslink_prefix, alwayslink_suffix = None):
+    if alwayslink_suffix:
+        flag_groups = _surround_static_library(alwayslink_prefix, alwayslink_suffix)
+    else:
+        flag_groups = _prefix_static_library(alwayslink_prefix)
+    return flag_group(
+        flag_groups = flag_groups,
+        expand_if_equal = variable_with_value(
+            name = "libraries_to_link.type",
+            value = "static_library",
+        ),
+    )
+
+def _iterate_flag_group(iterate_over, flags = [], flag_groups = []):
+    return flag_group(
+        iterate_over = iterate_over,
+        expand_if_available = iterate_over,
+        flag_groups = flag_groups,
+        flags = flags,
+    )
+
+def _libraries_to_link_group(flavour):
+    if flavour != "linux":
+        fail("Oops, we have tailored cc_toolchain_config.bzl to support Linux ONLY")
+    else:
+        return _iterate_flag_group(
+            iterate_over = "libraries_to_link",
+            flag_groups = [
+                flag_group(
+                    flags = ["-Wl,--start-lib"],
+                    expand_if_equal = variable_with_value(
+                        name = "libraries_to_link.type",
+                        value = "object_file_group",
+                    ),
+                ),
+                _library_to_link("", "object_file_group", "object_files"),
+                flag_group(
+                    flags = ["-Wl,--end-lib"],
+                    expand_if_equal = variable_with_value(
+                        name = "libraries_to_link.type",
+                        value = "object_file_group",
+                    ),
+                ),
+                _library_to_link("", "object_file"),
+                _library_to_link("", "interface_library"),
+                _static_library_to_link("-Wl,-whole-archive", "-Wl,-no-whole-archive"),
+                _library_to_link("-l", "dynamic_library"),
+                _library_to_link("-l:", "versioned_dynamic_library"),
+            ],
+        )
+
+def _action_configs_with_tool(path, actions):
+    return [
+        action_config(
+            action_name = name,
+            enabled = True,
+            tools = [tool(path = path)],
+        )
+        for name in actions
+    ]
+
+def _action_configs(assembly_path, c_compiler_path, cc_compiler_path, archiver_path, linker_path, strip_path):
+    return _action_configs_with_tool(
+        assembly_path,
+        all_assembly_actions(),
+    ) + _action_configs_with_tool(
+        c_compiler_path,
+        all_c_compile_actions(),
+    ) + _action_configs_with_tool(
+        cc_compiler_path,
+        all_cpp_compile_actions(),
+    ) + _action_configs_with_tool(
+        archiver_path,
+        all_archive_actions(),
+    ) + _action_configs_with_tool(
+        linker_path,
+        all_link_actions(),
+    ) + _action_configs_with_tool(
+        strip_path,
+        all_strip_actions(),
+    )
+
+def _tool_paths(cpu, ctx):
+    if cpu == "local":
+        return [
+            tool_path(name = "gcc", path = ctx.attr.host_compiler_path),
+            tool_path(name = "ar", path = ctx.attr.host_compiler_prefix + (
+                "/ar" if cpu == "local" else "/libtool"
+            )),
+            tool_path(name = "compat-ld", path = ctx.attr.host_compiler_prefix + "/ld"),
+            tool_path(name = "cpp", path = ctx.attr.host_compiler_prefix + "/cpp"),
+            tool_path(name = "dwp", path = ctx.attr.host_compiler_prefix + "/dwp"),
+            tool_path(name = "gcov", path = ctx.attr.host_compiler_prefix + "/gcov"),
+            tool_path(name = "ld", path = ctx.attr.host_compiler_prefix + "/ld"),
+            tool_path(name = "nm", path = ctx.attr.host_compiler_prefix + "/nm"),
+            tool_path(name = "objcopy", path = ctx.attr.host_compiler_prefix + "/objcopy"),
+            tool_path(name = "objdump", path = ctx.attr.host_compiler_prefix + "/objdump"),
+            tool_path(name = "strip", path = ctx.attr.host_compiler_prefix + "/strip"),
+        ]
+    else:
+        fail("Unreachable")
+
+def _sysroot_group():
+    return flag_group(
+        flags = ["--sysroot=%{sysroot}"],
+        expand_if_available = "sysroot",
+    )
+
+def _no_canonical_prefixes_group(extra_flags):
+    return flag_group(
+        flags = [
+            "-no-canonical-prefixes",
+        ] + extra_flags,
+    )
+
+def _cuda_set(cuda_path, actions):
+    if cuda_path:
+        return flag_set(
+            actions = actions,
+            flag_groups = [
+                flag_group(
+                    flags = ["--cuda-path=" + cuda_path],
+                ),
+            ],
+        )
+    else:
+        return []
+
+def _nologo():
+    return flag_group(flags = ["/nologo"])
+
+def _features(cpu, compiler, ctx):
+    if cpu != "local":
+        fail("Oops, we have tailored cc_toolchain_config.bzl to support Linux ONLY")
+
+    return [
+        feature(name = "no_legacy_features"),
+        feature(
+            name = "all_compile_flags",
+            enabled = True,
+            flag_sets = [
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [
+                        flag_group(
+                            flags = ["-MD", "-MF", "%{dependency_file}"],
+                            expand_if_available = "dependency_file",
+                        ),
+                        flag_group(
+                            flags = ["-gsplit-dwarf"],
+                            expand_if_available = "per_object_debug_info_file",
+                        ),
+                    ],
+                ),
+                flag_set(
+                    actions = all_preprocessed_actions(),
+                    flag_groups = [
+                        flag_group(
+                            flags = ["-frandom-seed=%{output_file}"],
+                            expand_if_available = "output_file",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-D%{preprocessor_defines}"],
+                            iterate_over = "preprocessor_defines",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-include", "%{includes}"],
+                            iterate_over = "includes",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-iquote", "%{quote_include_paths}"],
+                            iterate_over = "quote_include_paths",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-I%{include_paths}"],
+                            iterate_over = "include_paths",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-isystem", "%{system_include_paths}"],
+                            iterate_over = "system_include_paths",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-F", "%{framework_include_paths}"],
+                            iterate_over = "framework_include_paths",
+                        ),
+                    ],
+                ),
+                flag_set(
+                    actions = all_cpp_compile_actions(),
+                    flag_groups = [
+                        flag_group(flags = ["-fexperimental-new-pass-manager"]),
+                    ] if compiler == "clang" else [],
+                ),
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [
+                        flag_group(
+                            flags = [
+                                "-Wno-builtin-macro-redefined",
+                                "-D__DATE__=\"redacted\"",
+                                "-D__TIMESTAMP__=\"redacted\"",
+                                "-D__TIME__=\"redacted\"",
+                            ],
+                        ),
+                        flag_group(
+                            flags = ["-fPIC"],
+                            expand_if_available = "pic",
+                        ),
+                        flag_group(
+                            flags = ["-fPIE"],
+                            expand_if_not_available = "pic",
+                        ),
+                        flag_group(
+                            flags = [
+                                "-U_FORTIFY_SOURCE",
+                                "-D_FORTIFY_SOURCE=1",
+                                "-fstack-protector",
+                                "-Wall",
+                            ] + ctx.attr.host_compiler_warnings + [
+                                "-fno-omit-frame-pointer",
+                            ],
+                        ),
+                        _no_canonical_prefixes_group(
+                            ctx.attr.extra_no_canonical_prefixes_flags,
+                        ),
+                    ],
+                ),
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [flag_group(flags = ["-DNDEBUG"])],
+                    with_features = [with_feature_set(features = ["disable-assertions"])],
+                ),
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [
+                        flag_group(
+                            flags = [
+                                "-g0",
+                                "-O2",
+                                "-ffunction-sections",
+                                "-fdata-sections",
+                            ],
+                        ),
+                    ],
+                    with_features = [with_feature_set(features = ["opt"])],
+                ),
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [flag_group(flags = ["-g"])],
+                    with_features = [with_feature_set(features = ["dbg"])],
+                ),
+            ] + _cuda_set(
+                ctx.attr.cuda_path,
+                all_compile_actions,
+            ) + [
+                flag_set(
+                    actions = all_compile_actions(),
+                    flag_groups = [
+                        _iterate_flag_group(
+                            flags = ["%{user_compile_flags}"],
+                            iterate_over = "user_compile_flags",
+                        ),
+                        _sysroot_group(),
+                        flag_group(
+                            expand_if_available = "source_file",
+                            flags = ["-c", "%{source_file}"],
+                        ),
+                        flag_group(
+                            expand_if_available = "output_assembly_file",
+                            flags = ["-S"],
+                        ),
+                        flag_group(
+                            expand_if_available = "output_preprocess_file",
+                            flags = ["-E"],
+                        ),
+                        flag_group(
+                            expand_if_available = "output_file",
+                            flags = ["-o", "%{output_file}"],
+                        ),
+                    ],
+                ),
+            ],
+        ),
+        feature(
+            name = "all_archive_flags",
+            enabled = True,
+            flag_sets = [
+                flag_set(
+                    actions = all_archive_actions(),
+                    flag_groups = [
+                        flag_group(
+                            expand_if_available = "linker_param_file",
+                            flags = ["@%{linker_param_file}"],
+                        ),
+                        flag_group(flags = ["rcsD"]),
+                        flag_group(
+                            flags = ["%{output_execpath}"],
+                            expand_if_available = "output_execpath",
+                        ),
+                        flag_group(
+                            iterate_over = "libraries_to_link",
+                            flag_groups = [
+                                flag_group(
+                                    flags = ["%{libraries_to_link.name}"],
+                                    expand_if_equal = variable_with_value(
+                                        name = "libraries_to_link.type",
+                                        value = "object_file",
+                                    ),
+                                ),
+                                flag_group(
+                                    flags = ["%{libraries_to_link.object_files}"],
+                                    iterate_over = "libraries_to_link.object_files",
+                                    expand_if_equal = variable_with_value(
+                                        name = "libraries_to_link.type",
+                                        value = "object_file_group",
+                                    ),
+                                ),
+                            ],
+                            expand_if_available = "libraries_to_link",
+                        ),
+                    ],
+                ),
+            ],
+        ),
+        feature(
+            name = "all_link_flags",
+            enabled = True,
+            flag_sets = [
+                flag_set(
+                    actions = all_shared_library_link_actions(),
+                    flag_groups = [flag_group(flags = ["-shared"])],
+                ),
+                flag_set(
+                    actions = all_link_actions(),
+                    flag_groups = [
+                        flag_group(flags = (
+                            ["-Wl,-no-as-needed"] if cpu == "local" else []
+                        ) + [
+                            "-B" + ctx.attr.linker_bin_path,
+                        ]),
+                        flag_group(
+                            flags = ["@%{linker_param_file}"],
+                            expand_if_available = "linker_param_file",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["%{linkstamp_paths}"],
+                            iterate_over = "linkstamp_paths",
+                        ),
+                        flag_group(
+                            flags = ["-o", "%{output_execpath}"],
+                            expand_if_available = "output_execpath",
+                        ),
+                        _iterate_flag_group(
+                            flags = ["-L%{library_search_directories}"],
+                            iterate_over = "library_search_directories",
+                        ),
+                        _iterate_flag_group(
+                            iterate_over = "runtime_library_search_directories",
+                            flags = [
+                                "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}",
+                            ] if cpu == "local" else [
+                                "-Wl,-rpath,@loader_path/%{runtime_library_search_directories}",
+                            ],
+                        ),
+                        _libraries_to_link_group("linux"),
+                        _iterate_flag_group(
+                            flags = ["%{user_link_flags}"],
+                            iterate_over = "user_link_flags",
+                        ),
+                        flag_group(
+                            flags = ["-Wl,--gdb-index"],
+                            expand_if_available = "is_using_fission",
+                        ),
+                        flag_group(
+                            flags = ["-Wl,-S"],
+                            expand_if_available = "strip_debug_symbols",
+                        ),
+                        flag_group(flags = ["-lstdc++"]),
+                        _no_canonical_prefixes_group(
+                            ctx.attr.extra_no_canonical_prefixes_flags,
+                        ),
+                    ],
+                ),
+                flag_set(
+                    actions = all_executable_link_actions(),
+                    flag_groups = [flag_group(flags = ["-pie"])],
+                ),
+            ] + ([
+                flag_set(
+                    actions = all_link_actions(),
+                    flag_groups = [flag_group(flags = [
+                        "-Wl,-z,relro,-z,now",
+                    ])],
+                ),
+            ] if cpu == "local" else []) + ([
+                flag_set(
+                    actions = all_link_actions(),
+                    flag_groups = [
+                        flag_group(flags = ["-Wl,--gc-sections"]),
+                        flag_group(
+                            flags = ["-Wl,--build-id=md5", "-Wl,--hash-style=gnu"],
+                        ),
+                    ],
+                ),
+            ] if cpu == "local" else []) +
+            _cuda_set(
+                ctx.attr.cuda_path,
+                all_link_actions(),
+            ) + [
+                flag_set(
+                    actions = all_link_actions(),
+                    flag_groups = [
+                        _sysroot_group(),
+                    ],
+                ),
+            ],
+        ),
+        feature(name = "opt"),
+        feature(name = "fastbuild"),
+        feature(name = "dbg"),
+        feature(name = "supports_dynamic_linker", enabled = True),
+        feature(name = "pic", enabled = True),
+        feature(name = "supports_pic", enabled = True),
+        feature(name = "has_configured_linker_path", enabled = True),
+    ]
+
+def _impl(ctx):
+    cpu = ctx.attr.cpu
+    compiler = ctx.attr.compiler
+
+    if (cpu == "local"):
+        toolchain_identifier = "local_linux"
+        target_cpu = "local"
+        target_libc = "local"
+        compiler = "compiler"
+        action_configs = _action_configs(
+            assembly_path = ctx.attr.host_compiler_path,
+            c_compiler_path = ctx.attr.host_compiler_path,
+            cc_compiler_path = ctx.attr.host_compiler_path,
+            archiver_path = ctx.attr.host_compiler_prefix + "/ar",
+            linker_path = ctx.attr.host_compiler_path,
+            strip_path = ctx.attr.host_compiler_prefix + "/strip",
+        )
+    else:
+        fail("Unreachable")
+
+    out = ctx.actions.declare_file(ctx.label.name)
+    ctx.actions.write(out, "Fake executable")
+    return [
+        cc_common.create_cc_toolchain_config_info(
+            ctx = ctx,
+            features = _features(cpu, compiler, ctx),
+            action_configs = action_configs,
+            artifact_name_patterns = [],
+            cxx_builtin_include_directories = ctx.attr.builtin_include_directories,
+            toolchain_identifier = toolchain_identifier,
+            host_system_name = "local",
+            target_system_name = "local",
+            target_cpu = target_cpu,
+            target_libc = target_libc,
+            compiler = compiler,
+            abi_version = "local",
+            abi_libc_version = "local",
+            tool_paths = _tool_paths(cpu, ctx),
+            make_variables = [],
+            builtin_sysroot = ctx.attr.builtin_sysroot,
+            cc_target_os = None,
+        ),
+        DefaultInfo(
+            executable = out,
+        ),
+    ]
+
+cc_toolchain_config = rule(
+    implementation = _impl,
+    attrs = {
+        "cpu": attr.string(mandatory = True, values = ["local"]),
+        "compiler": attr.string(values = ["clang", "unknown"], default = "unknown"),
+        "builtin_include_directories": attr.string_list(),
+        "extra_no_canonical_prefixes_flags": attr.string_list(),
+        "host_compiler_path": attr.string(),
+        "host_compiler_prefix": attr.string(),
+        "host_compiler_warnings": attr.string_list(),
+        "host_unfiltered_compile_flags": attr.string_list(),
+        "linker_bin_path": attr.string(),
+        "builtin_sysroot": attr.string(),
+        "cuda_path": attr.string(),
+    },
+    provides = [CcToolchainConfigInfo],
+    executable = True,
+)

tools/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl

+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs.
+
+SYNOPSIS:
+  crosstool_wrapper_is_not_gcc [options passed in by cc_library()
+                                or cc_binary() rule]
+
+DESCRIPTION:
+  This script is expected to be called by the cc_library() or cc_binary() bazel
+  rules. When the option "-x cuda" is present in the list of arguments passed
+  to this script, it invokes the nvcc CUDA compiler. Most arguments are passed
+  as is as a string to --compiler-options of nvcc. When "-x cuda" is not
+  present, this wrapper invokes hybrid_driver_is_not_gcc with the input
+  arguments as is.
+
+NOTES(storypku): Move this file to
+    //tools/gpus/crosstool/crosstool_wrapper_is_not_gcc ?
+
+"""
+
+from __future__ import print_function
+
+__author__ = 'keveman@google.com (Manjunath Kudlur)'
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('%{cpu_compiler}')
+GCC_HOST_COMPILER_PATH = ('%{gcc_host_compiler_path}')
+
+NVCC_PATH = '%{nvcc_path}'
+PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
+NVCC_VERSION = '%{cuda_version}'
+
+def Log(s):
+  print('//tools/gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from the argv list.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    option: The option whose value to extract, without the leading '-'.
+
+  Returns:
+    A list of values, either directly following the option,
+    (eg., -opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., -opt val1 -opt val2).
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument(option, nargs='*', action='append')
+  option = option.lstrip('-').replace('-', '_')
+  args, _ = parser.parse_known_args(argv)
+  if not args or not vars(args)[option]:
+    return []
+  else:
+    return sum(vars(args)[option], [])
+
+
+def GetHostCompilerOptions(argv):
+  """Collect the -isystem, -iquote, and --sysroot option values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be used as the --compiler-options to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-isystem', nargs='*', action='append')
+  parser.add_argument('-iquote', nargs='*', action='append')
+  parser.add_argument('--sysroot', nargs=1)
+  parser.add_argument('-g', nargs='*', action='append')
+  parser.add_argument('-fno-canonical-system-headers', action='store_true')
+  parser.add_argument('-no-canonical-prefixes', action='store_true')
+
+  args, _ = parser.parse_known_args(argv)
+
+  opts = ''
+
+  if args.isystem:
+    opts += ' -isystem ' + ' -isystem '.join(sum(args.isystem, []))
+  if args.iquote:
+    opts += ' -iquote ' + ' -iquote '.join(sum(args.iquote, []))
+  if args.g:
+    opts += ' -g' + ' -g'.join(sum(args.g, []))
+  if args.fno_canonical_system_headers:
+    opts += ' -fno-canonical-system-headers'
+  if args.no_canonical_prefixes:
+    opts += ' -no-canonical-prefixes'
+  if args.sysroot:
+    opts += ' --sysroot ' + args.sysroot[0]
+
+  return opts
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be passed directly to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, _ = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return ' '.join(['--'+a for a in options])
+  return ''
+
+def system(cmd):
+  """Invokes cmd with os.system().
+
+  Args:
+    cmd: The command.
+
+  Returns:
+    The exit code if the process exited with exit() or -signal
+    if the process was terminated by a signal.
+  """
+  retv = os.system(cmd)
+  if os.WIFEXITED(retv):
+    return os.WEXITSTATUS(retv)
+  else:
+    return -os.WTERMSIG(retv)
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling system('nvcc ' + args)
+  """
+
+  host_compiler_options = GetHostCompilerOptions(argv)
+  nvcc_compiler_options = GetNvccOptions(argv)
+  opt_option = GetOptionValue(argv, '-O')
+  m_options = GetOptionValue(argv, '-m')
+  m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']])
+  include_options = GetOptionValue(argv, '-I')
+  out_file = GetOptionValue(argv, '-o')
+  depfiles = GetOptionValue(argv, '-MF')
+  defines = GetOptionValue(argv, '-D')
+  defines = ''.join([' -D' + define for define in defines])
+  undefines = GetOptionValue(argv, '-U')
+  undefines = ''.join([' -U' + define for define in undefines])
+  std_options = GetOptionValue(argv, '-std')
+  # Supported -std flags as of CUDA 10.2. Only keep last to mimic gcc/clang.
+  # See: https://gist.github.com/ax3l/9489132#device-side-c-standard-support
+  # Updated by storypku
+  nvcc_allowed_std_options = ["c++03", "c++11", "c++14"]
+  std_options = ''.join([' -std=' + define
+      for define in std_options if define in nvcc_allowed_std_options][-1:])
+
+  # The list of source files get passed after the -c option. I don't know of
+  # any other reliable way to just get the list of source files to be compiled.
+  src_files = GetOptionValue(argv, '-c')
+
+  # Pass -w through from host to nvcc, but don't do anything fancier with
+  # warnings-related flags, since they're not necessarily the same across
+  # compilers.
+  warning_options = ' -w' if '-w' in argv else ''
+
+  if len(src_files) == 0:
+    return 1
+  if len(out_file) != 1:
+    return 1
+
+  opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0)
+         else ' -g')
+
+  includes = (' -I ' + ' -I '.join(include_options)
+              if len(include_options) > 0
+              else '')
+
+  # Unfortunately, there are other options that have -c prefix too.
+  # So allowing only those look like C/C++ files.
+  src_files = [f for f in src_files if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.cu$|\.C$', f)]
+  srcs = ' '.join(src_files)
+  out = ' -o ' + out_file[0]
+
+  nvccopts = '-D_FORCE_INLINES '
+  for capability in GetOptionValue(argv, "--cuda-gpu-arch"):
+    capability = capability[len('sm_'):]
+    nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s\" ' % (capability,
+                                                               capability)
+  for capability in GetOptionValue(argv, '--cuda-include-ptx'):
+    capability = capability[len('sm_'):]
+    nvccopts += r'-gencode=arch=compute_%s,\"code=compute_%s\" ' % (capability,
+                                                                    capability)
+  nvccopts += nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += std_options
+  nvccopts += m_options
+  nvccopts += warning_options
+
+  if depfiles:
+    # Generate the dependency file
+    depfile = depfiles[0]
+    cmd = (NVCC_PATH + ' ' + nvccopts +
+           ' --compiler-options "' + host_compiler_options + '"' +
+           ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+           ' -I .' +
+           ' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile)
+    if log: Log(cmd)
+    exit_status = system(cmd)
+    if exit_status != 0:
+      return exit_status
+
+  cmd = (NVCC_PATH + ' ' + nvccopts +
+         ' --compiler-options "' + host_compiler_options + ' -fPIC"' +
+         ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+         ' -I .' +
+         ' -x cu ' + opt + includes + ' -c ' + srcs + out)
+
+  # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'.
+  # Need to investigate and fix.
+  cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd
+  if log: Log(cmd)
+  return system(cmd)
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    args.cuda_log = True
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/gpus/cuda/BUILD.tpl

+load(":build_defs.bzl", "cuda_header_library")
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
+
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "using_nvcc",
+    values = {
+        "define": "using_cuda_nvcc=true",
+    },
+)
+
+config_setting(
+    name = "using_clang",
+    values = {
+        "define": "using_cuda_clang=true",
+    },
+)
+
+# Equivalent to using_clang && -c opt.
+config_setting(
+    name = "using_clang_opt",
+    values = {
+        "define": "using_cuda_clang=true",
+        "compilation_mode": "opt",
+    },
+)
+
+cuda_header_library(
+    name = "cuda_headers",
+    hdrs = [
+        "cuda/cuda_config.h",
+        ":cuda-include"
+    ],
+    include_prefix = "tools/gpus",
+    includes = [
+        ".",  # required to include cuda/cuda/cuda_config.h as cuda/config.h
+        "cuda/include",
+    ],
+)
+
+cc_library(
+    name = "cudart_static",
+    srcs = ["cuda/lib/%{cudart_static_lib}"],
+    linkopts = [
+        "-ldl",
+        "-lpthread",
+        %{cudart_static_linkopt}
+    ],
+)
+
+cc_library(
+    name = "cuda_driver",
+    srcs = ["cuda/lib/%{cuda_driver_lib}"],
+)
+
+cc_library(
+    name = "cudart",
+    srcs = ["cuda/lib/%{cudart_lib}"],
+    data = ["cuda/lib/%{cudart_lib}"],
+    linkstatic = 1,
+)
+
+cuda_header_library(
+    name = "cublas_headers",
+    hdrs = [":cublas-include"],
+    include_prefix = "tools/gpus/cuda/include",
+    strip_include_prefix = "cublas/include",
+    deps = [":cuda_headers"],
+    includes = ["cublas/include"],
+)
+
+cc_library(
+    name = "cublas",
+    srcs = ["cuda/lib/%{cublas_lib}"],
+    data = ["cuda/lib/%{cublas_lib}"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "cusolver",
+    srcs = ["cuda/lib/%{cusolver_lib}"],
+    data = ["cuda/lib/%{cusolver_lib}"],
+    linkopts = ["-lgomp"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "cudnn",
+    srcs = ["cuda/lib/%{cudnn_lib}"],
+    data = ["cuda/lib/%{cudnn_lib}"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "cudnn_header",
+    hdrs = [":cudnn-include"],
+    include_prefix = "tools/gpus/cudnn",
+    strip_include_prefix = "cudnn/include",
+    deps = [":cuda_headers"],
+)
+
+cc_library(
+    name = "cufft",
+    srcs = ["cuda/lib/%{cufft_lib}"],
+    data = ["cuda/lib/%{cufft_lib}"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "curand",
+    srcs = ["cuda/lib/%{curand_lib}"],
+    data = ["cuda/lib/%{curand_lib}"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "cuda",
+    deps = [
+        ":cublas",
+        ":cuda_headers",
+        ":cudart",
+        ":cudnn",
+        ":cufft",
+        ":curand",
+    ],
+)
+
+cuda_header_library(
+    name = "cupti_headers",
+    hdrs = [":cuda-extras"],
+    include_prefix="tools/gpus",
+    includes = ["cuda/extras/CUPTI/include/"],
+    deps = [":cuda_headers"],
+)
+
+cc_library(
+    name = "cupti_dsos",
+    data = ["cuda/lib/%{cupti_lib}"],
+)
+
+cc_library(
+    name = "cusparse",
+    srcs = ["cuda/lib/%{cusparse_lib}"],
+    data = ["cuda/lib/%{cusparse_lib}"],
+    linkopts = ["-lgomp"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "libdevice_root",
+    data = [":cuda-nvvm"],
+)
+
+filegroup(
+    name = "cuda_root",
+    srcs = [
+        "cuda/bin/fatbinary",
+        "cuda/bin/bin2c",
+    ],
+)
+
+bzl_library(
+    name = "build_defs_bzl",
+    srcs = ["build_defs.bzl"],
+    deps = [
+        "@bazel_skylib//lib:selects",
+    ],
+)
+
+%{copy_rules}

tools/gpus/cuda/LICENSE

+Copyright 2015 The TensorFlow Authors.  All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2015, The TensorFlow Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tools/gpus/cuda/build_defs.bzl.tpl

+# Macros for building CUDA code.
+def if_cuda(if_true, if_false = []):
+    """Shorthand for select()'ing on whether we're building with CUDA.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with CUDA enabled.  Otherwise, the select statement evaluates to if_false.
+
+    """
+    return select({
+        "@local_config_cuda//cuda:using_nvcc": if_true,
+        "@local_config_cuda//cuda:using_clang": if_true,
+        "//conditions:default": if_false,
+    })
+
+def if_cuda_clang(if_true, if_false = []):
+   """Shorthand for select()'ing on wheteher we're building with cuda-clang.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with cuda-clang.  Otherwise, the select statement evaluates to if_false.
+
+   """
+   return select({
+       "@local_config_cuda//cuda:using_clang": if_true,
+       "//conditions:default": if_false
+   })
+
+def if_cuda_clang_opt(if_true, if_false = []):
+   """Shorthand for select()'ing on wheteher we're building with cuda-clang
+   in opt mode.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with cuda-clang in opt mode. Otherwise, the select statement evaluates to
+    if_false.
+
+   """
+   return select({
+       "@local_config_cuda//cuda:using_clang_opt": if_true,
+       "//conditions:default": if_false
+   })
+
+def cuda_default_copts():
+    """Default options for all CUDA compilations."""
+    return if_cuda(
+        ["-x", "cuda", "-DAPOLLO_CUDA=1"]
+    ) + if_cuda_clang_opt(
+        # Some important CUDA optimizations are only enabled at O3.
+        ["-O3"]
+    ) + %{cuda_extra_copts}
+
+def cuda_is_configured():
+    """Returns true if CUDA was enabled during the configure process."""
+    return %{cuda_is_configured}
+
+def cuda_gpu_architectures():
+    """Returns a list of supported GPU architectures."""
+    return %{cuda_gpu_architectures}
+
+def if_cuda_is_configured(x):
+    """Tests if the CUDA was enabled during the configure process.
+
+    Unlike if_cuda(), this does not require that we are building with
+    --config=cuda. Used to allow non-CUDA code to depend on CUDA libraries.
+    """
+    if cuda_is_configured():
+      return select({"//conditions:default": x})
+    return select({"//conditions:default": []})
+
+def cuda_header_library(
+        name,
+        hdrs,
+        include_prefix = None,
+        strip_include_prefix = None,
+        deps = [],
+        **kwargs):
+    """Generates a cc_library containing both virtual and system include paths.
+
+    Generates both a header-only target with virtual includes plus the full
+    target without virtual includes. This works around the fact that bazel can't
+    mix 'includes' and 'include_prefix' in the same target."""
+
+    native.cc_library(
+        name = name + "_virtual",
+        hdrs = hdrs,
+        include_prefix = include_prefix,
+        strip_include_prefix = strip_include_prefix,
+        deps = deps,
+        visibility = ["//visibility:private"],
+    )
+
+    native.cc_library(
+        name = name,
+        textual_hdrs = hdrs,
+        deps = deps + [":%s_virtual" % name],
+        **kwargs
+    )
+
+def cuda_library(copts = [], **kwargs):
+    """Wrapper over cc_library which adds default CUDA options."""
+    native.cc_library(copts = cuda_default_copts() + copts, **kwargs)

tools/gpus/cuda/cuda_config.h.tpl

+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef CUDA_CUDA_CONFIG_H_
+#define CUDA_CUDA_CONFIG_H_
+
+#define TF_CUDA_VERSION "%{cuda_version}"
+#define TF_CUDA_LIB_VERSION "%{cuda_lib_version}"
+#define TF_CUDNN_VERSION "%{cudnn_version}"
+
+#define TF_CUDA_TOOLKIT_PATH "%{cuda_toolkit_path}"
+
+#endif  // CUDA_CUDA_CONFIG_H_

tools/gpus/cuda_configure.bzl

+"""Repository rule for CUDA autoconfiguration.
+
+`cuda_configure` depends on the following environment variables:
+
+  * `TF_NEED_CUDA`: Whether to enable building with CUDA.
+  * `GCC_HOST_COMPILER_PATH`: The GCC host compiler path
+  * `TF_CUDA_CLANG`: Whether to use clang as a cuda compiler.
+  * `CLANG_CUDA_COMPILER_PATH`: The clang compiler path that will be used for
+    both host and device code compilation if TF_CUDA_CLANG is 1.
+  * `TF_SYSROOT`: The sysroot to use when compiling.
+  * `TF_CUDA_PATHS`: The base paths to look for CUDA and cuDNN. Default is
+    `/usr/local/cuda,usr/`.
+  * `CUDA_TOOLKIT_PATH` (deprecated): The path to the CUDA toolkit. Default is
+    `/usr/local/cuda`.
+  * `TF_CUDA_VERSION`: The version of the CUDA toolkit. If this is blank, then
+    use the system default.
+  * `TF_CUDNN_VERSION`: The version of the cuDNN library.
+  * `CUDNN_INSTALL_PATH` (deprecated): The path to the cuDNN library. Default is
+    `/usr/local/cuda`.
+  * `TF_CUDA_COMPUTE_CAPABILITIES`: The CUDA compute capabilities. Default is
+    `6.0,6.1,7.0,7.2,7.5`
+  * `PYTHON_BIN_PATH`: The python binary path
+"""
+
+load(
+    "@bazel_tools//tools/cpp:lib_cc_configure.bzl",
+    "escape_string",
+    "get_env_var",
+)
+load(
+    "//tools/platform:common.bzl",
+    "config_repo_label",
+    "err_out",
+    "execute",
+    "get_bash_bin",
+    "get_cpu_value",
+    "get_host_environ",
+    "get_python_bin",
+    "raw_exec",
+    "read_dir",
+    "realpath",
+    "which",
+)
+
+_GCC_HOST_COMPILER_PATH = "GCC_HOST_COMPILER_PATH"
+_GCC_HOST_COMPILER_PREFIX = "GCC_HOST_COMPILER_PREFIX"
+_CLANG_CUDA_COMPILER_PATH = "CLANG_CUDA_COMPILER_PATH"
+_TF_SYSROOT = "TF_SYSROOT"
+_CUDA_TOOLKIT_PATH = "CUDA_TOOLKIT_PATH"
+_TF_CUDA_VERSION = "TF_CUDA_VERSION"
+_TF_CUDNN_VERSION = "TF_CUDNN_VERSION"
+_CUDNN_INSTALL_PATH = "CUDNN_INSTALL_PATH"
+_TF_CUDA_COMPUTE_CAPABILITIES = "TF_CUDA_COMPUTE_CAPABILITIES"
+_TF_CUDA_CONFIG_REPO = "TF_CUDA_CONFIG_REPO"
+_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
+
+def to_list_of_strings(elements):
+    """Convert the list of ["a", "b", "c"] into '"a", "b", "c"'.
+
+    This is to be used to put a list of strings into the bzl file templates
+    so it gets interpreted as list of strings in Starlark.
+
+    Args:
+      elements: list of string elements
+
+    Returns:
+      single string of elements wrapped in quotes separated by a comma."""
+    quoted_strings = ["\"" + element + "\"" for element in elements]
+    return ", ".join(quoted_strings)
+
+def verify_build_defines(params):
+    """Verify all variables that crosstool/BUILD.tpl expects are substituted.
+
+    Args:
+      params: dict of variables that will be passed to the BUILD.tpl template.
+    """
+    missing = []
+    for param in [
+        "cxx_builtin_include_directories",
+        "extra_no_canonical_prefixes_flags",
+        "host_compiler_path",
+        "host_compiler_prefix",
+        "host_compiler_warnings",
+        "linker_bin_path",
+        "compiler_deps",
+        "unfiltered_compile_flags",
+    ]:
+        if ("%{" + param + "}") not in params:
+            missing.append(param)
+
+    if missing:
+        auto_configure_fail(
+            "BUILD.tpl template is missing these variables: " +
+            str(missing) +
+            ".\nWe only got: " +
+            str(params) +
+            ".",
+        )
+
+# TODO(dzc): Once these functions have been factored out of Bazel's
+# cc_configure.bzl, load them from @bazel_tools instead.
+# BEGIN cc_configure common functions.
+def find_cc(repository_ctx):
+    """Find the C++ compiler."""
+    if _use_cuda_clang(repository_ctx):
+        target_cc_name = "clang"
+        cc_path_envvar = _CLANG_CUDA_COMPILER_PATH
+    else:
+        target_cc_name = "gcc"
+        cc_path_envvar = _GCC_HOST_COMPILER_PATH
+    cc_name = target_cc_name
+
+    cc_name_from_env = get_host_environ(repository_ctx, cc_path_envvar)
+    if cc_name_from_env:
+        cc_name = cc_name_from_env
+    if cc_name.startswith("/"):
+        # Absolute path, maybe we should make this supported by our which function.
+        return cc_name
+    cc = which(repository_ctx, cc_name)
+    if cc == None:
+        fail(("Cannot find {}, either correct your path or set the {}" +
+              " environment variable").format(target_cc_name, cc_path_envvar))
+    return cc
+
+_INC_DIR_MARKER_BEGIN = "#include <...>"
+
+def _cxx_inc_convert(path):
+    """Convert path returned by cc -E xc++ in a complete path."""
+    return path.strip()
+
+def _normalize_include_path(repository_ctx, path):
+    """Normalizes include paths before writing them to the crosstool.
+
+      If path points inside the 'crosstool' folder of the repository, a relative
+      path is returned.
+      If path points outside the 'crosstool' folder, an absolute path is returned.
+      """
+    path = str(repository_ctx.path(path))
+    crosstool_folder = str(repository_ctx.path(".").get_child("crosstool"))
+
+    if path.startswith(crosstool_folder):
+        # We drop the path to "$REPO/crosstool" and a trailing path separator.
+        return path[len(crosstool_folder) + 1:]
+    return path
+
+def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp, tf_sysroot):
+    """Compute the list of default C or C++ include directories."""
+    if lang_is_cpp:
+        lang = "c++"
+    else:
+        lang = "c"
+    sysroot = []
+    if tf_sysroot:
+        sysroot += ["--sysroot", tf_sysroot]
+    result = raw_exec(repository_ctx, [cc, "-E", "-x" + lang, "-", "-v"] +
+                                      sysroot)
+    stderr = err_out(result)
+    index1 = stderr.find(_INC_DIR_MARKER_BEGIN)
+    if index1 == -1:
+        return []
+    index1 = stderr.find("\n", index1)
+    if index1 == -1:
+        return []
+    index2 = stderr.rfind("\n ")
+    if index2 == -1 or index2 < index1:
+        return []
+    index2 = stderr.find("\n", index2 + 1)
+    if index2 == -1:
+        inc_dirs = stderr[index1 + 1:]
+    else:
+        inc_dirs = stderr[index1 + 1:index2].strip()
+
+    return [
+        _normalize_include_path(repository_ctx, _cxx_inc_convert(p))
+        for p in inc_dirs.split("\n")
+    ]
+
+def get_cxx_inc_directories(repository_ctx, cc, tf_sysroot):
+    """Compute the list of default C and C++ include directories."""
+
+    # For some reason `clang -xc` sometimes returns include paths that are
+    # different from the ones from `clang -xc++`. (Symlink and a dir)
+    # So we run the compiler with both `-xc` and `-xc++` and merge resulting lists
+    includes_cpp = _get_cxx_inc_directories_impl(
+        repository_ctx,
+        cc,
+        True,
+        tf_sysroot,
+    )
+    includes_c = _get_cxx_inc_directories_impl(
+        repository_ctx,
+        cc,
+        False,
+        tf_sysroot,
+    )
+
+    return includes_cpp + [
+        inc
+        for inc in includes_c
+        if inc not in includes_cpp
+    ]
+
+def auto_configure_fail(msg):
+    """Output failure message when cuda configuration fails."""
+    red = "\033[0;31m"
+    no_color = "\033[0m"
+    fail("\n%sCuda Configuration Error:%s %s\n" % (red, no_color, msg))
+
+# END cc_configure common functions (see TODO above).
+
+def _cuda_include_path(repository_ctx, cuda_config):
+    """Generates the Starlark string with cuda include directories.
+
+      Args:
+        repository_ctx: The repository context.
+        cc: The path to the gcc host compiler.
+
+      Returns:
+        A list of the gcc host compiler include directories.
+      """
+    nvcc_path = repository_ctx.path("{}/bin/nvcc".format(
+        cuda_config.cuda_toolkit_path,
+    ))
+
+    # The expected exit code of this command is non-zero. Bazel remote execution
+    # only caches commands with zero exit code. So force a zero exit code.
+    cmd = "%s -v /dev/null -o /dev/null ; [ $? -eq 1 ]" % str(nvcc_path)
+    result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
+    target_dir = ""
+    for one_line in err_out(result).splitlines():
+        if one_line.startswith("#$ _TARGET_DIR_="):
+            target_dir = (
+                cuda_config.cuda_toolkit_path + "/" + one_line.replace(
+                    "#$ _TARGET_DIR_=",
+                    "",
+                ) + "/include"
+            )
+    inc_entries = []
+    if target_dir != "":
+        inc_entries.append(realpath(repository_ctx, target_dir))
+    inc_entries.append(realpath(repository_ctx, cuda_config.cuda_toolkit_path + "/include"))
+    return inc_entries
+
+def enable_cuda(repository_ctx):
+    """Returns whether to build with CUDA support."""
+    return int(get_host_environ(repository_ctx, "TF_NEED_CUDA", False))
+
+def matches_version(environ_version, detected_version):
+    """Checks whether the user-specified version matches the detected version.
+
+      This function performs a weak matching so that if the user specifies only
+      the
+      major or major and minor versions, the versions are still considered
+      matching
+      if the version parts match. To illustrate:
+
+          environ_version  detected_version  result
+          -----------------------------------------
+          5.1.3            5.1.3             True
+          5.1              5.1.3             True
+          5                5.1               True
+          5.1.3            5.1               False
+          5.2.3            5.1.3             False
+
+      Args:
+        environ_version: The version specified by the user via environment
+          variables.
+        detected_version: The version autodetected from the CUDA installation on
+          the system.
+      Returns: True if user-specified version matches detected version and False
+        otherwise.
+    """
+    environ_version_parts = environ_version.split(".")
+    detected_version_parts = detected_version.split(".")
+    if len(detected_version_parts) < len(environ_version_parts):
+        return False
+    for i, part in enumerate(detected_version_parts):
+        if i >= len(environ_version_parts):
+            break
+        if part != environ_version_parts[i]:
+            return False
+    return True
+
+_NVCC_VERSION_PREFIX = "Cuda compilation tools, release "
+
+_DEFINE_CUDNN_MAJOR = "#define CUDNN_MAJOR"
+
+def compute_capabilities(repository_ctx):
+    """Returns a list of strings representing cuda compute capabilities.
+
+    Args:
+      repository_ctx: the repo rule's context.
+    Returns: list of cuda architectures to compile for. 'compute_xy' refers to
+      both PTX and SASS, 'sm_xy' refers to SASS only.
+    """
+    capabilities = get_host_environ(
+        repository_ctx,
+        _TF_CUDA_COMPUTE_CAPABILITIES,
+        "compute_60,compute_61",
+    ).split(",")
+
+    # Map old 'x.y' capabilities to 'compute_xy'.
+    for i, capability in enumerate(capabilities):
+        parts = capability.split(".")
+        if len(parts) != 2:
+            continue
+        capabilities[i] = "compute_%s%s" % (parts[0], parts[1])
+
+    # Make list unique
+    capabilities = dict(zip(capabilities, capabilities)).keys()
+
+    # Validate capabilities.
+    for capability in capabilities:
+        if not capability.startswith(("compute_", "sm_")):
+            auto_configure_fail("Invalid compute capability: %s" % capability)
+        for prefix in ["compute_", "sm_"]:
+            if not capability.startswith(prefix):
+                continue
+            if len(capability) == len(prefix) + 2 and capability[-2:].isdigit():
+                continue
+            auto_configure_fail("Invalid compute capability: %s" % capability)
+
+    return capabilities
+
+def lib_name(base_name, cpu_value, version = None, static = False):
+    """Constructs the platform-specific name of a library.
+
+      Args:
+        base_name: The name of the library, such as "cudart"
+        cpu_value: The name of the host operating system.
+        version: The version of the library.
+        static: True the library is static or False if it is a shared object.
+
+      Returns:
+        The platform-specific name of the library.
+      """
+    version = "" if not version else "." + version
+    if cpu_value == "Linux":
+        if static:
+            return "lib%s.a" % base_name
+        return "lib%s.so%s" % (base_name, version)
+    else:
+        auto_configure_fail("Support for cpu_value {} has been removed.".format(cpu_value))
+
+def _lib_path(lib, cpu_value, basedir, version, static):
+    file_name = lib_name(lib, cpu_value, version, static)
+    return "%s/%s" % (basedir, file_name)
+
+def _should_check_soname(version, static):
+    return version and not static
+
+def _check_cuda_lib_params(lib, cpu_value, basedir, version, static = False):
+    return (
+        _lib_path(lib, cpu_value, basedir, version, static),
+        _should_check_soname(version, static),
+    )
+
+def _check_cuda_libs(repository_ctx, script_path, libs):
+    python_bin = get_python_bin(repository_ctx)
+    contents = repository_ctx.read(script_path).splitlines()
+
+    cmd = "from os import linesep;"
+    cmd += "f = open('script.py', 'w');"
+    for line in contents:
+        cmd += "f.write('%s' + linesep);" % line
+    cmd += "f.close();"
+    cmd += "from os import system;"
+    args = " ".join(["\"" + path + "\" " + str(check) for path, check in libs])
+    cmd += "system('%s script.py %s');" % (python_bin, args)
+
+    all_paths = [path for path, _ in libs]
+    checked_paths = execute(repository_ctx, [python_bin, "-c", cmd]).stdout.splitlines()
+
+    # Filter out empty lines from splitting on '\r\n' on Windows
+    checked_paths = [path for path in checked_paths if len(path) > 0]
+    if all_paths != checked_paths:
+        auto_configure_fail("Error with installed CUDA libs. Expected '%s'. Actual '%s'." % (all_paths, checked_paths))
+
+def _find_libs(repository_ctx, check_cuda_libs_script, cuda_config):
+    """Returns the CUDA and cuDNN libraries on the system.
+
+      Also, verifies that the script actually exist.
+
+      Args:
+        repository_ctx: The repository context.
+        check_cuda_libs_script: The path to a script verifying that the cuda
+          libraries exist on the system.
+        cuda_config: The CUDA config as returned by _get_cuda_config
+
+      Returns:
+        Map of library names to structs of filename and path.
+      """
+    cpu_value = cuda_config.cpu_value
+    stub_dir = "/stubs"
+
+    check_cuda_libs_params = {
+        "cuda": _check_cuda_lib_params(
+            "cuda",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"] + stub_dir,
+            version = None,
+            static = False,
+        ),
+        "cudart": _check_cuda_lib_params(
+            "cudart",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_version,
+            static = False,
+        ),
+        "cudart_static": _check_cuda_lib_params(
+            "cudart_static",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_version,
+            static = True,
+        ),
+        "cublas": _check_cuda_lib_params(
+            "cublas",
+            cpu_value,
+            cuda_config.config["cublas_library_dir"],
+            cuda_config.cuda_lib_version,
+            static = False,
+        ),
+        "cusolver": _check_cuda_lib_params(
+            "cusolver",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_lib_version,
+            static = False,
+        ),
+        "curand": _check_cuda_lib_params(
+            "curand",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_lib_version,
+            static = False,
+        ),
+        "cufft": _check_cuda_lib_params(
+            "cufft",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_lib_version,
+            static = False,
+        ),
+        "cudnn": _check_cuda_lib_params(
+            "cudnn",
+            cpu_value,
+            cuda_config.config["cudnn_library_dir"],
+            cuda_config.cudnn_version,
+            static = False,
+        ),
+        "cupti": _check_cuda_lib_params(
+            "cupti",
+            cpu_value,
+            cuda_config.config["cupti_library_dir"],
+            cuda_config.cuda_version,
+            static = False,
+        ),
+        "cusparse": _check_cuda_lib_params(
+            "cusparse",
+            cpu_value,
+            cuda_config.config["cuda_library_dir"],
+            cuda_config.cuda_lib_version,
+            static = False,
+        ),
+    }
+
+    # Verify that the libs actually exist at their locations.
+    _check_cuda_libs(repository_ctx, check_cuda_libs_script, check_cuda_libs_params.values())
+
+    paths = {filename: v[0] for (filename, v) in check_cuda_libs_params.items()}
+    return paths
+
+def _cudart_static_linkopt(cpu_value):
+    """Returns additional platform-specific linkopts for cudart."""
+
+    # return "" if cpu_value == "Darwin" else "\"-lrt\","
+    return "\"-lrt\","
+
+def _exec_find_cuda_config(repository_ctx, script_path, cuda_libraries):
+    python_bin = get_python_bin(repository_ctx)
+
+    # If used with remote execution then repository_ctx.execute() can't
+    # access files from the source tree. A trick is to read the contents
+    # of the file in Starlark and embed them as part of the command. In
+    # this case the trick is not sufficient as the find_cuda_config.py
+    # script has more than 8192 characters. 8192 is the command length
+    # limit of cmd.exe on Windows. Thus we additionally need to compress
+    # the contents locally and decompress them as part of the execute().
+    # TODO(storypku): what if without Windows support?
+    compressed_contents = repository_ctx.read(script_path)
+    decompress_and_execute_cmd = (
+        "from zlib import decompress;" +
+        "from base64 import b64decode;" +
+        "from os import system;" +
+        "script = decompress(b64decode('%s'));" % compressed_contents +
+        "f = open('script.py', 'wb');" +
+        "f.write(script);" +
+        "f.close();" +
+        "system('\"%s\" script.py %s');" % (python_bin, " ".join(cuda_libraries))
+    )
+    return execute(repository_ctx, [python_bin, "-c", decompress_and_execute_cmd])
+
+# TODO(csigg): Only call once instead of from here, tensorrt_configure.bzl,
+# and nccl_configure.bzl.
+def find_cuda_config(repository_ctx, script_path, cuda_libraries):
+    """Returns CUDA config dictionary from running find_cuda_config.py"""
+    exec_result = _exec_find_cuda_config(repository_ctx, script_path, cuda_libraries)
+    if exec_result.return_code:
+        auto_configure_fail("Failed to run find_cuda_config.py: %s" % err_out(exec_result))
+
+    # Parse the dict from stdout.
+    return dict([tuple(x.split(": ")) for x in exec_result.stdout.splitlines()])
+
+def _get_cuda_config(repository_ctx, find_cuda_config_script):
+    """Detects and returns information about the CUDA installation on the system.
+
+      Args:
+        repository_ctx: The repository context.
+
+      Returns:
+        A struct containing the following fields:
+          cuda_toolkit_path: The CUDA toolkit installation directory.
+          cudnn_install_basedir: The cuDNN installation directory.
+          cuda_version: The version of CUDA on the system.
+          cudnn_version: The version of cuDNN on the system.
+          compute_capabilities: A list of the system's CUDA compute capabilities.
+          cpu_value: The name of the host operating system.
+      """
+    config = find_cuda_config(repository_ctx, find_cuda_config_script, ["cuda", "cudnn"])
+    cpu_value = get_cpu_value(repository_ctx)
+    toolkit_path = config["cuda_toolkit_path"]
+
+    cuda_version = config["cuda_version"].split(".")
+    cuda_major = cuda_version[0]
+    cuda_minor = cuda_version[1]
+
+    cuda_version = "{}.{}".format(cuda_major, cuda_minor)
+    cudnn_version = config["cudnn_version"]
+
+    # cuda_lib_version is for libraries like cuBLAS, cuFFT, cuSOLVER, etc.
+    # It changed from 'x.y' to just 'x' in CUDA 10.1.
+    if (int(cuda_major), int(cuda_minor)) >= (10, 1):
+        cuda_lib_version = cuda_major
+    else:
+        cuda_lib_version = cuda_version
+
+    return struct(
+        cuda_toolkit_path = toolkit_path,
+        cuda_version = cuda_version,
+        cudnn_version = cudnn_version,
+        cuda_lib_version = cuda_lib_version,
+        compute_capabilities = compute_capabilities(repository_ctx),
+        cpu_value = cpu_value,
+        config = config,
+    )
+
+def _tpl(repository_ctx, tpl, substitutions = {}, out = None):
+    if not out:
+        out = tpl.replace(":", "/")
+    repository_ctx.template(
+        out,
+        Label("//tools/%s.tpl" % tpl),
+        substitutions,
+    )
+
+def _file(repository_ctx, label):
+    repository_ctx.template(
+        label.replace(":", "/"),
+        Label("//tools/%s.tpl" % label),
+        {},
+    )
+
+_DUMMY_CROSSTOOL_BZL_FILE = """
+def error_gpu_disabled():
+  fail("ERROR: Building with --config=cuda but not configured " +
+       "to build with GPU support. Please re-run ./bootstrap.sh and enter 'Y' " +
+       "at the prompt to build with GPU support.")
+
+  native.genrule(
+      name = "error_gen_crosstool",
+      outs = ["CROSSTOOL"],
+      cmd = "echo 'Should not be run.' && exit 1",
+  )
+
+  native.filegroup(
+      name = "crosstool",
+      srcs = [":CROSSTOOL"],
+      output_licenses = ["unencumbered"],
+  )
+"""
+
+_DUMMY_CROSSTOOL_BUILD_FILE = """
+load("//crosstool:error_gpu_disabled.bzl", "error_gpu_disabled")
+
+error_gpu_disabled()
+"""
+
+def _create_dummy_repository(repository_ctx):
+    cpu_value = get_cpu_value(repository_ctx)
+
+    # Set up BUILD file for cuda/.
+    _tpl(
+        repository_ctx,
+        "cuda:build_defs.bzl",
+        {
+            "%{cuda_is_configured}": "False",
+            "%{cuda_extra_copts}": "[]",
+            "%{cuda_gpu_architectures}": "[]",
+        },
+    )
+    _tpl(
+        repository_ctx,
+        "cuda:BUILD",
+        {
+            "%{cuda_driver_lib}": lib_name("cuda", cpu_value),
+            "%{cudart_static_lib}": lib_name(
+                "cudart_static",
+                cpu_value,
+                static = True,
+            ),
+            "%{cudart_static_linkopt}": _cudart_static_linkopt(cpu_value),
+            "%{cudart_lib}": lib_name("cudart", cpu_value),
+            "%{cublas_lib}": lib_name("cublas", cpu_value),
+            "%{cusolver_lib}": lib_name("cusolver", cpu_value),
+            "%{cudnn_lib}": lib_name("cudnn", cpu_value),
+            "%{cufft_lib}": lib_name("cufft", cpu_value),
+            "%{curand_lib}": lib_name("curand", cpu_value),
+            "%{cupti_lib}": lib_name("cupti", cpu_value),
+            "%{cusparse_lib}": lib_name("cusparse", cpu_value),
+            "%{copy_rules}": """
+filegroup(name="cuda-include")
+filegroup(name="cublas-include")
+filegroup(name="cudnn-include")
+""",
+        },
+    )
+
+    # Create dummy files for the CUDA toolkit since they are still required by
+    # tensorflow/core/platform/default/build_config:cuda.
+    repository_ctx.file("cuda/cuda/include/cuda.h")
+    repository_ctx.file("cuda/cuda/include/cublas.h")
+    repository_ctx.file("cuda/cuda/include/cudnn.h")
+    repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h")
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cuda", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudart", cpu_value))
+    repository_ctx.file(
+        "cuda/cuda/lib/%s" % lib_name("cudart_static", cpu_value),
+    )
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cublas", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusolver", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudnn", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("curand", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cufft", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cupti", cpu_value))
+    repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusparse", cpu_value))
+
+    # Set up cuda_config.h
+    _tpl(
+        repository_ctx,
+        "cuda:cuda_config.h",
+        {
+            "%{cuda_version}": "",
+            "%{cuda_lib_version}": "",
+            "%{cudnn_version}": "",
+            "%{cuda_toolkit_path}": "",
+        },
+        "cuda/cuda/cuda_config.h",
+    )
+
+    # If cuda_configure is not configured to build with GPU support, and the user
+    # attempts to build with --config=cuda, add a dummy build rule to intercept
+    # this and fail with an actionable error message.
+    repository_ctx.file(
+        "crosstool/error_gpu_disabled.bzl",
+        _DUMMY_CROSSTOOL_BZL_FILE,
+    )
+    repository_ctx.file("crosstool/BUILD", _DUMMY_CROSSTOOL_BUILD_FILE)
+
+def _norm_path(path):
+    """Returns a path with '/' and remove the trailing slash."""
+    path = path.replace("\\", "/")
+    if path[-1] == "/":
+        path = path[:-1]
+    return path
+
+def make_copy_files_rule(repository_ctx, name, srcs, outs):
+    """Returns a rule to copy a set of files."""
+    cmds = []
+
+    # Copy files.
+    for src, out in zip(srcs, outs):
+        cmds.append('cp -f "%s" "$(location %s)"' % (src, out))
+    outs = [('        "%s",' % out) for out in outs]
+    return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""%s \""",
+)""" % (name, "\n".join(outs), " && \\\n".join(cmds))
+
+def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir, exceptions = None):
+    """Returns a rule to recursively copy a directory.
+    If exceptions is not None, it must be a list of files or directories in
+    'src_dir'; these will be excluded from copying.
+    """
+    src_dir = _norm_path(src_dir)
+    out_dir = _norm_path(out_dir)
+    outs = read_dir(repository_ctx, src_dir)
+    post_cmd = ""
+    if exceptions != None:
+        outs = [x for x in outs if not any([
+            x.startswith(src_dir + "/" + y)
+            for y in exceptions
+        ])]
+    outs = [('        "%s",' % out.replace(src_dir, out_dir)) for out in outs]
+
+    # '@D' already contains the relative path for a single file, see
+    # http://docs.bazel.build/versions/master/be/make-variables.html#predefined_genrule_variables
+    out_dir = "$(@D)/%s" % out_dir if len(outs) > 1 else "$(@D)"
+    if exceptions != None:
+        for x in exceptions:
+            post_cmd += " ; rm -fR " + out_dir + "/" + x
+    return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""cp -rLf "%s/." "%s/" %s\""",
+)""" % (name, "\n".join(outs), src_dir, out_dir, post_cmd)
+
+def _flag_enabled(repository_ctx, flag_name):
+    return get_host_environ(repository_ctx, flag_name) == "1"
+
+def _use_cuda_clang(repository_ctx):
+    return _flag_enabled(repository_ctx, "TF_CUDA_CLANG")
+
+def _tf_sysroot(repository_ctx):
+    return get_host_environ(repository_ctx, _TF_SYSROOT, "")
+
+def _compute_cuda_extra_copts(repository_ctx, compute_capabilities):
+    capability_flags = ["--no-cuda-include-ptx=all"]
+    for capability in compute_capabilities:
+        if capability.startswith("compute_"):
+            capability = capability.replace("compute_", "sm_")
+            capability_flags.append("--cuda-include-ptx=%s" % capability)
+        capability_flags.append("--cuda-gpu-arch=%s" % capability)
+
+    return str(capability_flags)
+
+    capability_flags = [
+        "--cuda-gpu-arch=sm_" + cap.replace(".", "")
+        for cap in compute_capabilities
+    ]
+    return str(capability_flags)
+
+def _tpl_path(repository_ctx, filename):
+    return repository_ctx.path(Label("//tools/gpus/%s.tpl" % filename))
+
+def _basename(repository_ctx, path_str):
+    """Returns the basename of a path of type string.
+    """
+    num_chars = len(path_str)
+    for i in range(num_chars):
+        r_i = num_chars - 1 - i
+        if path_str[r_i] == "/":
+            return path_str[r_i + 1:]
+    return path_str
+
+def _create_local_cuda_repository(repository_ctx):
+    """Creates the repository containing files set up to build with CUDA."""
+
+    # Resolve all labels before doing any real work. Resolving causes the
+    # function to be restarted with all previous state being lost. This
+    # can easily lead to a O(n^2) runtime in the number of labels.
+    # See https://github.com/tensorflow/tensorflow/commit/62bd3534525a036f07d9851b3199d68212904778
+    tpl_paths = {filename: _tpl_path(repository_ctx, filename) for filename in [
+        "cuda:build_defs.bzl",
+        "crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc",
+        "crosstool:BUILD",
+        "crosstool:cc_toolchain_config.bzl",
+        "cuda:cuda_config.h",
+    ]}
+    tpl_paths["cuda:BUILD"] = _tpl_path(repository_ctx, "cuda:BUILD")
+
+    find_cuda_config_script = repository_ctx.path(Label("//tools/gpus:find_cuda_config.py.gz.base64"))
+
+    cuda_config = _get_cuda_config(repository_ctx, find_cuda_config_script)
+
+    cuda_include_path = cuda_config.config["cuda_include_dir"]
+    cublas_include_path = cuda_config.config["cublas_include_dir"]
+    cudnn_header_dir = cuda_config.config["cudnn_include_dir"]
+    cupti_header_dir = cuda_config.config["cupti_include_dir"]
+    nvvm_libdevice_dir = cuda_config.config["nvvm_library_dir"]
+
+    # Create genrule to copy files from the installed CUDA toolkit into execroot.
+    copy_rules = [
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "cuda-include",
+            src_dir = cuda_include_path,
+            out_dir = "cuda/include",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "cuda-nvvm",
+            src_dir = nvvm_libdevice_dir,
+            out_dir = "cuda/nvvm/libdevice",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "cuda-extras",
+            src_dir = cupti_header_dir,
+            out_dir = "cuda/extras/CUPTI/include",
+        ),
+    ]
+
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cublas-include",
+        srcs = [
+            cublas_include_path + "/cublas.h",
+            cublas_include_path + "/cublas_v2.h",
+            cublas_include_path + "/cublas_api.h",
+        ],
+        outs = [
+            "cublas/include/cublas.h",
+            "cublas/include/cublas_v2.h",
+            "cublas/include/cublas_api.h",
+        ],
+    ))
+
+    check_cuda_libs_script = repository_ctx.path(Label("//tools/gpus:check_cuda_libs.py"))
+    cuda_libs = _find_libs(repository_ctx, check_cuda_libs_script, cuda_config)
+    cuda_lib_srcs = []
+    cuda_lib_outs = []
+    for path in cuda_libs.values():
+        cuda_lib_srcs.append(path)
+        cuda_lib_outs.append("cuda/lib/" + _basename(repository_ctx, path))
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cuda-lib",
+        srcs = cuda_lib_srcs,
+        outs = cuda_lib_outs,
+    ))
+
+    # copy files mentioned in tools/nccl/build_defs.bzl.tpl
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cuda-bin",
+        srcs = [
+            cuda_config.cuda_toolkit_path + "/bin/" + "crt/link.stub",
+            cuda_config.cuda_toolkit_path + "/bin/" + "nvlink",
+            cuda_config.cuda_toolkit_path + "/bin/" + "fatbinary",
+            cuda_config.cuda_toolkit_path + "/bin/" + "bin2c",
+        ],
+        outs = [
+            "cuda/bin/" + "crt/link.stub",
+            "cuda/bin/" + "nvlink",
+            "cuda/bin/" + "fatbinary",
+            "cuda/bin/" + "bin2c",
+        ],
+    ))
+
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cudnn-include",
+        srcs = [cudnn_header_dir + "/cudnn.h"],
+        outs = ["cudnn/include/cudnn.h"],
+    ))
+
+    # Set up BUILD file for cuda/
+    repository_ctx.template(
+        "cuda/build_defs.bzl",
+        tpl_paths["cuda:build_defs.bzl"],
+        {
+            "%{cuda_is_configured}": "True",
+            "%{cuda_extra_copts}": _compute_cuda_extra_copts(
+                repository_ctx,
+                cuda_config.compute_capabilities,
+            ),
+            "%{cuda_gpu_architectures}": str(cuda_config.compute_capabilities),
+        },
+    )
+
+    repository_ctx.template(
+        "cuda/BUILD",
+        tpl_paths["cuda:BUILD"],
+        {
+            "%{cuda_driver_lib}": _basename(repository_ctx, cuda_libs["cuda"]),
+            "%{cudart_static_lib}": _basename(repository_ctx, cuda_libs["cudart_static"]),
+            "%{cudart_static_linkopt}": _cudart_static_linkopt(cuda_config.cpu_value),
+            "%{cudart_lib}": _basename(repository_ctx, cuda_libs["cudart"]),
+            "%{cublas_lib}": _basename(repository_ctx, cuda_libs["cublas"]),
+            "%{cusolver_lib}": _basename(repository_ctx, cuda_libs["cusolver"]),
+            "%{cudnn_lib}": _basename(repository_ctx, cuda_libs["cudnn"]),
+            "%{cufft_lib}": _basename(repository_ctx, cuda_libs["cufft"]),
+            "%{curand_lib}": _basename(repository_ctx, cuda_libs["curand"]),
+            "%{cupti_lib}": _basename(repository_ctx, cuda_libs["cupti"]),
+            "%{cusparse_lib}": _basename(repository_ctx, cuda_libs["cusparse"]),
+            "%{copy_rules}": "\n".join(copy_rules),
+        },
+    )
+
+    is_cuda_clang = _use_cuda_clang(repository_ctx)
+    tf_sysroot = _tf_sysroot(repository_ctx)
+
+    cc = find_cc(repository_ctx)
+
+    host_compiler_includes = get_cxx_inc_directories(
+        repository_ctx,
+        cc,
+        tf_sysroot,
+    )
+    cuda_defines = {}
+    cuda_defines["%{builtin_sysroot}"] = tf_sysroot
+    cuda_defines["%{cuda_toolkit_path}"] = ""
+    cuda_defines["%{compiler}"] = "unknown"
+    if is_cuda_clang:
+        cuda_defines["%{cuda_toolkit_path}"] = cuda_config.config["cuda_toolkit_path"]
+        cuda_defines["%{compiler}"] = "clang"
+
+    host_compiler_prefix = get_host_environ(repository_ctx, _GCC_HOST_COMPILER_PREFIX)
+    if not host_compiler_prefix:
+        host_compiler_prefix = "/usr/bin"
+
+    cuda_defines["%{host_compiler_prefix}"] = host_compiler_prefix
+
+    # Note(storypku):
+    # As of today, according to https://github.com/tensorflow/tensorflow/pull/34202,
+    # Bazel issue#5634 has already been resolved.
+    # Besides, should_download_clang support was removed in this fork.
+    cuda_defines["%{linker_bin_path}"] = host_compiler_prefix
+
+    cuda_defines["%{extra_no_canonical_prefixes_flags}"] = ""
+    cuda_defines["%{unfiltered_compile_flags}"] = ""
+    if is_cuda_clang:
+        cuda_defines["%{host_compiler_path}"] = str(cc)
+        cuda_defines["%{host_compiler_warnings}"] = """
+        # Some parts of the codebase set -Werror and hit this warning, so
+        # switch it off for now.
+        "-Wno-invalid-partial-specialization"
+    """
+        cuda_defines["%{cxx_builtin_include_directories}"] = to_list_of_strings(host_compiler_includes)
+        cuda_defines["%{compiler_deps}"] = ":empty"
+        repository_ctx.file(
+            "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc",
+            "",
+        )
+    else:
+        cuda_defines["%{host_compiler_path}"] = "clang/bin/crosstool_wrapper_driver_is_not_gcc"
+        cuda_defines["%{host_compiler_warnings}"] = ""
+
+        # nvcc has the system include paths built in and will automatically
+        # search them; we cannot work around that, so we add the relevant cuda
+        # system paths to the allowed compiler specific include paths.
+        cuda_defines["%{cxx_builtin_include_directories}"] = to_list_of_strings(
+            host_compiler_includes + _cuda_include_path(
+                repository_ctx,
+                cuda_config,
+            ) + [cupti_header_dir, cudnn_header_dir],
+        )
+
+        # For gcc, do not canonicalize system header paths; some versions of gcc
+        # pick the shortest possible path for system includes when creating the
+        # .d file - given that includes that are prefixed with "../" multiple
+        # time quickly grow longer than the root of the tree, this can lead to
+        # bazel's header check failing.
+        cuda_defines["%{extra_no_canonical_prefixes_flags}"] = "\"-fno-canonical-system-headers\""
+
+        file_ext = ""
+        nvcc_path = "%s/nvcc" % (cuda_config.config["cuda_binary_dir"])
+        cuda_defines["%{compiler_deps}"] = ":crosstool_wrapper_driver_is_not_gcc"
+
+        wrapper_defines = {
+            "%{cpu_compiler}": str(cc),
+            "%{cuda_version}": cuda_config.cuda_version,
+            "%{nvcc_path}": nvcc_path,
+            "%{gcc_host_compiler_path}": str(cc),
+        }
+        repository_ctx.template(
+            "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc",
+            tpl_paths["crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+            wrapper_defines,
+        )
+
+    verify_build_defines(cuda_defines)
+
+    # Only expand template variables in the BUILD file
+    repository_ctx.template(
+        "crosstool/BUILD",
+        tpl_paths["crosstool:BUILD"],
+        cuda_defines,
+    )
+
+    # No templating of cc_toolchain_config - use attributes and templatize the
+    # BUILD file.
+    repository_ctx.template(
+        "crosstool/cc_toolchain_config.bzl",
+        tpl_paths["crosstool:cc_toolchain_config.bzl"],
+        {},
+    )
+
+    # Set up cuda_config.h
+    repository_ctx.template(
+        "cuda/cuda/cuda_config.h",
+        tpl_paths["cuda:cuda_config.h"],
+        {
+            "%{cuda_version}": cuda_config.cuda_version,
+            "%{cuda_lib_version}": cuda_config.cuda_lib_version,
+            "%{cudnn_version}": cuda_config.cudnn_version,
+            "%{cuda_toolkit_path}": cuda_config.cuda_toolkit_path,
+        },
+    )
+
+def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
+    """Creates pointers to a remotely configured repo set up to build with CUDA."""
+    _tpl(
+        repository_ctx,
+        "cuda:build_defs.bzl",
+        {
+            "%{cuda_is_configured}": "True",
+            "%{cuda_extra_copts}": _compute_cuda_extra_copts(
+                repository_ctx,
+                compute_capabilities(repository_ctx),
+            ),
+        },
+    )
+    repository_ctx.template(
+        "cuda/BUILD",
+        config_repo_label(remote_config_repo, "cuda:BUILD"),
+        {},
+    )
+    repository_ctx.template(
+        "cuda/build_defs.bzl",
+        config_repo_label(remote_config_repo, "cuda:build_defs.bzl"),
+        {},
+    )
+    repository_ctx.template(
+        "cuda/cuda/cuda_config.h",
+        config_repo_label(remote_config_repo, "cuda:cuda/cuda_config.h"),
+        {},
+    )
+
+    repository_ctx.template(
+        "crosstool/BUILD",
+        config_repo_label(remote_config_repo, "crosstool:BUILD"),
+        {},
+    )
+
+    repository_ctx.template(
+        "crosstool/cc_toolchain_config.bzl",
+        config_repo_label(remote_config_repo, "crosstool:cc_toolchain_config.bzl"),
+        {},
+    )
+
+    repository_ctx.template(
+        "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc",
+        config_repo_label(remote_config_repo, "crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc"),
+        {},
+    )
+
+def _cuda_autoconf_impl(repository_ctx):
+    """Implementation of the cuda_autoconf repository rule."""
+    if not enable_cuda(repository_ctx):
+        _create_dummy_repository(repository_ctx)
+    elif get_host_environ(repository_ctx, _TF_CUDA_CONFIG_REPO) != None:
+        has_cuda_version = get_host_environ(repository_ctx, _TF_CUDA_VERSION) != None
+        has_cudnn_version = get_host_environ(repository_ctx, _TF_CUDNN_VERSION) != None
+        if not has_cuda_version or not has_cudnn_version:
+            auto_configure_fail("%s and %s must also be set if %s is specified" %
+                                (_TF_CUDA_VERSION, _TF_CUDNN_VERSION, _TF_CUDA_CONFIG_REPO))
+        _create_remote_cuda_repository(
+            repository_ctx,
+            get_host_environ(repository_ctx, _TF_CUDA_CONFIG_REPO),
+        )
+    else:
+        _create_local_cuda_repository(repository_ctx)
+
+_ENVIRONS = [
+    _GCC_HOST_COMPILER_PATH,
+    _GCC_HOST_COMPILER_PREFIX,
+    _CLANG_CUDA_COMPILER_PATH,
+    "TF_NEED_CUDA",
+    "TF_CUDA_CLANG",
+    _CUDA_TOOLKIT_PATH,
+    _CUDNN_INSTALL_PATH,
+    _TF_CUDA_VERSION,
+    _TF_CUDNN_VERSION,
+    _TF_CUDA_COMPUTE_CAPABILITIES,
+    "NVVMIR_LIBRARY_DIR",
+    _PYTHON_BIN_PATH,
+    "TMPDIR",
+    "TF_CUDA_PATHS",
+]
+
+# Note(storypku): Uncomment the following rule iff "--experimental_repo_remote_exec"
+# is enabled in //tools/bazelrc
+#remote_cuda_configure = repository_rule(
+#    implementation = _create_local_cuda_repository,
+#    environ = _ENVIRONS,
+#    remotable = True,
+#    attrs = {
+#        "environ": attr.string_dict(),
+#    },
+#)
+
+cuda_configure = repository_rule(
+    implementation = _cuda_autoconf_impl,
+    environ = _ENVIRONS + [_TF_CUDA_CONFIG_REPO],
+)
+"""Detects and configures the local CUDA toolchain.
+
+Add the following to your WORKSPACE FILE:
+
+```python
+cuda_configure(name = "local_config_cuda")
+```
+
+Args:
+  name: A unique name for this workspace rule.
+"""

tools/gpus/find_cuda_config.py

+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# NOTE(storypku): Tailored to support Linux ONLY.
+# Origin: tensorflow/third_party/gpus/find_cuda_config.py
+# =============================================================================
+"""Prints CUDA library and header directories and versions found on the system.
+
+The script searches for CUDA library and header files on the system, inspects
+them to determine their version and prints the configuration to stdout.
+The paths to inspect and the required versions are specified through environment
+variables. If no valid configuration is found, the script prints to stderr and
+returns an error code.
+
+The list of libraries to find is specified as arguments. Supported libraries are
+CUDA (includes cuBLAS), cuDNN, NCCL, and TensorRT.
+
+The script takes a list of base directories specified by the TF_CUDA_PATHS
+environment variable as comma-separated glob list. The script looks for headers
+and library files in a hard-coded set of subdirectories from these base paths.
+If TF_CUDA_PATHS is not specified, a OS specific default is used:
+
+  Linux:   /usr/local/cuda, /usr, and paths from 'ldconfig -p'.
+
+For backwards compatibility, some libraries also use alternative base
+directories from other environment variables if they are specified. List of
+library-specific environment variables:
+
+  Library   Version env variable  Additional base directories
+  ----------------------------------------------------------------
+  CUDA      TF_CUDA_VERSION       CUDA_TOOLKIT_PATH
+  cuBLAS    TF_CUBLAS_VERSION     CUDA_TOOLKIT_PATH
+  cuDNN     TF_CUDNN_VERSION      CUDNN_INSTALL_PATH
+  NCCL      TF_NCCL_VERSION       NCCL_INSTALL_PATH, NCCL_HDR_PATH
+  TensorRT  TF_TENSORRT_VERSION   TENSORRT_INSTALL_PATH
+
+Versions environment variables can be of the form 'x' or 'x.y' to request a
+specific version, empty or unspecified to accept any version.
+
+The output of a found library is of the form:
+tf_<library>_version: x.y.z
+tf_<library>_header_dir: ...
+tf_<library>_library_dir: ...
+"""
+
+import io
+import os
+import glob
+import platform
+import re
+import subprocess
+import sys
+
+# pylint: disable=g-import-not-at-top
+try:
+    from shutil import which
+except ImportError:
+    from distutils.spawn import find_executable as which
+# pylint: enable=g-import-not-at-top
+
+
+class ConfigError(Exception):
+    pass
+
+
+def _is_linux():
+    return platform.system() == "Linux"
+
+
+def _matches_version(actual_version, required_version):
+    """Checks whether some version meets the requirements.
+
+      All elements of the required_version need to be present in the
+      actual_version.
+
+          required_version  actual_version  result
+          -----------------------------------------
+          1                 1.1             True
+          1.2               1               False
+          1.2               1.3             False
+                            1               True
+
+      Args:
+        required_version: The version specified by the user.
+        actual_version: The version detected from the CUDA installation.
+      Returns: Whether the actual version matches the required one.
+  """
+    if actual_version is None:
+        return False
+
+    # Strip spaces from the versions.
+    actual_version = actual_version.strip()
+    required_version = required_version.strip()
+    return actual_version.startswith(required_version)
+
+
+def _at_least_version(actual_version, required_version):
+    actual = [int(v) for v in actual_version.split(".")]
+    required = [int(v) for v in required_version.split(".")]
+    return actual >= required
+
+
+def _get_header_version(path, name):
+    """Returns preprocessor defines in C header file."""
+    for line in io.open(path, "r", encoding="utf-8").readlines():
+        match = re.match("#define %s +(\d+)" % name, line)
+        if match:
+            return match.group(1)
+    return ""
+
+
+def _cartesian_product(first, second):
+    """Returns all path combinations of first and second."""
+    return [os.path.join(f, s) for f in first for s in second]
+
+
+def _get_ld_config_paths():
+    """Returns all directories from 'ldconfig -p'."""
+    if not _is_linux():
+        return []
+    ldconfig_path = which("ldconfig") or "/sbin/ldconfig"
+    output = subprocess.check_output([ldconfig_path, "-p"])
+    pattern = re.compile(".* => (.*)")
+    result = set()
+    for line in output.splitlines():
+        try:
+            match = pattern.match(line.decode("ascii"))
+        except UnicodeDecodeError:
+            match = False
+        if match:
+            result.add(os.path.dirname(match.group(1)))
+    return sorted(list(result))
+
+
+def _get_default_cuda_paths(cuda_version):
+    if not cuda_version:
+        cuda_version = "*"
+    elif not "." in cuda_version:
+        cuda_version = cuda_version + ".*"
+
+    return [
+        "/usr/local/cuda-%s" % cuda_version, "/usr/local/cuda", "/usr",
+        "/usr/local/cudnn"
+    ] + _get_ld_config_paths()
+
+
+def _header_paths():
+    """Returns hard-coded set of relative paths to look for header files."""
+    return [
+        "",
+        "include",
+        "include/cuda",
+        "include/*-linux-gnu",
+        "extras/CUPTI/include",
+        "include/cuda/CUPTI",
+    ]
+
+
+def _library_paths():
+    """Returns hard-coded set of relative paths to look for library files."""
+    return [
+        "",
+        "lib64",
+        "lib",
+        "lib/*-linux-gnu",
+        "lib/x64",
+        "extras/CUPTI/*",
+    ]
+
+
+def _not_found_error(base_paths, relative_paths, filepattern):
+    base_paths = "".join(
+        ["\n        '%s'" % path for path in sorted(base_paths)])
+    relative_paths = "".join(
+        ["\n        '%s'" % path for path in relative_paths])
+    return ConfigError(
+        "Could not find any %s in any subdirectory:%s\nof:%s\n" %
+        (filepattern, relative_paths, base_paths))
+
+
+def _find_file(base_paths, relative_paths, filepattern):
+    for path in _cartesian_product(base_paths, relative_paths):
+        for file in glob.glob(os.path.join(path, filepattern)):
+            return file
+    raise _not_found_error(base_paths, relative_paths, filepattern)
+
+
+def _find_library(base_paths, library_name, required_version):
+    """Returns first valid path to the requested library."""
+    filepattern = ".".join(
+        ["lib" + library_name, "so"] + required_version.split(".")[:1]) + "*"
+    return _find_file(base_paths, _library_paths(), filepattern)
+
+
+def _find_versioned_file(base_paths, relative_paths, filepattern,
+                         required_version, get_version):
+    """Returns first valid path to a file that matches the requested version."""
+    for path in _cartesian_product(base_paths, relative_paths):
+        for file in glob.glob(os.path.join(path, filepattern)):
+            actual_version = get_version(file)
+            if _matches_version(actual_version, required_version):
+                return file, actual_version
+    raise _not_found_error(
+        base_paths, relative_paths,
+        filepattern + " matching version '%s'" % required_version)
+
+
+def _find_header(base_paths, header_name, required_version, get_version):
+    """Returns first valid path to a header that matches the requested version."""
+    return _find_versioned_file(base_paths, _header_paths(), header_name,
+                                required_version, get_version)
+
+
+def _find_cuda_config(base_paths, required_version):
+    def get_header_version(path):
+        version = int(_get_header_version(path, "CUDA_VERSION"))
+        if not version:
+            return None
+        return "%d.%d" % (version // 1000, version % 1000 // 10)
+
+    cuda_header_path, header_version = _find_header(
+        base_paths, "cuda.h", required_version, get_header_version)
+    cuda_version = header_version  # x.y, see above.
+
+    cuda_library_path = _find_library(base_paths, "cudart", cuda_version)
+
+    def get_nvcc_version(path):
+        pattern = "Cuda compilation tools, release \d+\.\d+, V(\d+\.\d+\.\d+)"
+        for line in subprocess.check_output([path, "--version"]).splitlines():
+            match = re.match(pattern, line.decode("ascii"))
+            if match:
+                return match.group(1)
+        return None
+
+    nvcc_name = "nvcc"
+    nvcc_path, nvcc_version = _find_versioned_file(base_paths, [
+        "",
+        "bin",
+    ], nvcc_name, cuda_version, get_nvcc_version)
+
+    nvvm_path = _find_file(base_paths, [
+        "nvvm/libdevice",
+        "share/cuda",
+        "lib/nvidia-cuda-toolkit/libdevice",
+    ], "libdevice*.10.bc")
+
+    cupti_header_path = _find_file(base_paths, _header_paths(), "cupti.h")
+    cupti_library_path = _find_library(base_paths, "cupti", required_version)
+
+    cuda_binary_dir = os.path.dirname(nvcc_path)
+    nvvm_library_dir = os.path.dirname(nvvm_path)
+
+    # XLA requires the toolkit path to find ptxas and libdevice.
+    # TODO(csigg): pass in both directories instead.
+    cuda_toolkit_paths = (
+        os.path.normpath(os.path.join(cuda_binary_dir, "..")),
+        os.path.normpath(os.path.join(nvvm_library_dir, "../..")),
+    )
+    if cuda_toolkit_paths[0] != cuda_toolkit_paths[1]:
+        raise ConfigError(
+            "Inconsistent CUDA toolkit path: %s vs %s" % cuda_toolkit_paths)
+
+    return {
+        "cuda_version": cuda_version,
+        "cuda_include_dir": os.path.dirname(cuda_header_path),
+        "cuda_library_dir": os.path.dirname(cuda_library_path),
+        "cuda_binary_dir": cuda_binary_dir,
+        "nvvm_library_dir": nvvm_library_dir,
+        "cupti_include_dir": os.path.dirname(cupti_header_path),
+        "cupti_library_dir": os.path.dirname(cupti_library_path),
+        "cuda_toolkit_path": cuda_toolkit_paths[0],
+    }
+
+
+def _find_cublas_config(base_paths, required_version, cuda_version):
+
+    if _at_least_version(cuda_version, "10.1"):
+
+        def get_header_version(path):
+            version = (_get_header_version(path, name)
+                       for name in ("CUBLAS_VER_MAJOR", "CUBLAS_VER_MINOR",
+                                    "CUBLAS_VER_PATCH"))
+            return ".".join(version)
+
+        header_path, header_version = _find_header(
+            base_paths, "cublas_api.h", required_version, get_header_version)
+        # cuBLAS uses the major version only.
+        cublas_version = header_version.split(".")[0]
+
+        if not _matches_version(cuda_version, cublas_version):
+            raise ConfigError(
+                "cuBLAS version %s does not match CUDA version %s" %
+                (cublas_version, cuda_version))
+
+    else:
+        # There is no version info available before CUDA 10.1, just find the file.
+        header_path = _find_file(base_paths, _header_paths(), "cublas_api.h")
+        # cuBLAS version is the same as CUDA version (x.y).
+        cublas_version = required_version
+
+    library_path = _find_library(base_paths, "cublas", cublas_version)
+
+    return {
+        "cublas_include_dir": os.path.dirname(header_path),
+        "cublas_library_dir": os.path.dirname(library_path),
+    }
+
+
+def _find_cudnn_config(base_paths, required_version):
+    def get_header_version(path):
+        version = (_get_header_version(path, name)
+                   for name in ("CUDNN_MAJOR", "CUDNN_MINOR",
+                                "CUDNN_PATCHLEVEL"))
+        return ".".join(version)
+
+    header_path, header_version = _find_header(
+        base_paths, "cudnn.h", required_version, get_header_version)
+    cudnn_version = header_version.split(".")[0]
+
+    library_path = _find_library(base_paths, "cudnn", cudnn_version)
+
+    return {
+        "cudnn_version": cudnn_version,
+        "cudnn_include_dir": os.path.dirname(header_path),
+        "cudnn_library_dir": os.path.dirname(library_path),
+    }
+
+
+def _find_nccl_config(base_paths, required_version):
+    def get_header_version(path):
+        version = (_get_header_version(path, name)
+                   for name in ("NCCL_MAJOR", "NCCL_MINOR", "NCCL_PATCH"))
+        return ".".join(version)
+
+    header_path, header_version = _find_header(
+        base_paths, "nccl.h", required_version, get_header_version)
+    nccl_version = header_version.split(".")[0]
+
+    library_path = _find_library(base_paths, "nccl", nccl_version)
+
+    return {
+        "nccl_version": nccl_version,
+        "nccl_include_dir": os.path.dirname(header_path),
+        "nccl_library_dir": os.path.dirname(library_path),
+    }
+
+
+def _find_tensorrt_config(base_paths, required_version):
+    def get_header_version(path):
+        version = (_get_header_version(path, name)
+                   for name in ("NV_TENSORRT_MAJOR", "NV_TENSORRT_MINOR",
+                                "NV_TENSORRT_PATCH"))
+        # `version` is a generator object, so we convert it to a list before using
+        # it (muitiple times below).
+        version = list(version)
+        if not all(version):
+            return None  # Versions not found, make _matches_version returns False.
+        return ".".join(version)
+
+    try:
+        header_path, header_version = _find_header(
+            base_paths, "NvInfer.h", required_version, get_header_version)
+    except ConfigError:
+        # TensorRT 6 moved the version information to NvInferVersion.h.
+        header_path, header_version = _find_header(
+            base_paths, "NvInferVersion.h", required_version,
+            get_header_version)
+
+    tensorrt_version = header_version.split(".")[0]
+    library_path = _find_library(base_paths, "nvinfer", tensorrt_version)
+
+    return {
+        "tensorrt_version": tensorrt_version,
+        "tensorrt_include_dir": os.path.dirname(header_path),
+        "tensorrt_library_dir": os.path.dirname(library_path),
+    }
+
+
+def _list_from_env(env_name, default=[]):
+    """Returns comma-separated list from environment variable."""
+    if env_name in os.environ:
+        return os.environ[env_name].split(",")
+    return default
+
+
+def _get_legacy_path(env_name, default=[]):
+    """Returns a path specified by a legacy environment variable.
+
+  CUDNN_INSTALL_PATH, NCCL_INSTALL_PATH, TENSORRT_INSTALL_PATH set to
+  '/usr/lib/x86_64-linux-gnu' would previously find both library and header
+  paths. Detect those and return '/usr', otherwise forward to _list_from_env().
+  """
+    if env_name in os.environ:
+        match = re.match("^(/[^/ ]*)+/lib/\w+-linux-gnu/?$",
+                         os.environ[env_name])
+        if match:
+            return [match.group(1)]
+    return _list_from_env(env_name, default)
+
+
+def _normalize_path(path):
+    """Returns normalized path, with forward slashes on Windows."""
+    return os.path.realpath(path)
+
+
+def find_cuda_config():
+    """Returns a dictionary of CUDA library and header file paths."""
+    libraries = [argv.lower() for argv in sys.argv[1:]]
+    cuda_version = os.environ.get("TF_CUDA_VERSION", "")
+    base_paths = _list_from_env("TF_CUDA_PATHS",
+                                _get_default_cuda_paths(cuda_version))
+    base_paths = [path for path in base_paths if os.path.exists(path)]
+
+    result = {}
+    if "cuda" in libraries:
+        cuda_paths = _list_from_env("CUDA_TOOLKIT_PATH", base_paths)
+        result.update(_find_cuda_config(cuda_paths, cuda_version))
+
+        cuda_version = result["cuda_version"]
+        cublas_paths = base_paths
+        if tuple(int(v) for v in cuda_version.split(".")) < (10, 1):
+            # Before CUDA 10.1, cuBLAS was in the same directory as the toolkit.
+            cublas_paths = cuda_paths
+        cublas_version = os.environ.get("TF_CUBLAS_VERSION", "")
+        result.update(
+            _find_cublas_config(cublas_paths, cublas_version, cuda_version))
+
+    if "cudnn" in libraries:
+        cudnn_paths = _get_legacy_path("CUDNN_INSTALL_PATH", base_paths)
+        cudnn_version = os.environ.get("TF_CUDNN_VERSION", "")
+        result.update(_find_cudnn_config(cudnn_paths, cudnn_version))
+
+    if "nccl" in libraries:
+        nccl_paths = _get_legacy_path("NCCL_INSTALL_PATH", base_paths)
+        nccl_version = os.environ.get("TF_NCCL_VERSION", "")
+        result.update(_find_nccl_config(nccl_paths, nccl_version))
+
+    if "tensorrt" in libraries:
+        tensorrt_paths = _get_legacy_path("TENSORRT_INSTALL_PATH", base_paths)
+        tensorrt_version = os.environ.get("TF_TENSORRT_VERSION", "")
+        result.update(_find_tensorrt_config(tensorrt_paths, tensorrt_version))
+
+    for k, v in result.items():
+        if k.endswith("_dir") or k.endswith("_path"):
+            result[k] = _normalize_path(v)
+
+    return result
+
+
+def main():
+    try:
+        for key, value in sorted(find_cuda_config().items()):
+            print("{}: {}".format(key, value))
+    except ConfigError as e:
+        sys.stderr.write(str(e))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/gpus/find_cuda_config.py.gz.base64

+eJzNPGtz47a13/UrUG53TO7K1Dq3k+lV63Qc7+7Et66dsbWbm/G6KkRCEmOKVPmQrWby33vOAUACIKW1nc1MOBOvCOIcnPcDBPOCnebrbZEslhX76s3R/7LJUrCJyMq8eJ/m9+ykrpZ5UYbsJE3ZFU4r2ZUoRbERcTh4MXjBzpMIpouY1VksClYB/MmaR/CPejJkH0VRJnnGvgrfMB8neOqRF/wFMGzzmq34lmV5xepSAIqkZPMkFUw8RGJdsSRjUb5apwnPIsHuk2pJyygkQAb7UaHIZxWH2Rzmr+Fubs5jvCKC8VpW1Xo8Gt3f34eciA3zYjFK5cRydH52+u7i+t0hEEwgH7JUlCUrxL/rpABWZ1vG10BPxGdAZcrvWV4wvigEPKtypPe+SKokWwxZmc+re14IwBInZVUks7qyhKWpA57NCSAunjHv5JqdXXvs25Prs+sh4PjhbPLd5YcJ++Hk6urkYnL27ppdXrHTy4u3Z5Ozywu4e89OLn5kfz+7eDtkAkQFy4iHdYH0A5EJipFUx66FsAiY55Kgci2iZJ5EwFe2qPlCsEW+EUUG7LC1KFZJicosgbwYsKTJKql4RSMdpnCZ4y96DQDjxeXknV9WebFd39XBmE14kuaFFH1Zr9d5UQEBWf3ALi/Of0QaLsHAk2zMKjLsORj2CIysiKdrXlTb0WJdl6N5ksXTqI75NMqzebII19svTf3A87zviyQDHzr98PYEZDcreLFFSbKl4Ci8GOwrAtYSQQJmG+k64A85SBetglS0LSuxCgcD9NYyKhJwklLwAgy5JD3uQo9eVdpYhmCuqPKqHMDgCoUYiwr1nJF9JIUmghCtJf0IL+VUF6R9En4V53UVElVrXi1L6QyEnYARqnGihjXwDm10qMRlkdeLJRPZJinybCWyarDhRYKuBnHobA5xgm14msQOAYkS0lAyJ6WiySXiRFGQ1RaiqguyYAZDIK4oj4WSZgo+iIFDCg/VALBoG4i/pZIj2YsaqQOirqXVwXgLhj5PavCTLErrGIai+tvzk+tgCD/eXlwM2cXp6fmQBCMj7tXEVmnF7xBRQ9OMg5uaFtLSAxEJuZ68n+Ka0+9PJt9dDwwRMi1CpByC6YoflgKsnyPRizSf0SIhM1ZP8/xOWpM0nnKAlGqjkpZEsXbJi/gQRRiDERKhZT0zyZwX+QrJA+qJBbKNcACqtOhFCWMOaLgC2bDL6zYgxWLO67TCeZAn4vFgwKSjjyGkj+qyGKV5xNMRevGQBqR0pS0SFQdpLK2GHa4PQNrvgb0Zj+4gRMckGJibzJI0qbYYu1fC1Gha5pSgeAr+kcHEjWRo0GE2l3G3R/4gM8pKW9vsQ2CElDxQAj5suO7FoniXumBNhoW5rabZSRwn6Bo87ZgOQB/+ygtQkH3TpRX58d3VNeQhOUjPp5PLy/O/n01IxwAjnaCBwRsLqh8G/MVY5+LCXkgOnV1cT07OzzUQeldDHN44xNGQCSMdcvrd2yuNQrsloZhASXB5dTUx0DRD1tKDjzqy9RtABHFnJnR1Ag4GZvlwgNn54CHcHmDAwSApwB74oDEDFS4hqa/W1RZn15kRNXPGIyqWeLbVc1U0gZi8rskvuUoj2onBkQwqxoNqPv2revbNVCEZMyAq/I/9TEaEKRjUmIVhaD9UP9qnkPYGA6g8MDEnuf6Vl/oXxh/9e53yConR9xBF1S8IKusij6CQaUa2JdYD620KQX6M5RMK+HhxKJ8fQjA55NVhla8HVbEdD1Dr5J7lsq6SlCk098skWg5UrXlGY+8wLRgAWJkhSBmWa36faUiqGcSDiOpKx1aJrCVKZDtpGgyilENZdkoBiZb03xEVIPZArr6GCTARIh+bJiWIFqKdr57JNNZILJT53A+gaMEKG2Z6GnTFK6wNtE59HlU1T6eNTemkrEfUCqC406WI7pAtQSGNQqKuB1ZCqFJAwct0OBhIB8OWQaRyUNuZuxDLVM0MDoFFKnpKQuWJQmJT2uCW/Du4nMk4o4R8YYA8Jbjp64i511Foj02KWpgA4VcugHP/HlLJZyDC/9kL0b3cNYgorYliUY4baFduY8r6WmidmgIyXhE2wLaIbVCsGyMsKHTClwkCKsCKpylVaRrRlSzBxuwHZVg4W+JuzUsarV015plAHBhSEA8kU0fpENMuYJLJLrmJlCCNQu8DfdYaWOWRUZ40Fakk0sF77Joi9mprPxj0iRRmu0POfKKpgxHakRL7W7/jkNqTeTVNBS+rp7qyku0xu4Gw5G8CKuw2VMA5RECDWPle6AW3Fmt9oF0eO8AGn+ybViqanYWodDLRDGG1NmQZX4k2CilzwRChcgCQAAigR6Ei9NRscEJtHEhnim0MzEjyMF8Ljd0rPEikGRSt0NUee3U1P/yzF4QFIEGAUodYvMgMSaEh/fS9F3Jl9rJkr/1P8evAYy+J4CEtFzSgYJ0EMrY8VwmFnoQL6HbW/pFlFl4TtyMwCFEmPJsC23EdVf48KcoKSlMBZWzcFRD4GZW7WMrOkkw15hB8CY6qYQnaCEmteZOXIQKGP+VJ5s9hBannOQpPAuMtSVtiuDV1mMaqbZ5Sse33U9apk+1y3HBq7AM6Cc8kV9qXBqdVQUeUfX1PD3sB1kneqARRjJpBglRF0bFRVoQR5rqpfOLfWLjBZA7X3m2gknKFDYC0CewZwOjA6F+x42+YH74KPK1NzD64hKiU35sWKdeRLtMxuqZeca1Qra1MEeHCWGDz5Xu8jJLEC1rzUyXNhyzBCW9pmlHZuKjtHLPLdpGnkMexrw0GlIq279v2HFgWXVJ37GOP6UsUQWCaj+rs5P6LtCD6accwZRfmk5Y6cxSY8V5JPYtUQUFQoo3ExwBbt68BFJBZztIAek7befiyxGBgIhh2JnlqyBvuwpNlkvpbWL3fv7T0VPDc5XXd3rwQqexcmx0abPSNPl+29p340JJqkq32NnqGFKfd8VeH5NSHi6w2H4uHquDl6PTD95Oz0WfQylnqaROHdN/xRWRh7XM8UhgA8/WfnAHndhf3+OjBBrYE8splFkx6Ss3clLawfGzwJefDhi19jzyouKGk0s5GV/Fk0G+WvvE+Zfr3wcvyAC2aAiwKhn4kjUu3mIJb7fLm6s/Gb6O5tcKJ2TG1AjvN6zQmX6f9OmyEX8otKvhlbEltxy/LT1k+p39g7QaDbwiqK0WD08b7qAFEqCfK32S0J8nvRmakCMrO+JYEcGAHHeIf30rjMnOZywe9pQjOkPLlSSmeb1yWWJQHWcDaRWWxtLvr1C4rCw+520vyAhfV7YAo2w3XbVvzteSg6YVd20OnhLBqk+KVuYfBdk9FezM+ug0wG7yyYsEOG3Cj0R5BqaXE0yxpaCnSulwmhgxTyJOEzKVpVUtedfowKXgtILPY/j2YdKdtM3gnBw+s6cnzd0d2ONLQIWGfYzV49ii9lY5h2mCGUi/4Vk7zqmPpzuaRzE1meUshqojo98pnWY8qJZ5gP5Y77XEJp+Kxid+7Q/J517DkZLwLdKy31xoQcEc3a9hLa5bYS+9ufz1zN92s6VU126lgDRHi9ofbL3kv4/BljMbhaxJGI3b05s2bYUPTS7qX44Gsd0kGhsAbabd8WEbVa88eYgmX3i7TslEG7cLtIs6q7AVuSmMXLBif5RsRGuSagbehry8ZEV1F5Q2t1RTnWp3ZJop2KdPIM6eAgclWUL8OzVPpywJfvXyKX38K4c+QffT1b/oTeFb0083hzq5UN6OHiiboSXe0kHh19i6a6mZ/56gsracBNGyqZwPDtUIaJBGie6Kg8MZrh9VmjyHkRmV7YsCO+hu6fF0mD9tVh0435mo10FRuVrbN7FsWZ4/AqGKxSSKrTymhzeg2P1jfZ5skTvghtYloHndJ1UEBhHvN2Kvw6E04i7zGGddVYnrjbko7QdIjYHDCwED1FE+B+T0ObIYJ3G6Sb30Al7s50Cg7aIVtvCfqhVD6CPSm7f+fn2gCZDJRQmzSDpX96+qBy7MTjRxDhWBy+fbSj8pksQjG9GYFPW2WA7C5MYXb1SC9sOVMrdM0NG2Y00RneYFvjpd2reKIBeQYQh0ZDB8J7gqJ4EcGikDvinSpvHlzy/5w3Pfg6NbYTKOqpLebIqs9yyD/lQnII6vkXr4p8jG2V5uSGVse1lKBtWfyc+sMpj96Y9s9nVmq60f2YaZrI256ClxwQ3q7wE0n6MC3ytN0Guq0o4GzVkd5JmZ0vs9x5vh60EHwOd5cD+8wZypLs+cakYT5xSmMZikvH1MaOZl1PNAG232N4eyYQeA78jQAXo8rr/Bq88ie+opeL+yqFjERU76C8OB77RmF6T9O/u/yyqPqrB07u8Cxz1aeJHcD7vuTyel3bs7V1ZpuW+0wi9dzijG8nHBOGuTr5IllGV4v9BGOulRxeMV/yttTYnmWbtv3hWqpXaWc2V+/uW3Z1K8A3NbMNhMbubu3sT+4kT4UJ00FXLI4F/IUkiydKOi1j83dIn35NhWOzSvVibQ0Xkm+wNemhZAHnhr0STaH1mnDoYTEwwQzAXaoXqGiOwzZT3Wp9rboyAa+6uqzi6eVBYYl9OjYeKtKp+rQK3hpi8WHWjzYo3HXvKREnlJ8IEKvo+7d6YWm7Q+wO0Mrwe6PrT1R1Y2QcZb9hr3jcwKbG9TwxJQRz+j2kaFMTacIdv7u47tzM4ztD2FfopfMsqc3k6COp4SgJ/WQWSZbyHaNfZVPO0vm3PZ+6M57pgUj6K804CyK0t+1/dJJvcZ85Z20XnXXSa+/sV2ixJ5oliTk38YqETXQYq6w0ybNSVi4GrdDZ9azDJIgf6U9yoP7RfX7tsmP7RnR1jTNwcfGVxOoY8gv2L8Uef/CtMyBy0wUHLpXls9+gjYWDy6zezqfDxMrBu0a7cnSMXJVVdRlki0MlDDHX9VJlaxx1z9ZQRE0E2l+b+T1VlT0Gr9TF6qSjaepb2vC8UDcGMIlP6rTVvKNnTy5v+J3olP0MX1enw4ohI/0aOsExRepmi82Z9lcFE/0cnUIw6hDrTpQnzD+mq3yjYjNc2hUEBar5vMKtb6SW7jsLf5+NXsN+j4uLeg+jqXotbc+MrghzBNi2yZBOoE8d52dIc6d6I07sMOe2c8Kdw3080Me+tcUz0dNRbbx4T+1j6lOyRzf3HZfw7jfdJCz0xmrvkPo5lErjZ/OJJWhmt45cdU+utEQt1qPQ896Pa/otM6HiQWPJLePZIjLbT3rOCrEMMLTzxNqv/slwLDvoH/vwX06FVLlgOVAnsbBMxl//nr69Z/aMxsH7J5OGawLsUnyuky3siGjXcTuB1cDpj52YW/pbCy4d44fkMAEJSta6mAovxi5T+R3ePg1Cnq8YwmBc/L1c5rrnlz8pz+6+eeI3b4KXhN7n+5ft7yN/vbHfcmpzwAeedTxxn5VYJ0N/Zy1B+2JF4iFafIfGQnMRG5YTTNJvowcyq9FtUhL6O2W8gO4H0Br+X3nZI9200LwtF1GkdB5LdhntXES0Tc3BX1/uu9DPGUamoL2Q6NjdsOLxSaE/AsRW57DxAF6MbQtQ/x9czS+vW23qNtI26opBMfzPefLHKxJlLdah4AcNXjWl1mPqFkedYyvZ116oWUdHjAeg1FphYgHoK+U6rjVcV6dr/z5F+0RtLFJ5/waaTqH/Hbx2/nsyLNO/BjRkA5A1usYwqzffVPcLtK/EdSjMYnyxt4Wv3X3UzThLVGm71U1FG6+ez7bxGjk3ID9lflHb4bsyKnQXrBvO5tOaiPonpfqwwy5CdQcpsLtIONlTGghdGhvhbN7t6jXfs2vxAwL7irEWr1vu9qkyN1R6teYMivo8nfbFXTcjWG56c7r5qQdtuXuU/S7cvvx215J9OxEGYS6GxYGr9Q77mCVerndnHYy7Q5Gnc63h0/ze71HsGnuV7Q0Og2wwaMu0nbx2RRxu3ntLSF28NtTEPfw7H5g+Ai+3b7YprunQlYywAhxN9QfcRDepBIr6/wAyOkOaIzlByke1bF0nt4apZdHnUaPAtrdLQrOSdsbu0hXX2jJ/Lri0MUpXFb7RuSK7RCPGdXCOH3azciaD4ck+g7c937+ZQzZwgtlX+W3OINdzRqGN2PnHtOv/JI8xP/Fg/DLqvCF0Z3jBMhVFZ6IGAzwVReVM9MpfZU3nSKP06knMUqGB/8F08JPUg==
\ No newline at end of file

tools/platform/common.bzl

+# This file was adapted from third_party/remote_config/common.bzl in tensorflow.git
+"""Functions common across configure rules."""
+
+BAZEL_SH = "BAZEL_SH"
+PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
+
+def auto_config_fail(msg):
+    """Output failure message when auto configuration fails."""
+    red = "\033[0;31m"
+    no_color = "\033[0m"
+    fail("%sConfiguration Error:%s %s\n" % (red, no_color, msg))
+
+def which(repository_ctx, program_name):
+    """Returns the full path to a program on the execution platform.
+
+    Args:
+      repository_ctx: the repository_ctx
+      program_name: name of the program on the PATH
+
+    Returns:
+      The full path to a program on the execution platform.
+    """
+    result = execute(repository_ctx, ["which", program_name])
+    return result.stdout.rstrip()
+
+def get_python_bin(repository_ctx):
+    """Gets the python bin path.
+
+    Args:
+      repository_ctx: the repository_ctx
+
+    Returns:
+      The python bin path.
+    """
+    python_bin = get_host_environ(repository_ctx, PYTHON_BIN_PATH)
+    if python_bin != None:
+        return python_bin
+    python_bin_path = which(repository_ctx, "python")
+    if python_bin_path == None:
+        auto_config_fail("Cannot find python in PATH, please make sure " +
+                         "python is installed and add its directory in PATH, or --define " +
+                         "%s='/something/else'.\nPATH=%s" % (
+                             PYTHON_BIN_PATH,
+                             get_environ("PATH", ""),
+                         ))
+    return python_bin_path
+
+def get_bash_bin(repository_ctx):
+    """Gets the bash bin path.
+
+    Args:
+      repository_ctx: the repository_ctx
+
+    Returns:
+      The bash bin path.
+    """
+    bash_bin = get_host_environ(repository_ctx, BAZEL_SH)
+    if bash_bin != None:
+        return bash_bin
+    bash_bin_path = which(repository_ctx, "bash")
+    if bash_bin_path == None:
+        auto_config_fail("Cannot find bash in PATH, please make sure " +
+                         "bash is installed and add its directory in PATH, or --define " +
+                         "%s='/path/to/bash'.\nPATH=%s" % (
+                             BAZEL_SH,
+                             get_environ("PATH", ""),
+                         ))
+    return bash_bin_path
+
+def read_dir(repository_ctx, src_dir):
+    """Returns a sorted list with all files in a directory.
+
+    Finds all files inside a directory, traversing subfolders and following
+    symlinks.
+
+    Args:
+      repository_ctx: the repository_ctx
+      src_dir: the directory to traverse
+
+    Returns:
+      A sorted list with all files in a directory.
+    """
+    find_result = execute(
+        repository_ctx,
+        ["find", src_dir, "-follow", "-type", "f"],
+        empty_stdout_fine = True,
+    )
+    result = find_result.stdout
+    return sorted(result.splitlines())
+
+def get_environ(repository_ctx, name, default_value = None):
+    """Returns the value of an environment variable on the execution platform.
+
+    Args:
+      repository_ctx: the repository_ctx
+      name: the name of environment variable
+      default_value: the value to return if not set
+
+    Returns:
+      The value of the environment variable 'name' on the execution platform
+      or 'default_value' if it's not set.
+    """
+    cmd = "echo -n \"$%s\"" % name
+    result = execute(
+        repository_ctx,
+        [get_bash_bin(repository_ctx), "-c", cmd],
+        empty_stdout_fine = True,
+    )
+    if len(result.stdout) == 0:
+        return default_value
+    return result.stdout
+
+def get_host_environ(repository_ctx, name, default_value = None):
+    """Returns the value of an environment variable on the host platform.
+
+    The host platform is the machine that Bazel runs on.
+
+    Args:
+      repository_ctx: the repository_ctx
+      name: the name of environment variable
+
+    Returns:
+      The value of the environment variable 'name' on the host platform.
+    """
+    if name in repository_ctx.os.environ:
+        return repository_ctx.os.environ.get(name).strip()
+
+    if hasattr(repository_ctx.attr, "environ") and name in repository_ctx.attr.environ:
+        return repository_ctx.attr.environ.get(name).strip()
+
+    return default_value
+
+def get_cpu_value(repository_ctx):
+    """Returns the name of the host operating system.
+
+    Args:
+      repository_ctx: The repository context.
+    Returns:
+      A string containing the name of the host operating system.
+    """
+    result = raw_exec(repository_ctx, ["uname", "-s"])
+    return result.stdout.strip()
+
+def execute(
+        repository_ctx,
+        cmdline,
+        error_msg = None,
+        error_details = None,
+        empty_stdout_fine = False):
+    """Executes an arbitrary shell command.
+
+    Args:
+      repository_ctx: the repository_ctx object
+      cmdline: list of strings, the command to execute
+      error_msg: string, a summary of the error if the command fails
+      error_details: string, details about the error or steps to fix it
+      empty_stdout_fine: bool, if True, an empty stdout result is fine,
+        otherwise it's an error
+    Returns:
+      The result of repository_ctx.execute(cmdline)
+    """
+    result = raw_exec(repository_ctx, cmdline)
+    if result.stderr or not (empty_stdout_fine or result.stdout):
+        fail(
+            "\n".join([
+                error_msg.strip() if error_msg else "Repository command failed",
+                result.stderr.strip(),
+                error_details if error_details else "",
+            ]),
+        )
+    return result
+
+def raw_exec(repository_ctx, cmdline):
+    """Executes a command via repository_ctx.execute() and returns the result.
+
+    This method is useful for debugging purposes. For example, to print all
+    commands executed as well as their return code.
+
+    Args:
+      repository_ctx: the repository_ctx
+      cmdline: the list of args
+
+    Returns:
+      The 'exec_result' of repository_ctx.execute().
+    """
+    return repository_ctx.execute(cmdline)
+
+def files_exist(repository_ctx, paths, bash_bin = None):
+    """Checks which files in paths exists.
+
+    Args:
+      repository_ctx: the repository_ctx
+      paths: a list of paths
+      bash_bin: path to the bash interpreter
+
+    Returns:
+      Returns a list of Bool. True means that the path at the
+      same position in the paths list exists.
+    """
+    if bash_bin == None:
+        bash_bin = get_bash_bin(repository_ctx)
+
+    cmd_tpl = "[ -e \"%s\" ] && echo True || echo False"
+    cmds = [cmd_tpl % path for path in paths]
+    cmd = " ; ".join(cmds)
+
+    stdout = execute(repository_ctx, [bash_bin, "-c", cmd]).stdout.strip()
+    return [val == "True" for val in stdout.splitlines()]
+
+def realpath(repository_ctx, path, bash_bin = None):
+    """Returns the result of "realpath path".
+
+    Args:
+      repository_ctx: the repository_ctx
+      path: a path on the file system
+      bash_bin: path to the bash interpreter
+
+    Returns:
+      Returns the result of "realpath path"
+    """
+    if bash_bin == None:
+        bash_bin = get_bash_bin(repository_ctx)
+
+    return execute(repository_ctx, [bash_bin, "-c", "realpath \"%s\"" % path]).stdout.strip()
+
+def err_out(result):
+    """Returns stderr if set, else stdout.
+
+    This function is a workaround for a bug in RBE where stderr is returned as stdout. Instead
+    of using result.stderr use err_out(result) instead.
+
+    Args:
+      result: the exec_result.
+
+    Returns:
+      The stderr if set, else stdout
+    """
+    if len(result.stderr) == 0:
+        return result.stdout
+    return result.stderr
+
+def config_repo_label(config_repo, target):
+    """Construct a label from config_repo and target.
+
+    This function exists to ease the migration from preconfig to remote config. In preconfig
+    the TF_*_CONFIG_REPO environ variables are set to packages in the main repo while in
+    remote config they will point to remote repositories.
+
+    Args:
+      config_repo: a remote repository or package.
+      target: a target
+    Returns:
+      A label constructed from config_repo and target.
+    """
+    if config_repo.startswith("@") and not config_repo.find("//") > 0:
+        # remote config is being used.
+        return Label(config_repo + "//" + target)
+    elif target.startswith(":"):
+        return Label(config_repo + target)
+    else:
+        return Label(config_repo + "/" + target)

tools/platform/cuda_build_defs.bzl

+"""Open source build configurations for CUDA."""
+
+load("@local_config_cuda//cuda:build_defs.bzl", _if_cuda_is_configured = "if_cuda_is_configured")
+
+# We perform this indirection so that the copybara tool can distinguish this
+# macro from others provided by the same file.
+def if_cuda_is_configured(x):
+    return _if_cuda_is_configured(x)

tools/sample.bazelrc

+build --action_env PYTHON_BIN_PATH="/usr/bin/python3"
+build --action_env PYTHON_LIB_PATH="/usr/lib/python3/dist-packages"
+build --python_path="/usr/bin/python3"
+# build --config=tensorrt
+build --action_env CUDA_TOOLKIT_PATH="/usr/local/cuda-10.2"
+build --action_env TF_CUDA_COMPUTE_CAPABILITIES="6.0,6.1,7.0,7.2,7.5"
+build --action_env GCC_HOST_COMPILER_PATH="/usr/bin/x86_64-linux-gnu-gcc-7"
+build --config=cuda
+
+## The following was adapted from tensorflow/.bazelrc
+
+# This config refers to building with CUDA available. It does not necessarily
+# mean that we build CUDA op kernels.
+build:using_cuda --define=using_cuda=true
+build:using_cuda --action_env TF_NEED_CUDA=1
+build:using_cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
+
+# This config refers to building CUDA op kernels with nvcc.
+build:cuda --config=using_cuda
+build:cuda --define=using_cuda_nvcc=true
+
+# This config refers to building CUDA op kernels with clang.
+build:cuda_clang --config=using_cuda
+build:cuda_clang --define=using_cuda_clang=true
+build:cuda_clang --define=using_clang=true
+build:cuda_clang --action_env TF_CUDA_CLANG=1
+
+build:tensorrt --action_env TF_NEED_TENSORRT=1
+build:nonccl --define=no_nccl_support=true