buildman - U-Boot multi-threaded builder and summary tool

This tool handles building U-Boot to check that you have not broken it
with your patch series. It can build each individual commit and report
which boards fail on which commits, and which errors come up. It also
shows differences in image sizes due to particular commits.

Buildman aims to make full use of multi-processor machines.

Documentation and caveats are in tools/buildman/README.
Signed-off-by: NSimon Glass <sjg@chromium.org>

tools/buildman/.gitignore

+*.pyc

tools/buildman/board.py

+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+class Board:
+    """A particular board that we can build"""
+    def __init__(self, target, arch, cpu, board_name, vendor, soc, options):
+        """Create a new board type.
+
+        Args:
+            target: Target name (use make <target>_config to configure)
+            arch: Architecture name (e.g. arm)
+            cpu: Cpu name (e.g. arm1136)
+            board_name: Name of board (e.g. integrator)
+            vendor: Name of vendor (e.g. armltd)
+            soc: Name of SOC, or '' if none (e.g. mx31)
+            options: board-specific options (e.g. integratorcp:CM1136)
+        """
+        self.target = target
+        self.arch = arch
+        self.cpu = cpu
+        self.board_name = board_name
+        self.vendor = vendor
+        self.soc = soc
+        self.props = [self.target, self.arch, self.cpu, self.board_name,
+                      self.vendor, self.soc]
+        self.options = options
+        self.build_it = False
+
+
+class Boards:
+    """Manage a list of boards."""
+    def __init__(self):
+        # Use a simple list here, sinc OrderedDict requires Python 2.7
+        self._boards = []
+
+    def AddBoard(self, board):
+        """Add a new board to the list.
+
+        The board's target member must not already exist in the board list.
+
+        Args:
+            board: board to add
+        """
+        self._boards.append(board)
+
+    def ReadBoards(self, fname):
+        """Read a list of boards from a board file.
+
+        Create a board object for each and add it to our _boards list.
+
+        Args:
+            fname: Filename of boards.cfg file
+        """
+        with open(fname, 'r') as fd:
+            for line in fd:
+                if line[0] == '#':
+                    continue
+                fields = line.split()
+                if not fields:
+                    continue
+                for upto in range(len(fields)):
+                    if fields[upto] == '-':
+                        fields[upto] = ''
+                while len(fields) < 7:
+                    fields.append('')
+
+                board = Board(*fields)
+                self.AddBoard(board)
+
+
+    def GetList(self):
+        """Return a list of available boards.
+
+        Returns:
+            List of Board objects
+        """
+        return self._boards
+
+    def GetDict(self):
+        """Build a dictionary containing all the boards.
+
+        Returns:
+            Dictionary:
+                key is board.target
+                value is board
+        """
+        board_dict = {}
+        for board in self._boards:
+            board_dict[board.target] = board
+        return board_dict
+
+    def GetSelectedDict(self):
+        """Return a dictionary containing the selected boards
+
+        Returns:
+            List of Board objects that are marked selected
+        """
+        board_dict = {}
+        for board in self._boards:
+            if board.build_it:
+                board_dict[board.target] = board
+        return board_dict
+
+    def GetSelected(self):
+        """Return a list of selected boards
+
+        Returns:
+            List of Board objects that are marked selected
+        """
+        return [board for board in self._boards if board.build_it]
+
+    def GetSelectedNames(self):
+        """Return a list of selected boards
+
+        Returns:
+            List of board names that are marked selected
+        """
+        return [board.target for board in self._boards if board.build_it]
+
+    def SelectBoards(self, args):
+        """Mark boards selected based on args
+
+        Args:
+            List of strings specifying boards to include, either named, or
+            by their target, architecture, cpu, vendor or soc. If empty, all
+            boards are selected.
+
+        Returns:
+            Dictionary which holds the number of boards which were selected
+            due to each argument, arranged by argument.
+        """
+        result = {}
+        for arg in args:
+            result[arg] = 0
+        result['all'] = 0
+
+        for board in self._boards:
+            if args:
+                for arg in args:
+                    if arg in board.props:
+                        if not board.build_it:
+                            board.build_it = True
+                            result[arg] += 1
+                            result['all'] += 1
+            else:
+                board.build_it = True
+                result['all'] += 1
+
+        return result

tools/buildman/bsettings.py

+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+import ConfigParser
+import os
+
+
+def Setup(fname=''):
+    """Set up the buildman settings module by reading config files
+
+    Args:
+        config_fname:   Config filename to read ('' for default)
+    """
+    global settings
+    global config_fname
+
+    settings = ConfigParser.SafeConfigParser()
+    config_fname = fname
+    if config_fname == '':
+        config_fname = '%s/.buildman' % os.getenv('HOME')
+    if config_fname:
+        settings.read(config_fname)
+
+def GetItems(section):
+    """Get the items from a section of the config.
+
+    Args:
+        section: name of section to retrieve
+
+    Returns:
+        List of (name, value) tuples for the section
+    """
+    try:
+        return settings.items(section)
+    except ConfigParser.NoSectionError as e:
+        print e
+        print ("Warning: No tool chains - please add a [toolchain] section "
+                "to your buildman config file %s. See README for details" %
+                config_fname)
+        return []
+    except:
+        raise

tools/buildman/buildman

+buildman.py
\ No newline at end of file

tools/buildman/buildman.py

+#!/usr/bin/python
+#
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+"""See README for more information"""
+
+import multiprocessing
+from optparse import OptionParser
+import os
+import re
+import sys
+import unittest
+
+# Bring in the patman libraries
+our_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(our_path, '../patman'))
+
+# Our modules
+import board
+import builder
+import checkpatch
+import command
+import control
+import doctest
+import gitutil
+import patchstream
+import terminal
+import toolchain
+
+def RunTests():
+    import test
+
+    sys.argv = [sys.argv[0]]
+    suite = unittest.TestLoader().loadTestsFromTestCase(test.TestBuild)
+    result = unittest.TestResult()
+    suite.run(result)
+
+    # TODO: Surely we can just 'print' result?
+    print result
+    for test, err in result.errors:
+        print err
+    for test, err in result.failures:
+        print err
+
+
+parser = OptionParser()
+parser.add_option('-b', '--branch', type='string',
+       help='Branch name to build')
+parser.add_option('-B', '--bloat', dest='show_bloat',
+       action='store_true', default=False,
+       help='Show changes in function code size for each board')
+parser.add_option('-c', '--count', dest='count', type='int',
+       default=-1, help='Run build on the top n commits')
+parser.add_option('-e', '--show_errors', action='store_true',
+       default=False, help='Show errors and warnings')
+parser.add_option('-f', '--force-build', dest='force_build',
+       action='store_true', default=False,
+       help='Force build of boards even if already built')
+parser.add_option('-d', '--detail', dest='show_detail',
+       action='store_true', default=False,
+       help='Show detailed information for each board in summary')
+parser.add_option('-g', '--git', type='string',
+       help='Git repo containing branch to build', default='.')
+parser.add_option('-H', '--full-help', action='store_true', dest='full_help',
+       default=False, help='Display the README file')
+parser.add_option('-j', '--jobs', dest='jobs', type='int',
+       default=None, help='Number of jobs to run at once (passed to make)')
+parser.add_option('-k', '--keep-outputs', action='store_true',
+       default=False, help='Keep all build output files (e.g. binaries)')
+parser.add_option('--list-tool-chains', action='store_true', default=False,
+       help='List available tool chains')
+parser.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
+       default=False, help="Do a try run (describe actions, but no nothing)")
+parser.add_option('-Q', '--quick', action='store_true',
+       default=False, help='Do a rough build, with limited warning resolution')
+parser.add_option('-s', '--summary', action='store_true',
+       default=False, help='Show a build summary')
+parser.add_option('-S', '--show-sizes', action='store_true',
+       default=False, help='Show image size variation in summary')
+parser.add_option('--step', type='int',
+       default=1, help='Only build every n commits (0=just first and last)')
+parser.add_option('-t', '--test', action='store_true', dest='test',
+                  default=False, help='run tests')
+parser.add_option('-T', '--threads', type='int',
+       default=None, help='Number of builder threads to use')
+parser.add_option('-u', '--show_unknown', action='store_true',
+       default=False, help='Show boards with unknown build result')
+
+parser.usage = """buildman -b <branch> [options]
+
+Build U-Boot for all commits in a branch. Use -n to do a dry run"""
+
+(options, args) = parser.parse_args()
+
+# Run our meagre tests
+if options.test:
+    RunTests()
+elif options.full_help:
+    pager = os.getenv('PAGER')
+    if not pager:
+        pager = 'more'
+    fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
+    command.Run(pager, fname)
+
+# Build selected commits for selected boards
+else:
+    control.DoBuildman(options, args)

tools/buildman/control.py

+# Copyright (c) 2013 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+import multiprocessing
+import os
+import sys
+
+import board
+import bsettings
+from builder import Builder
+import gitutil
+import patchstream
+import terminal
+import toolchain
+
+def GetPlural(count):
+    """Returns a plural 's' if count is not 1"""
+    return 's' if count != 1 else ''
+
+def GetActionSummary(is_summary, count, selected, options):
+    """Return a string summarising the intended action.
+
+    Returns:
+        Summary string.
+    """
+    count = (count + options.step - 1) / options.step
+    str = '%s %d commit%s for %d boards' % (
+        'Summary of' if is_summary else 'Building', count, GetPlural(count),
+        len(selected))
+    str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
+            GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
+    return str
+
+def ShowActions(series, why_selected, boards_selected, builder, options):
+    """Display a list of actions that we would take, if not a dry run.
+
+    Args:
+        series: Series object
+        why_selected: Dictionary where each key is a buildman argument
+                provided by the user, and the value is the boards brought
+                in by that argument. For example, 'arm' might bring in
+                400 boards, so in this case the key would be 'arm' and
+                the value would be a list of board names.
+        boards_selected: Dict of selected boards, key is target name,
+                value is Board object
+        builder: The builder that will be used to build the commits
+        options: Command line options object
+    """
+    col = terminal.Color()
+    print 'Dry run, so not doing much. But I would do this:'
+    print
+    print GetActionSummary(False, len(series.commits), boards_selected,
+            options)
+    print 'Build directory: %s' % builder.base_dir
+    for upto in range(0, len(series.commits), options.step):
+        commit = series.commits[upto]
+        print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
+        print commit.subject
+    print
+    for arg in why_selected:
+        if arg != 'all':
+            print arg, ': %d boards' % why_selected[arg]
+    print ('Total boards to build for each commit: %d\n' %
+            why_selected['all'])
+
+def DoBuildman(options, args):
+    """The main control code for buildman
+
+    Args:
+        options: Command line options object
+        args: Command line arguments (list of strings)
+    """
+    gitutil.Setup()
+
+    bsettings.Setup()
+    options.git_dir = os.path.join(options.git, '.git')
+
+    toolchains = toolchain.Toolchains()
+    toolchains.Scan(options.list_tool_chains)
+    if options.list_tool_chains:
+        toolchains.List()
+        print
+        return
+
+    # Work out how many commits to build. We want to build everything on the
+    # branch. We also build the upstream commit as a control so we can see
+    # problems introduced by the first commit on the branch.
+    col = terminal.Color()
+    count = options.count
+    if count == -1:
+        if not options.branch:
+            str = 'Please use -b to specify a branch to build'
+            print col.Color(col.RED, str)
+            sys.exit(1)
+        count = gitutil.CountCommitsInBranch(options.git_dir, options.branch)
+        count += 1   # Build upstream commit also
+
+    if not count:
+        str = ("No commits found to process in branch '%s': "
+               "set branch's upstream or use -c flag" % options.branch)
+        print col.Color(col.RED, str)
+        sys.exit(1)
+
+    # Work out what subset of the boards we are building
+    boards = board.Boards()
+    boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
+    why_selected = boards.SelectBoards(args)
+    selected = boards.GetSelected()
+    if not len(selected):
+        print col.Color(col.RED, 'No matching boards found')
+        sys.exit(1)
+
+    # Read the metadata from the commits. First look at the upstream commit,
+    # then the ones in the branch. We would like to do something like
+    # upstream/master~..branch but that isn't possible if upstream/master is
+    # a merge commit (it will list all the commits that form part of the
+    # merge)
+    range_expr = gitutil.GetRangeInBranch(options.git_dir, options.branch)
+    upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch)
+    series = patchstream.GetMetaDataForList(upstream_commit, options.git_dir,
+            1)
+    series = patchstream.GetMetaDataForList(range_expr, options.git_dir, None,
+            series)
+
+    # By default we have one thread per CPU. But if there are not enough jobs
+    # we can have fewer threads and use a high '-j' value for make.
+    if not options.threads:
+        options.threads = min(multiprocessing.cpu_count(), len(selected))
+    if not options.jobs:
+        options.jobs = max(1, (multiprocessing.cpu_count() +
+                len(selected) - 1) / len(selected))
+
+    if not options.step:
+        options.step = len(series.commits) - 1
+
+    # Create a new builder with the selected options
+    output_dir = os.path.join('..', options.branch)
+    builder = Builder(toolchains, output_dir, options.git_dir,
+            options.threads, options.jobs, checkout=True,
+            show_unknown=options.show_unknown, step=options.step)
+    builder.force_config_on_failure = not options.quick
+
+    # For a dry run, just show our actions as a sanity check
+    if options.dry_run:
+        ShowActions(series, why_selected, selected, builder, options)
+    else:
+        builder.force_build = options.force_build
+
+        # Work out which boards to build
+        board_selected = boards.GetSelectedDict()
+
+        print GetActionSummary(options.summary, count, board_selected, options)
+
+        if options.summary:
+            # We can't show function sizes without board details at present
+            if options.show_bloat:
+                options.show_detail = True
+            builder.ShowSummary(series.commits, board_selected,
+                    options.show_errors, options.show_sizes,
+                    options.show_detail, options.show_bloat)
+        else:
+            builder.BuildBoards(series.commits, board_selected,
+                    options.show_errors, options.keep_outputs)

tools/buildman/test.py

+#
+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+import os
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+# Bring in the patman libraries
+our_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(our_path, '../patman'))
+
+import board
+import bsettings
+import builder
+import control
+import command
+import commit
+import toolchain
+
+errors = [
+    '''main.c: In function 'main_loop':
+main.c:260:6: warning: unused variable 'joe' [-Wunused-variable]
+''',
+    '''main.c: In function 'main_loop':
+main.c:295:2: error: 'fred' undeclared (first use in this function)
+main.c:295:2: note: each undeclared identifier is reported only once for each function it appears in
+make[1]: *** [main.o] Error 1
+make: *** [common/libcommon.o] Error 2
+Make failed
+''',
+    '''main.c: In function 'main_loop':
+main.c:280:6: warning: unused variable 'mary' [-Wunused-variable]
+''',
+    '''powerpc-linux-ld: warning: dot moved backwards before `.bss'
+powerpc-linux-ld: warning: dot moved backwards before `.bss'
+powerpc-linux-ld: u-boot: section .text lma 0xfffc0000 overlaps previous sections
+powerpc-linux-ld: u-boot: section .rodata lma 0xfffef3ec overlaps previous sections
+powerpc-linux-ld: u-boot: section .reloc lma 0xffffa400 overlaps previous sections
+powerpc-linux-ld: u-boot: section .data lma 0xffffcd38 overlaps previous sections
+powerpc-linux-ld: u-boot: section .u_boot_cmd lma 0xffffeb40 overlaps previous sections
+powerpc-linux-ld: u-boot: section .bootpg lma 0xfffff198 overlaps previous sections
+'''
+]
+
+
+# hash, subject, return code, list of errors/warnings
+commits = [
+    ['1234', 'upstream/master, ok', 0, []],
+    ['5678', 'Second commit, a warning', 0, errors[0:1]],
+    ['9012', 'Third commit, error', 1, errors[0:2]],
+    ['3456', 'Fourth commit, warning', 0, [errors[0], errors[2]]],
+    ['7890', 'Fifth commit, link errors', 1, [errors[0], errors[3]]],
+    ['abcd', 'Sixth commit, fixes all errors', 0, []]
+]
+
+boards = [
+    ['board0', 'arm', 'armv7', 'ARM Board 1', 'Tester', '', ''],
+    ['board1', 'arm', 'armv7', 'ARM Board 2', 'Tester', '', ''],
+    ['board2', 'powerpc', 'powerpc', 'PowerPC board 1', 'Tester', '', ''],
+    ['board3', 'powerpc', 'mpc5xx', 'PowerPC board 2', 'Tester', '', ''],
+    ['board4', 'sandbox', 'sandbox', 'Sandbox board', 'Tester', '', '']
+]
+
+class Options:
+    """Class that holds build options"""
+    pass
+
+class TestBuild(unittest.TestCase):
+    """Test buildman
+
+    TODO: Write tests for the rest of the functionality
+    """
+    def setUp(self):
+        # Set up commits to build
+        self.commits = []
+        sequence = 0
+        for commit_info in commits:
+            comm = commit.Commit(commit_info[0])
+            comm.subject = commit_info[1]
+            comm.return_code = commit_info[2]
+            comm.error_list = commit_info[3]
+            comm.sequence = sequence
+            sequence += 1
+            self.commits.append(comm)
+
+        # Set up boards to build
+        self.boards = board.Boards()
+        for brd in boards:
+            self.boards.AddBoard(board.Board(*brd))
+        self.boards.SelectBoards([])
+
+        # Set up the toolchains
+        bsettings.Setup()
+        self.toolchains = toolchain.Toolchains()
+        self.toolchains.Add('arm-linux-gcc', test=False)
+        self.toolchains.Add('sparc-linux-gcc', test=False)
+        self.toolchains.Add('powerpc-linux-gcc', test=False)
+        self.toolchains.Add('gcc', test=False)
+
+    def Make(self, commit, brd, stage, *args, **kwargs):
+        result = command.CommandResult()
+        boardnum = int(brd.target[-1])
+        result.return_code = 0
+        result.stderr = ''
+        result.stdout = ('This is the test output for board %s, commit %s' %
+                (brd.target, commit.hash))
+        if boardnum >= 1 and boardnum >= commit.sequence:
+            result.return_code = commit.return_code
+            result.stderr = ''.join(commit.error_list)
+        if stage == 'build':
+            target_dir = None
+            for arg in args:
+                if arg.startswith('O='):
+                    target_dir = arg[2:]
+
+            if not os.path.isdir(target_dir):
+                os.mkdir(target_dir)
+            #time.sleep(.2 + boardnum * .2)
+
+        result.combined = result.stdout + result.stderr
+        return result
+
+    def testBasic(self):
+        """Test basic builder operation"""
+        output_dir = tempfile.mkdtemp()
+        if not os.path.isdir(output_dir):
+            os.mkdir(output_dir)
+        build = builder.Builder(self.toolchains, output_dir, None, 1, 2,
+                                checkout=False, show_unknown=False)
+        build.do_make = self.Make
+        board_selected = self.boards.GetSelectedDict()
+
+        #build.BuildCommits(self.commits, board_selected, False)
+        build.BuildBoards(self.commits, board_selected, False, False)
+        build.ShowSummary(self.commits, board_selected, True, False,
+                          False, False)
+
+    def _testGit(self):
+        """Test basic builder operation by building a branch"""
+        base_dir = tempfile.mkdtemp()
+        if not os.path.isdir(base_dir):
+            os.mkdir(base_dir)
+        options = Options()
+        options.git = os.getcwd()
+        options.summary = False
+        options.jobs = None
+        options.dry_run = False
+        #options.git = os.path.join(base_dir, 'repo')
+        options.branch = 'test-buildman'
+        options.force_build = False
+        options.list_tool_chains = False
+        options.count = -1
+        options.git_dir = None
+        options.threads = None
+        options.show_unknown = False
+        options.quick = False
+        options.show_errors = False
+        options.keep_outputs = False
+        args = ['tegra20']
+        control.DoBuildman(options, args)
+
+if __name__ == "__main__":
+    unittest.main()

tools/buildman/toolchain.py

+# Copyright (c) 2012 The Chromium OS Authors.
+#
+# See file CREDITS for list of people who contributed to this
+# project.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307 USA
+#
+
+import glob
+import os
+
+import bsettings
+import command
+
+class Toolchain:
+    """A single toolchain
+
+    Public members:
+        gcc: Full path to C compiler
+        path: Directory path containing C compiler
+        cross: Cross compile string, e.g. 'arm-linux-'
+        arch: Architecture of toolchain as determined from the first
+                component of the filename. E.g. arm-linux-gcc becomes arm
+    """
+
+    def __init__(self, fname, test, verbose=False):
+        """Create a new toolchain object.
+
+        Args:
+            fname: Filename of the gcc component
+            test: True to run the toolchain to test it
+        """
+        self.gcc = fname
+        self.path = os.path.dirname(fname)
+        self.cross = os.path.basename(fname)[:-3]
+        pos = self.cross.find('-')
+        self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
+
+        env = self.MakeEnvironment()
+
+        # As a basic sanity check, run the C compiler with --version
+        cmd = [fname, '--version']
+        if test:
+            result = command.RunPipe([cmd], capture=True, env=env)
+            self.ok = result.return_code == 0
+            if verbose:
+                print 'Tool chain test: ',
+                if self.ok:
+                    print 'OK'
+                else:
+                    print 'BAD'
+                    print 'Command: ', cmd
+                    print result.stdout
+                    print result.stderr
+        else:
+            self.ok = True
+        self.priority = self.GetPriority(fname)
+
+    def GetPriority(self, fname):
+        """Return the priority of the toolchain.
+
+        Toolchains are ranked according to their suitability by their
+        filename prefix.
+
+        Args:
+            fname: Filename of toolchain
+        Returns:
+            Priority of toolchain, 0=highest, 20=lowest.
+        """
+        priority_list = ['-elf', '-unknown-linux-gnu', '-linux', '-elf',
+            '-none-linux-gnueabi', '-uclinux', '-none-eabi',
+            '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
+        for prio in range(len(priority_list)):
+            if priority_list[prio] in fname:
+                return prio
+        return prio
+
+    def MakeEnvironment(self):
+        """Returns an environment for using the toolchain.
+
+        Thie takes the current environment, adds CROSS_COMPILE and
+        augments PATH so that the toolchain will operate correctly.
+        """
+        env = dict(os.environ)
+        env['CROSS_COMPILE'] = self.cross
+        env['PATH'] += (':' + self.path)
+        return env
+
+
+class Toolchains:
+    """Manage a list of toolchains for building U-Boot
+
+    We select one toolchain for each architecture type
+
+    Public members:
+        toolchains: Dict of Toolchain objects, keyed by architecture name
+        paths: List of paths to check for toolchains (may contain wildcards)
+    """
+
+    def __init__(self):
+        self.toolchains = {}
+        self.paths = []
+        for name, value in bsettings.GetItems('toolchain'):
+            if '*' in value:
+                self.paths += glob.glob(value)
+            else:
+                self.paths.append(value)
+
+
+    def Add(self, fname, test=True, verbose=False):
+        """Add a toolchain to our list
+
+        We select the given toolchain as our preferred one for its
+        architecture if it is a higher priority than the others.
+
+        Args:
+            fname: Filename of toolchain's gcc driver
+            test: True to run the toolchain to test it
+        """
+        toolchain = Toolchain(fname, test, verbose)
+        add_it = toolchain.ok
+        if toolchain.arch in self.toolchains:
+            add_it = (toolchain.priority <
+                        self.toolchains[toolchain.arch].priority)
+        if add_it:
+            self.toolchains[toolchain.arch] = toolchain
+
+    def Scan(self, verbose):
+        """Scan for available toolchains and select the best for each arch.
+
+        We look for all the toolchains we can file, figure out the
+        architecture for each, and whether it works. Then we select the
+        highest priority toolchain for each arch.
+
+        Args:
+            verbose: True to print out progress information
+        """
+        if verbose: print 'Scanning for tool chains'
+        for path in self.paths:
+            if verbose: print "   - scanning path '%s'" % path
+            for subdir in ['.', 'bin', 'usr/bin']:
+                dirname = os.path.join(path, subdir)
+                if verbose: print "      - looking in '%s'" % dirname
+                for fname in glob.glob(dirname + '/*gcc'):
+                    if verbose: print "         - found '%s'" % fname
+                    self.Add(fname, True, verbose)
+
+    def List(self):
+        """List out the selected toolchains for each architecture"""
+        print 'List of available toolchains (%d):' % len(self.toolchains)
+        if len(self.toolchains):
+            for key, value in sorted(self.toolchains.iteritems()):
+                print '%-10s: %s' % (key, value.gcc)
+        else:
+            print 'None'
+
+    def Select(self, arch):
+        """Returns the toolchain for a given architecture
+
+        Args:
+            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
+
+        returns:
+            toolchain object, or None if none found
+        """
+        for name, value in bsettings.GetItems('toolchain-alias'):
+            if arch == name:
+                arch = value
+
+        if not arch in self.toolchains:
+            raise ValueError, ("No tool chain found for arch '%s'" % arch)
+        return self.toolchains[arch]