Commit c05aa036 authored by Simon Glass's avatar Simon Glass
Browse files

buildman: Convert to Python 3



Convert buildman to Python 3 and make it use that, to meet the 2020
deadline.
Signed-off-by: Simon Glass's avatarSimon Glass <sjg@chromium.org>
parent e3986d9b
# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2012 The Chromium OS Authors.
from collections import OrderedDict
import re
class Expr:
......@@ -120,7 +121,7 @@ class Boards:
Args:
fname: Filename of boards.cfg file
"""
with open(fname, 'r') as fd:
with open(fname, 'r', encoding='utf-8') as fd:
for line in fd:
if line[0] == '#':
continue
......@@ -155,7 +156,7 @@ class Boards:
key is board.target
value is board
"""
board_dict = {}
board_dict = OrderedDict()
for board in self._boards:
board_dict[board.target] = board
return board_dict
......@@ -166,7 +167,7 @@ class Boards:
Returns:
List of Board objects that are marked selected
"""
board_dict = {}
board_dict = OrderedDict()
for board in self._boards:
if board.build_it:
board_dict[board.target] = board
......@@ -259,7 +260,7 @@ class Boards:
due to each argument, arranged by argument.
List of errors found
"""
result = {}
result = OrderedDict()
warnings = []
terms = self._BuildTerms(args)
......
# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2012 The Chromium OS Authors.
import ConfigParser
import configparser
import os
import StringIO
import io
def Setup(fname=''):
......@@ -15,20 +15,20 @@ def Setup(fname=''):
global settings
global config_fname
settings = ConfigParser.SafeConfigParser()
settings = configparser.SafeConfigParser()
if fname is not None:
config_fname = fname
if config_fname == '':
config_fname = '%s/.buildman' % os.getenv('HOME')
if not os.path.exists(config_fname):
print 'No config file found ~/.buildman\nCreating one...\n'
print('No config file found ~/.buildman\nCreating one...\n')
CreateBuildmanConfigFile(config_fname)
print 'To install tool chains, please use the --fetch-arch option'
print('To install tool chains, please use the --fetch-arch option')
if config_fname:
settings.read(config_fname)
def AddFile(data):
settings.readfp(StringIO.StringIO(data))
settings.readfp(io.StringIO(data))
def GetItems(section):
"""Get the items from a section of the config.
......@@ -41,7 +41,7 @@ def GetItems(section):
"""
try:
return settings.items(section)
except ConfigParser.NoSectionError as e:
except configparser.NoSectionError as e:
return []
except:
raise
......@@ -68,10 +68,10 @@ def CreateBuildmanConfigFile(config_fname):
try:
f = open(config_fname, 'w')
except IOError:
print "Couldn't create buildman config file '%s'\n" % config_fname
print("Couldn't create buildman config file '%s'\n" % config_fname)
raise
print >>f, '''[toolchain]
print('''[toolchain]
# name = path
# e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux
......@@ -93,5 +93,5 @@ openrisc = or1k
# snapper-boards=ENABLE_AT91_TEST=1
# snapper9260=${snapper-boards} BUILD_TAG=442
# snapper9g45=${snapper-boards} BUILD_TAG=443
'''
''', file=f)
f.close();
......@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
import glob
import os
import re
import Queue
import queue
import shutil
import signal
import string
......@@ -92,11 +92,10 @@ u-boot/ source directory
"""
# Possible build outcomes
OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
# Translate a commit subject into a valid filename (and handle unicode)
trans_valid_chars = string.maketrans('/: ', '---')
trans_valid_chars = trans_valid_chars.decode('latin-1')
trans_valid_chars = str.maketrans('/: ', '---')
BASE_CONFIG_FILENAMES = [
'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
......@@ -122,8 +121,8 @@ class Config:
def __hash__(self):
val = 0
for fname in self.config:
for key, value in self.config[fname].iteritems():
print key, value
for key, value in self.config[fname].items():
print(key, value)
val = val ^ hash(key) & hash(value)
return val
......@@ -293,8 +292,8 @@ class Builder:
self._re_dtb_warning = re.compile('(.*): Warning .*')
self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
self.queue = Queue.Queue()
self.out_queue = Queue.Queue()
self.queue = queue.Queue()
self.out_queue = queue.Queue()
for i in range(self.num_threads):
t = builderthread.BuilderThread(self, i, incremental,
per_board_out_dir)
......@@ -781,7 +780,7 @@ class Builder:
config = {}
environment = {}
for board in boards_selected.itervalues():
for board in boards_selected.values():
outcome = self.GetBuildOutcome(commit_upto, board.target,
read_func_sizes, read_config,
read_environment)
......@@ -814,13 +813,13 @@ class Builder:
tconfig = Config(self.config_filenames, board.target)
for fname in self.config_filenames:
if outcome.config:
for key, value in outcome.config[fname].iteritems():
for key, value in outcome.config[fname].items():
tconfig.Add(fname, key, value)
config[board.target] = tconfig
tenvironment = Environment(board.target)
if outcome.environment:
for key, value in outcome.environment.iteritems():
for key, value in outcome.environment.items():
tenvironment.Add(key, value)
environment[board.target] = tenvironment
......@@ -1040,12 +1039,12 @@ class Builder:
# We now have a list of image size changes sorted by arch
# Print out a summary of these
for arch, target_list in arch_list.iteritems():
for arch, target_list in arch_list.items():
# Get total difference for each type
totals = {}
for result in target_list:
total = 0
for name, diff in result.iteritems():
for name, diff in result.items():
if name.startswith('_'):
continue
total += diff
......@@ -1250,7 +1249,7 @@ class Builder:
if self._show_unknown:
self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
self.col.MAGENTA)
for arch, target_list in arch_list.iteritems():
for arch, target_list in arch_list.items():
Print('%10s: %s' % (arch, target_list))
self._error_lines += 1
if better_err:
......@@ -1283,13 +1282,13 @@ class Builder:
environment_minus = {}
environment_change = {}
base = tbase.environment
for key, value in tenvironment.environment.iteritems():
for key, value in tenvironment.environment.items():
if key not in base:
environment_plus[key] = value
for key, value in base.iteritems():
for key, value in base.items():
if key not in tenvironment.environment:
environment_minus[key] = value
for key, value in base.iteritems():
for key, value in base.items():
new_value = tenvironment.environment.get(key)
if new_value and value != new_value:
desc = '%s -> %s' % (value, new_value)
......@@ -1342,15 +1341,15 @@ class Builder:
config_minus = {}
config_change = {}
base = tbase.config[name]
for key, value in tconfig.config[name].iteritems():
for key, value in tconfig.config[name].items():
if key not in base:
config_plus[key] = value
all_config_plus[key] = value
for key, value in base.iteritems():
for key, value in base.items():
if key not in tconfig.config[name]:
config_minus[key] = value
all_config_minus[key] = value
for key, value in base.iteritems():
for key, value in base.items():
new_value = tconfig.config.get(key)
if new_value and value != new_value:
desc = '%s -> %s' % (value, new_value)
......@@ -1368,7 +1367,7 @@ class Builder:
summary[target] = '\n'.join(lines)
lines_by_target = {}
for target, lines in summary.iteritems():
for target, lines in summary.items():
if lines in lines_by_target:
lines_by_target[lines].append(target)
else:
......@@ -1392,7 +1391,7 @@ class Builder:
Print('%s:' % arch)
_OutputConfigInfo(lines)
for lines, targets in lines_by_target.iteritems():
for lines, targets in lines_by_target.items():
if not lines:
continue
Print('%s :' % ' '.join(sorted(targets)))
......@@ -1463,7 +1462,7 @@ class Builder:
commits: Selected commits to build
"""
# First work out how many commits we will build
count = (self.commit_count + self._step - 1) / self._step
count = (self.commit_count + self._step - 1) // self._step
self.count = len(board_selected) * count
self.upto = self.warned = self.fail = 0
self._timestamps = collections.deque()
......@@ -1566,7 +1565,7 @@ class Builder:
self.ProcessResult(None)
# Create jobs to build all commits for each board
for brd in board_selected.itervalues():
for brd in board_selected.values():
job = builderthread.BuilderJob()
job.board = brd
job.commits = commits
......
......@@ -28,7 +28,7 @@ def Mkdir(dirname, parents = False):
except OSError as err:
if err.errno == errno.EEXIST:
if os.path.realpath('.') == os.path.realpath(dirname):
print "Cannot create the current working directory '%s'!" % dirname
print("Cannot create the current working directory '%s'!" % dirname)
sys.exit(1)
pass
else:
......@@ -291,15 +291,13 @@ class BuilderThread(threading.Thread):
outfile = os.path.join(build_dir, 'log')
with open(outfile, 'w') as fd:
if result.stdout:
# We don't want unicode characters in log files
fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
fd.write(result.stdout)
errfile = self.builder.GetErrFile(result.commit_upto,
result.brd.target)
if result.stderr:
with open(errfile, 'w') as fd:
# We don't want unicode characters in log files
fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
fd.write(result.stderr)
elif os.path.exists(errfile):
os.remove(errfile)
......@@ -314,17 +312,17 @@ class BuilderThread(threading.Thread):
else:
fd.write('%s' % result.return_code)
with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
print >>fd, 'gcc', result.toolchain.gcc
print >>fd, 'path', result.toolchain.path
print >>fd, 'cross', result.toolchain.cross
print >>fd, 'arch', result.toolchain.arch
print('gcc', result.toolchain.gcc, file=fd)
print('path', result.toolchain.path, file=fd)
print('cross', result.toolchain.cross, file=fd)
print('arch', result.toolchain.arch, file=fd)
fd.write('%s' % result.return_code)
# Write out the image and function size information and an objdump
env = result.toolchain.MakeEnvironment(self.builder.full_path)
with open(os.path.join(build_dir, 'env'), 'w') as fd:
for var in sorted(env.keys()):
print >>fd, '%s="%s"' % (var, env[var])
print('%s="%s"' % (var, env[var]), file=fd)
lines = []
for fname in ['u-boot', 'spl/u-boot-spl']:
cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
......@@ -335,7 +333,7 @@ class BuilderThread(threading.Thread):
nm = self.builder.GetFuncSizesFile(result.commit_upto,
result.brd.target, fname)
with open(nm, 'w') as fd:
print >>fd, nm_result.stdout,
print(nm_result.stdout, end=' ', file=fd)
cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
dump_result = command.RunPipe([cmd], capture=True,
......@@ -346,7 +344,7 @@ class BuilderThread(threading.Thread):
objdump = self.builder.GetObjdumpFile(result.commit_upto,
result.brd.target, fname)
with open(objdump, 'w') as fd:
print >>fd, dump_result.stdout,
print(dump_result.stdout, end=' ', file=fd)
for line in dump_result.stdout.splitlines():
fields = line.split()
if len(fields) > 5 and fields[1] == '.rodata':
......@@ -378,7 +376,7 @@ class BuilderThread(threading.Thread):
sizes = self.builder.GetSizesFile(result.commit_upto,
result.brd.target)
with open(sizes, 'w') as fd:
print >>fd, '\n'.join(lines)
print('\n'.join(lines), file=fd)
# Write out the configuration files, with a special case for SPL
for dirname in ['', 'spl', 'tpl']:
......
#!/usr/bin/env python2
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0+
#
# Copyright (c) 2012 The Chromium OS Authors.
......@@ -6,6 +6,8 @@
"""See README for more information"""
from __future__ import print_function
import multiprocessing
import os
import re
......@@ -46,11 +48,11 @@ def RunTests(skip_net_tests):
suite = unittest.TestLoader().loadTestsFromTestCase(module)
suite.run(result)
print result
print(result)
for test, err in result.errors:
print err
print(err)
for test, err in result.failures:
print err
print(err)
options, args = cmdline.ParseArgs()
......
......@@ -30,7 +30,7 @@ def GetActionSummary(is_summary, commits, selected, options):
"""
if commits:
count = len(commits)
count = (count + options.step - 1) / options.step
count = (count + options.step - 1) // options.step
commit_str = '%d commit%s' % (count, GetPlural(count))
else:
commit_str = 'current source'
......@@ -59,31 +59,31 @@ def ShowActions(series, why_selected, boards_selected, builder, options,
board_warnings: List of warnings obtained from board selected
"""
col = terminal.Color()
print 'Dry run, so not doing much. But I would do this:'
print
print('Dry run, so not doing much. But I would do this:')
print()
if series:
commits = series.commits
else:
commits = None
print GetActionSummary(False, commits, boards_selected,
options)
print 'Build directory: %s' % builder.base_dir
print(GetActionSummary(False, commits, boards_selected,
options))
print('Build directory: %s' % builder.base_dir)
if commits:
for upto in range(0, len(series.commits), options.step):
commit = series.commits[upto]
print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
print commit.subject
print
print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
print(commit.subject)
print()
for arg in why_selected:
if arg != 'all':
print arg, ': %d boards' % len(why_selected[arg])
print(arg, ': %d boards' % len(why_selected[arg]))
if options.verbose:
print ' %s' % ' '.join(why_selected[arg])
print ('Total boards to build for each commit: %d\n' %
len(why_selected['all']))
print(' %s' % ' '.join(why_selected[arg]))
print(('Total boards to build for each commit: %d\n' %
len(why_selected['all'])))
if board_warnings:
for warning in board_warnings:
print col.Color(col.YELLOW, warning)
print(col.Color(col.YELLOW, warning))
def CheckOutputDir(output_dir):
"""Make sure that the output directory is not within the current directory
......@@ -146,17 +146,17 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
if options.fetch_arch:
if options.fetch_arch == 'list':
sorted_list = toolchains.ListArchs()
print col.Color(col.BLUE, 'Available architectures: %s\n' %
' '.join(sorted_list))
print(col.Color(col.BLUE, 'Available architectures: %s\n' %
' '.join(sorted_list)))
return 0
else:
fetch_arch = options.fetch_arch
if fetch_arch == 'all':
fetch_arch = ','.join(toolchains.ListArchs())
print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
fetch_arch)
print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
fetch_arch))
for arch in fetch_arch.split(','):
print
print()
ret = toolchains.FetchAndInstall(arch)
if ret:
return ret
......@@ -167,7 +167,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
toolchains.Scan(options.list_tool_chains and options.verbose)
if options.list_tool_chains:
toolchains.List()
print
print()
return 0
# Work out how many commits to build. We want to build everything on the
......@@ -191,7 +191,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
options.branch))
if msg:
print col.Color(col.YELLOW, msg)
print(col.Color(col.YELLOW, msg))
count += 1 # Build upstream commit also
if not count:
......@@ -268,7 +268,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
options.threads = min(multiprocessing.cpu_count(), len(selected))
if not options.jobs:
options.jobs = max(1, (multiprocessing.cpu_count() +
len(selected) - 1) / len(selected))
len(selected) - 1) // len(selected))
if not options.step:
options.step = len(series.commits) - 1
......
......@@ -270,7 +270,7 @@ class TestFunctional(unittest.TestCase):
stdout=''.join(commit_log[:count]))
# Not handled, so abort
print 'git log', args
print('git log', args)
sys.exit(1)
def _HandleCommandGitConfig(self, args):
......@@ -286,7 +286,7 @@ class TestFunctional(unittest.TestCase):
stdout='refs/heads/master\n')
# Not handled, so abort
print 'git config', args
print('git config', args)
sys.exit(1)
def _HandleCommandGit(self, in_args):
......@@ -320,7 +320,7 @@ class TestFunctional(unittest.TestCase):
return command.CommandResult(return_code=0)
# Not handled, so abort
print 'git', git_args, sub_cmd, args
print('git', git_args, sub_cmd, args)
sys.exit(1)
def _HandleCommandNm(self, args):
......@@ -351,7 +351,7 @@ class TestFunctional(unittest.TestCase):
if pipe_list[1] == ['wc', '-l']:
wc = True
else:
print 'invalid pipe', kwargs
print('invalid pipe', kwargs)
sys.exit(1)
cmd = pipe_list[0][0]
args = pipe_list[0][1:]
......@@ -371,7 +371,7 @@ class TestFunctional(unittest.TestCase):
if not result:
# Not handled, so abort
print 'unknown command', kwargs
print('unknown command', kwargs)
sys.exit(1)
if wc:
......@@ -404,14 +404,14 @@ class TestFunctional(unittest.TestCase):
return command.CommandResult(return_code=0)
# Not handled, so abort
print 'make', stage
print('make', stage)
sys.exit(1)
# Example function to print output lines
def print_lines(self, lines):
print len(lines)
print(len(lines))
for line in lines:
print line
print(line)
#self.print_lines(terminal.GetPrintTestLines())
def testNoBoards(self):
......
......@@ -212,11 +212,11 @@ class TestBuild(unittest.TestCase):
self.assertEqual(lines[1].text, '02: %s' % commits[1][1])
col = terminal.Color()
self.assertSummary(lines[2].text, 'sandbox', 'w+', ['board4'],
self.assertSummary(lines[2].text, 'arm', 'w+', ['board1'],
outcome=OUTCOME_WARN)
self.assertSummary(lines[3].text, 'arm', 'w+', ['board1'],
self.assertSummary(lines[3].text, 'powerpc', 'w+', ['board2', 'board3'],
outcome=OUTCOME_WARN)
self.assertSummary(lines[4].text, 'powerpc', 'w+', ['board2', 'board3'],
self.assertSummary(lines[4].text, 'sandbox', 'w+', ['board4'],
outcome=OUTCOME_WARN)
# Second commit: The warnings should be listed
......@@ -226,10 +226,10 @@ class TestBuild(unittest.TestCase):
# Third commit: Still fails
self.assertEqual(lines[6].text, '03: %s' % commits[2][1])
self.assertSummary(lines[7].text, 'sandbox', '+', ['board4'])
self.assertSummary(lines[8].text, 'arm', '', ['board1'],
self.assertSummary(lines[7].text, 'arm', '', ['board1'],
outcome=OUTCOME_OK)
self.assertSummary(lines[9].text, 'powerpc', '+', ['board2', 'board3'])
self.assertSummary(lines[8].text, 'powerpc', '+', ['board2', 'board3'])
self.assertSummary(lines[9].text, 'sandbox', '+', ['board4'])
# Expect a compiler error
self.assertEqual(lines[10].text, '+%s' %
......@@ -237,8 +237,6 @@ class TestBuild(unittest.TestCase):
# Fourth commit: Compile errors are fixed, just have warning for board3
self.assertEqual(lines[11].text, '04: %s' % commits[3][1])
self.assertSummary(lines[12].text, 'sandbox', 'w+', ['board4'],
outcome=OUTCOME_WARN)
expect = '%10s: ' % 'powerpc'
expect += ' ' + col.Color(col.GREEN, '')
expect += ' '
......@@ -246,7 +244,9 @@ class TestBuild(unittest.TestCase):
expect += ' ' + col.Color(col.YELLOW, 'w+')
expect += ' '
expect += col.Color(col.YELLOW, ' %s' % 'board3')
self.assertEqual(lines[13].text, expect)
self.assertEqual(lines[12].text, expect)
self.assertSummary(lines[13].text, 'sandbox', 'w+', ['board4'],
outcome=OUTCOME_WARN)
# Compile error fixed
self.assertEqual(lines[14].text, '-%s' %
......@@ -259,9 +259,9 @@ class TestBuild(unittest.TestCase):
# Fifth commit
self.assertEqual(lines[16].text, '05: %s' % commits[4][1])
self.assertSummary(lines[17].text, 'sandbox', '+', ['board4'])
self.assertSummary(lines[18].text, 'powerpc', '', ['board3'],
self.assertSummary(lines[17].text, 'powerpc', '', ['board3'],
outcome=OUTCOME_OK)
self.assertSummary(lines[18].text, 'sandbox', '+', ['board4'])
# The second line of errors[3] is a duplicate, so buildman will drop it
expect = errors[3].rstrip().split('\n')
......
......@@ -4,18 +4,19 @@
import re
import glob
from HTMLParser import HTMLParser
from html.parser import HTMLParser
import os
import sys
import tempfile
import urllib2
import urllib.request, urllib.error, urllib.parse
import bsettings
import command
import terminal
import tools