From e8c506531336c7c446be19e33792146cd33355fd Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Sun, 10 Mar 2019 15:37:08 -0700 Subject: [PATCH] scripts: snapshot of kernel build utility scripts This is a snapshot of build scripts taken as of msm-4.19 commit <3fc21416733e696> (Merge "ARM: dts: msm: Add dpdm regulator on kona"). Change-Id: I3a60f8563227de4e6f59d5abcca7c8edd2c8c12f Signed-off-by: Raghavendra Rao Ananta --- Makefile | 6 +- scripts/build-all.py | 357 ++++++++++++++++++++++++++ scripts/check-config-debug-exemptions | 58 +++++ scripts/check-config-perf-exemptions | 61 +++++ scripts/check-config.py | 124 +++++++++ scripts/gcc-wrapper.py | 79 ++++++ 6 files changed, 684 insertions(+), 1 deletion(-) create mode 100755 scripts/build-all.py create mode 100644 scripts/check-config-debug-exemptions create mode 100644 scripts/check-config-perf-exemptions create mode 100755 scripts/check-config.py create mode 100755 scripts/gcc-wrapper.py diff --git a/Makefile b/Makefile index b6ee29cdae95..43fa032c8d85 100644 --- a/Makefile +++ b/Makefile @@ -407,7 +407,7 @@ KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) # Make variables (CC, etc...) AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld -CC = $(CROSS_COMPILE)gcc +REAL_CC = $(CROSS_COMPILE)gcc CPP = $(CC) -E AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm @@ -426,6 +426,10 @@ PYTHON2 = python2 PYTHON3 = python3 CHECK = sparse +# Use the wrapper for the compiler. This wrapper scans for new +# warnings and causes the build to stop upon encountering them +CC = $(srctree)/scripts/gcc-wrapper.py $(REAL_CC) + CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF) NOSTDINC_FLAGS := diff --git a/scripts/build-all.py b/scripts/build-all.py new file mode 100755 index 000000000000..0cb1e4406cb9 --- /dev/null +++ b/scripts/build-all.py @@ -0,0 +1,357 @@ +#! /usr/bin/env python2 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2009-2015, 2017-19, The Linux Foundation. All rights reserved. + +# Build the kernel for all targets using the Android build environment. + +from collections import namedtuple +import glob +from optparse import OptionParser +import os +import re +import shutil +import subprocess +import sys +import threading +import Queue + +version = 'build-all.py, version 1.99' + +build_dir = '../all-kernels' +make_command = ["vmlinux", "modules", "dtbs"] +all_options = {} +compile64 = os.environ.get('CROSS_COMPILE64') +clang_bin = os.environ.get('CLANG_BIN') + +def error(msg): + sys.stderr.write("error: %s\n" % msg) + +def fail(msg): + """Fail with a user-printed message""" + error(msg) + sys.exit(1) + +if not os.environ.get('CROSS_COMPILE'): + fail("CROSS_COMPILE must be set in the environment") + +def check_kernel(): + """Ensure that PWD is a kernel directory""" + if not os.path.isfile('MAINTAINERS'): + fail("This doesn't seem to be a kernel dir") + +def check_build(): + """Ensure that the build directory is present.""" + if not os.path.isdir(build_dir): + try: + os.makedirs(build_dir) + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + +failed_targets = [] + +BuildResult = namedtuple('BuildResult', ['status', 'messages']) + +class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])): + + def set_width(self, width): + self.width = width + + def __enter__(self): + self.log = open(self.log_name, 'w') + def __exit__(self, type, value, traceback): + self.log.close() + + def run(self): + self.status = None + messages = ["Building: " + self.short_name] + def printer(line): + text = "[%-*s] %s" % (self.width, self.short_name, line) + messages.append(text) + self.log.write(text) + self.log.write('\n') + for step in self.steps: + st = step.run(printer) + if st: + self.status = BuildResult(self.short_name, messages) + break + if not self.status: + self.status = BuildResult(None, messages) + +class BuildTracker: + """Manages all of the steps necessary to perform a build. The + build consists of one or more sequences of steps. The different + sequences can be processed independently, while the steps within a + sequence must be done in order.""" + + def __init__(self, parallel_builds): + self.sequence = [] + self.lock = threading.Lock() + self.parallel_builds = parallel_builds + + def add_sequence(self, log_name, short_name, steps): + self.sequence.append(BuildSequence(log_name, short_name, steps)) + + def longest_name(self): + longest = 0 + for seq in self.sequence: + longest = max(longest, len(seq.short_name)) + return longest + + def __repr__(self): + return "BuildTracker(%s)" % self.sequence + + def run_child(self, seq): + seq.set_width(self.longest) + tok = self.build_tokens.get() + with self.lock: + print "Building:", seq.short_name + with seq: + seq.run() + self.results.put(seq.status) + self.build_tokens.put(tok) + + def run(self): + self.longest = self.longest_name() + self.results = Queue.Queue() + children = [] + errors = [] + self.build_tokens = Queue.Queue() + nthreads = self.parallel_builds + print "Building with", nthreads, "threads" + for i in range(nthreads): + self.build_tokens.put(True) + for seq in self.sequence: + child = threading.Thread(target=self.run_child, args=[seq]) + children.append(child) + child.start() + for child in children: + stats = self.results.get() + if all_options.verbose: + with self.lock: + for line in stats.messages: + print line + sys.stdout.flush() + if stats.status: + errors.append(stats.status) + for child in children: + child.join() + if errors: + fail("\n ".join(["Failed targets:"] + errors)) + +class PrintStep: + """A step that just prints a message""" + def __init__(self, message): + self.message = message + + def run(self, outp): + outp(self.message) + +class MkdirStep: + """A step that makes a directory""" + def __init__(self, direc): + self.direc = direc + + def run(self, outp): + outp("mkdir %s" % self.direc) + os.mkdir(self.direc) + +class RmtreeStep: + def __init__(self, direc): + self.direc = direc + + def run(self, outp): + outp("rmtree %s" % self.direc) + shutil.rmtree(self.direc, ignore_errors=True) + +class CopyfileStep: + def __init__(self, src, dest): + self.src = src + self.dest = dest + + def run(self, outp): + outp("cp %s %s" % (self.src, self.dest)) + shutil.copyfile(self.src, self.dest) + +class ExecStep: + def __init__(self, cmd, **kwargs): + self.cmd = cmd + self.kwargs = kwargs + + def run(self, outp): + outp("exec: %s" % (" ".join(self.cmd),)) + with open('/dev/null', 'r') as devnull: + proc = subprocess.Popen(self.cmd, stdin=devnull, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + **self.kwargs) + stdout = proc.stdout + while True: + line = stdout.readline() + if not line: + break + line = line.rstrip('\n') + outp(line) + result = proc.wait() + if result != 0: + return ('error', result) + else: + return None + +class Builder(): + + def __init__(self, name, defconfig): + self.name = name + self.defconfig = defconfig + + self.confname = re.sub('arch/arm[64]*/configs/', '', self.defconfig) + + # Determine if this is a 64-bit target based on the location + # of the defconfig. + self.make_env = os.environ.copy() + if "/arm64/" in defconfig: + if compile64: + self.make_env['CROSS_COMPILE'] = compile64 + else: + fail("Attempting to build 64-bit, without setting CROSS_COMPILE64") + self.make_env['ARCH'] = 'arm64' + else: + self.make_env['ARCH'] = 'arm' + self.make_env['KCONFIG_NOTIMESTAMP'] = 'true' + self.log_name = "%s/log-%s.log" % (build_dir, self.name) + + def build(self): + steps = [] + dest_dir = os.path.join(build_dir, self.name) + log_name = "%s/log-%s.log" % (build_dir, self.name) + steps.append(PrintStep('Building %s in %s log %s' % + (self.name, dest_dir, log_name))) + if not os.path.isdir(dest_dir): + steps.append(MkdirStep(dest_dir)) + defconfig = self.defconfig + dotconfig = '%s/.config' % dest_dir + savedefconfig = '%s/defconfig' % dest_dir + + staging_dir = 'install_staging' + modi_dir = '%s' % staging_dir + hdri_dir = '%s/usr' % staging_dir + steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir))) + + steps.append(ExecStep(['make', 'O=%s' % dest_dir, + self.confname], env=self.make_env)) + + # Build targets can be dependent upon the completion of + # previous build targets, so build them one at a time. + cmd_line = ['make', + 'INSTALL_HDR_PATH=%s' % hdri_dir, + 'INSTALL_MOD_PATH=%s' % modi_dir, + 'O=%s' % dest_dir, + 'REAL_CC=%s' % clang_bin] + build_targets = [] + for c in make_command: + if re.match(r'^-{1,2}\w', c): + cmd_line.append(c) + else: + build_targets.append(c) + for t in build_targets: + steps.append(ExecStep(cmd_line + [t], env=self.make_env)) + + return steps + +def scan_configs(): + """ + Get the full list of defconfigs appropriate for this tree, + except for gki_defconfig. + """ + names = [] + for defconfig in glob.glob('arch/arm*/configs/vendor/*_defconfig'): + target = os.path.basename(defconfig)[:-10] + if 'gki' == target: + continue + name = target + "-llvm" + if 'arch/arm64' in defconfig: + name = name + "-64" + names.append(Builder(name, defconfig)) + + return names + +def build_many(targets): + print "Building %d target(s)" % len(targets) + + # To try and make up for the link phase being serial, try to do + # two full builds in parallel. Don't do too many because lots of + # parallel builds tends to use up available memory rather quickly. + parallel = 2 + if all_options.jobs and all_options.jobs > 1: + j = max(all_options.jobs / parallel, 2) + make_command.append("-j" + str(j)) + + tracker = BuildTracker(parallel) + for target in targets: + steps = target.build() + tracker.add_sequence(target.log_name, target.name, steps) + tracker.run() + +def main(): + global make_command + + check_kernel() + check_build() + + configs = scan_configs() + + usage = (""" + %prog [options] all -- Build all targets + %prog [options] target target ... -- List specific targets + """) + parser = OptionParser(usage=usage, version=version) + parser.add_option('--list', action='store_true', + dest='list', + help='List available targets') + parser.add_option('-v', '--verbose', action='store_true', + dest='verbose', + help='Output to stdout in addition to log file') + parser.add_option('-j', '--jobs', type='int', dest="jobs", + help="Number of simultaneous jobs") + parser.add_option('-l', '--load-average', type='int', + dest='load_average', + help="Don't start multiple jobs unless load is below LOAD_AVERAGE") + parser.add_option('-k', '--keep-going', action='store_true', + dest='keep_going', default=False, + help="Keep building other targets if a target fails") + parser.add_option('-m', '--make-target', action='append', + help='Build the indicated make target (default: %s)' % + ' '.join(make_command)) + + (options, args) = parser.parse_args() + global all_options + all_options = options + + if options.list: + print "Available targets:" + for target in configs: + print " %s" % target.name + sys.exit(0) + + if options.make_target: + make_command = options.make_target + + if args == ['all']: + build_many(configs) + elif len(args) > 0: + all_configs = {} + for t in configs: + all_configs[t.name] = t + targets = [] + for t in args: + if t not in all_configs: + parser.error("Target '%s' not one of %s" % (t, all_configs.keys())) + targets.append(all_configs[t]) + build_many(targets) + else: + parser.error("Must specify a target to build, or 'all'") + +if __name__ == "__main__": + main() diff --git a/scripts/check-config-debug-exemptions b/scripts/check-config-debug-exemptions new file mode 100644 index 000000000000..331a924e3612 --- /dev/null +++ b/scripts/check-config-debug-exemptions @@ -0,0 +1,58 @@ +CP15_BARRIER_EMULATION +DEVKMEM +DEVMEM +HID_A4TECH +HID_ACRUX +HID_BELKIN +HID_CHERRY +HID_CHICONY +HID_CYPRESS +HID_DRAGONRISE +HID_EMS_FF +HID_EZKEY +HID_GREENASIA +HID_GYRATION +HID_HOLTEK +HID_KENSINGTON +HID_KEYTOUCH +HID_KYE +HID_LCPOWER +HID_LOGITECH +HID_MONTEREY +HID_NTRIG +HID_ORTEK +HID_PANTHERLORD +HID_PETALYNX +HID_PICOLCD +HID_PRIMAX +HID_PRODIKEYS +HID_ROCCAT +HID_SAITEK +HID_SAMSUNG +HID_SMARTJOYPLUS +HID_SONY +HID_SPEEDLINK +HID_SUNPLUS +HID_THRUSTMASTER +HID_TIVO +HID_TOPSEED +HID_TWINHAN +HID_UCLOGIC +HID_WACOM +HID_WALTOP +HID_WIIMOTE +HID_ZEROPLUS +HID_ZYDACRON +JOYSTICK_XPAD_FF +JOYSTICK_XPAD_LEDS +KSM +MODULES +PSTORE +SETEND_EMULATION +TABLET_USB_ACECAD +TABLET_USB_AIPTEK +TABLET_USB_GTCO +TABLET_USB_HANWANG +TABLET_USB_KBTAB +USB_CONFIGFS +USB_OTG_WAKELOCK diff --git a/scripts/check-config-perf-exemptions b/scripts/check-config-perf-exemptions new file mode 100644 index 000000000000..d499f5abeb1e --- /dev/null +++ b/scripts/check-config-perf-exemptions @@ -0,0 +1,61 @@ +CGROUP_DEBUG +CP15_BARRIER_EMULATION +DEVKMEM +DEVMEM +HID_A4TECH +HID_ACRUX +HID_BELKIN +HID_CHERRY +HID_CHICONY +HID_CYPRESS +HID_DRAGONRISE +HID_EMS_FF +HID_EZKEY +HID_GREENASIA +HID_GYRATION +HID_HOLTEK +HID_KENSINGTON +HID_KEYTOUCH +HID_KYE +HID_LCPOWER +HID_LOGITECH +HID_MONTEREY +HID_NTRIG +HID_ORTEK +HID_PANTHERLORD +HID_PETALYNX +HID_PICOLCD +HID_PRIMAX +HID_PRODIKEYS +HID_ROCCAT +HID_SAITEK +HID_SAMSUNG +HID_SMARTJOYPLUS +HID_SONY +HID_SPEEDLINK +HID_SUNPLUS +HID_THRUSTMASTER +HID_TIVO +HID_TOPSEED +HID_TWINHAN +HID_UCLOGIC +HID_WACOM +HID_WALTOP +HID_WIIMOTE +HID_ZEROPLUS +HID_ZYDACRON +JOYSTICK_XPAD_FF +JOYSTICK_XPAD_LEDS +KSM +MODULES +PM_DEBUG +PSTORE +SETEND_EMULATION +SUSPEND_TIME +TABLET_USB_ACECAD +TABLET_USB_AIPTEK +TABLET_USB_GTCO +TABLET_USB_HANWANG +TABLET_USB_KBTAB +USB_CONFIGFS +USB_OTG_WAKELOCK diff --git a/scripts/check-config.py b/scripts/check-config.py new file mode 100755 index 000000000000..17d5d5ad5218 --- /dev/null +++ b/scripts/check-config.py @@ -0,0 +1,124 @@ +#! /usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only + +# Copyright (c) 2015, 2018 The Linux Foundation. All rights reserved. + +""" +Android kernel configuration validator. + +The Android kernel reference trees contain some config stubs of +configuration options that are required for Android to function +correctly, and additional ones that are recommended. + +This script can help compare these base configs with the ".config" +output of the compiler to determine if the proper configs are defined. +""" + +from collections import namedtuple +from optparse import OptionParser +import re +import sys + +version = "check-config.py, version 0.0.1" + +req_re = re.compile(r'''^CONFIG_(.*)=(.*)$''') +forb_re = re.compile(r'''^# CONFIG_(.*) is not set$''') +comment_re = re.compile(r'''^(#.*|)$''') + +Enabled = namedtuple('Enabled', ['name', 'value']) +Disabled = namedtuple('Disabled', ['name']) + +def walk_config(name): + with open(name, 'r') as fd: + for line in fd: + line = line.rstrip() + m = req_re.match(line) + if m: + yield Enabled(m.group(1), m.group(2)) + continue + + m = forb_re.match(line) + if m: + yield Disabled(m.group(1)) + continue + + m = comment_re.match(line) + if m: + continue + + print "WARNING: Unknown .config line: ", line + +class Checker(): + def __init__(self): + self.required = {} + self.exempted = set() + self.forbidden = set() + + def add_required(self, fname): + for ent in walk_config(fname): + if type(ent) is Enabled: + self.required[ent.name] = ent.value + elif type(ent) is Disabled: + if ent.name in self.required: + del self.required[ent.name] + self.forbidden.add(ent.name) + + def add_exempted(self, fname): + with open(fname, 'r') as fd: + for line in fd: + line = line.rstrip() + self.exempted.add(line) + + def check(self, path): + failure = False + + # Don't run this for mdm targets + if re.search('mdm', path): + print "Not applicable to mdm targets... bypassing" + else: + for ent in walk_config(path): + # Go to the next iteration if this config is exempt + if ent.name in self.exempted: + continue + + if type(ent) is Enabled: + if ent.name in self.forbidden: + print "error: Config should not be present: %s" %ent.name + failure = True + + if ent.name in self.required and ent.value != self.required[ent.name]: + print "error: Config has wrong value: %s %s expecting: %s" \ + %(ent.name, ent.value, self.required[ent.name]) + failure = True + + elif type(ent) is Disabled: + if ent.name in self.required: + print "error: Config should be present, but is disabled: %s" %ent.name + failure = True + + if failure: + sys.exit(1) + +def main(): + usage = """%prog [options] path/to/.config""" + parser = OptionParser(usage=usage, version=version) + parser.add_option('-r', '--required', dest="required", + action="append") + parser.add_option('-e', '--exempted', dest="exempted", + action="append") + (options, args) = parser.parse_args() + if len(args) != 1: + parser.error("Expecting a single path argument to .config") + elif options.required is None or options.exempted is None: + parser.error("Expecting a file containing required configurations") + + ch = Checker() + for r in options.required: + ch.add_required(r) + for e in options.exempted: + ch.add_exempted(e) + + ch.check(args[0]) + +if __name__ == '__main__': + main() diff --git a/scripts/gcc-wrapper.py b/scripts/gcc-wrapper.py new file mode 100755 index 000000000000..0d64f731c8fb --- /dev/null +++ b/scripts/gcc-wrapper.py @@ -0,0 +1,79 @@ +#! /usr/bin/env python2 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2011-2017, 2018 The Linux Foundation. All rights reserved. + +# -*- coding: utf-8 -*- + +# Invoke gcc, looking for warnings, and causing a failure if there are +# non-whitelisted warnings. + +import errno +import re +import os +import sys +import subprocess + +# Note that gcc uses unicode, which may depend on the locale. TODO: +# force LANG to be set to en_US.UTF-8 to get consistent warnings. + +allowed_warnings = set([ + "umid.c:138", + "umid.c:213", + "umid.c:388", + "coresight-catu.h:116", + "mprotect.c:42", + "signal.c:93", + "signal.c:51", + ]) + +# Capture the name of the object file, can find it. +ofile = None + +warning_re = re.compile(r'''(.*/|)([^/]+\.[a-z]+:\d+):(\d+:)? warning:''') +def interpret_warning(line): + """Decode the message from gcc. The messages we care about have a filename, and a warning""" + line = line.rstrip('\n') + m = warning_re.match(line) + if m and m.group(2) not in allowed_warnings: + print "error, forbidden warning:", m.group(2) + + # If there is a warning, remove any object if it exists. + if ofile: + try: + os.remove(ofile) + except OSError: + pass + sys.exit(1) + +def run_gcc(): + args = sys.argv[1:] + # Look for -o + try: + i = args.index('-o') + global ofile + ofile = args[i+1] + except (ValueError, IndexError): + pass + + compiler = sys.argv[0] + + try: + proc = subprocess.Popen(args, stderr=subprocess.PIPE) + for line in proc.stderr: + print line, + interpret_warning(line) + + result = proc.wait() + except OSError as e: + result = e.errno + if result == errno.ENOENT: + print args[0] + ':',e.strerror + print 'Is your PATH set correctly?' + else: + print ' '.join(args), str(e) + + return result + +if __name__ == '__main__': + status = run_gcc() + sys.exit(status)