[Buildroot] [git commit branch/next] support/scripts/gen-bootlin-toolchains: add new script to support Bootlin toolchains
Yann E. MORIN
yann.morin.1998 at free.fr
Sat Aug 29 13:47:39 UTC 2020
commit: https://git.buildroot.net/buildroot/commit/?id=184d47a7adb8f58a036b1dcaa72d09219a65feb2
branch: https://git.buildroot.net/buildroot/commit/?id=refs/heads/next
https://toolchains.bootlin.com/ has been providing for a few years a
number of ready-to-use pre-built toolchains, for a wide range of
architectures (which it turns out, are all built using Buildroot).
While toolchains.bootlin.com provides Buildroot config fragments to
easily use those toolchains with Buildroot (see [0] for example), this
is not visible anywhere. So instead, we would like to add support for
these toolchains in Buildroot just like we have existing support for
Linaro, ARM, Synopsys, etc. toolchains.
[0] https://toolchains.bootlin.com/downloads/releases/toolchains/aarch64/fragments/aarch64--glibc--bleeding-edge-2020.02-2.frag
However, the number of toolchains provided by toolchains.bootlin.com
is really large, and they are regularly updated. Maintaining that
manually would be time consuming and error-prone. So instead, this
commit introduces a script that automatically generates:
- toolchain/toolchain-external/toolchain-external-bootlin/Config.in.options
- toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.mk
- toolchain/toolchain-external/toolchain-external-bootlin/toolchain-external-bootlin.hash
- support/testing/tests/toolchain/test_external_bootlin.py
We create a single external toolchain package, with a Kconfig "choice"
as a sub-option to select the toolchain variant to be used. The script
contains a Python dict that provides the mapping between the
toolchains provided by toolchains.bootlin.com, and the architecture
options/variants they are applicable to.
The test cases allow to verify that the toolchain configuration is
correct, and that it is able to build a Busybox based system. It
doesn't do any runtime testing as such testing is already done by
toolchains.bootlin.com: the test cases here are only meant to verify
that the toolchain-external-bootlin package works as expected.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
Reviewed-by: Titouan Christophe <titouan.christophe at railnova.eu>
Tested-by: Titouan Christophe <titouan.christophe at railnova.eu>
Signed-off-by: Yann E. MORIN <yann.morin.1998 at free.fr>
---
support/scripts/gen-bootlin-toolchains | 473 +++++++++++++++++++++++++++++++++
1 file changed, 473 insertions(+)
diff --git a/support/scripts/gen-bootlin-toolchains b/support/scripts/gen-bootlin-toolchains
new file mode 100755
index 0000000000..a5e4b1a99b
--- /dev/null
+++ b/support/scripts/gen-bootlin-toolchains
@@ -0,0 +1,473 @@
+#!/usr/bin/env python3
+
+import os.path
+import re
+import requests
+import textwrap
+
+BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
+
+AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
+# Do not edit
+"""
+
+# In the below dict:
+
+# - 'conditions' indicate the cumulative conditions under which the
+# toolchain will be made available. In several situations, a given
+# toolchain is usable on several architectures variants (for
+# example, an ARMv6 toolchain can be used on ARMv7)
+# - 'test_options' indicate one specific configuration where the
+# toolchain can be used. It is used to create the runtime test
+# cases. If 'test_options' does not exist, the code assumes it can
+# be made equal to 'conditions'
+# - 'prefix' is the prefix of the cross-compilation toolchain tools
+
+arches = {
+ 'aarch64': {
+ 'conditions': ['BR2_aarch64'],
+ 'prefix': 'aarch64',
+ },
+ 'aarch64be': {
+ 'conditions': ['BR2_aarch64_be'],
+ 'prefix': 'aarch64_be',
+ },
+ 'arcle-750d': {
+ 'conditions': ['BR2_arcle', 'BR2_arc750d'],
+ 'prefix': 'arc',
+ },
+ 'arcle-hs38': {
+ 'conditions': ['BR2_arcle', 'BR2_archs38'],
+ 'prefix': 'arc',
+ },
+ 'armv5-eabi': {
+ 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
+ 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
+ 'prefix': 'arm',
+ },
+ 'armv6-eabihf': {
+ 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
+ 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
+ 'prefix': 'arm',
+ },
+ 'armv7-eabihf': {
+ 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
+ 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
+ 'prefix': 'arm',
+ },
+ 'armv7m': {
+ 'conditions': ['BR2_ARM_CPU_ARMV7M'],
+ 'test_options': ['BR2_arm', 'BR2_cortex_m4'],
+ 'prefix': 'arm',
+ },
+ 'm68k-68xxx': {
+ 'conditions': ['BR2_m68k_m68k'],
+ 'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
+ 'prefix': 'm68k',
+ },
+ 'm68k-coldfire': {
+ 'conditions': ['BR2_m68k_cf'],
+ 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
+ 'prefix': 'm68k',
+ },
+ 'microblazebe': {
+ 'conditions': ['BR2_microblazebe'],
+ 'prefix': 'microblaze',
+ },
+ 'microblazeel': {
+ 'conditions': ['BR2_microblazeel'],
+ 'prefix': 'microblazeel',
+ },
+ 'mips32': {
+ # Not sure it could be used by other mips32 variants?
+ 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips',
+ },
+ 'mips32el': {
+ # Not sure it could be used by other mips32el variants?
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips32r5el': {
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips32r6el': {
+ 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mipsel',
+ },
+ 'mips64': {
+ # Not sure it could be used by other mips64 variants?
+ 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64',
+ },
+ 'mips64-n32': {
+ # Not sure it could be used by other mips64 variants?
+ 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64',
+ },
+ 'mips64el-n32': {
+ # Not sure it could be used by other mips64el variants?
+ 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64el',
+ },
+ 'mips64r6el-n32': {
+ 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
+ 'prefix': 'mips64el',
+ },
+ 'nios2': {
+ 'conditions': ['BR2_nios2'],
+ 'prefix': 'nios2',
+ },
+ 'openrisc': {
+ 'conditions': ['BR2_or1k'],
+ 'prefix': 'or1k',
+ },
+ 'powerpc-e500mc': {
+ # Not sure it could be used by other powerpc variants?
+ 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
+ 'prefix': 'powerpc',
+ },
+ 'powerpc64-e5500': {
+ 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
+ 'prefix': 'powerpc64',
+ },
+ 'powerpc64-power8': {
+ 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
+ 'prefix': 'powerpc64',
+ },
+ 'powerpc64le-power8': {
+ 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
+ 'prefix': 'powerpc64le',
+ },
+ 'riscv32-ilp32d': {
+ 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
+ 'prefix': 'riscv32',
+ },
+ 'riscv64': {
+ 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
+ 'prefix': 'riscv64',
+ },
+ 'sh-sh4': {
+ 'conditions': ['BR2_sh', 'BR2_sh4'],
+ 'prefix': 'sh4',
+ },
+ 'sh-sh4aeb': {
+ 'conditions': ['BR2_sh', 'BR2_sh4aeb'],
+ 'prefix': 'sh4aeb',
+ },
+ 'sparc64': {
+ 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
+ 'prefix': 'sparc64',
+ },
+ 'sparcv8': {
+ 'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
+ 'prefix': 'sparc',
+ },
+ 'x86-64-core-i7': {
+ 'conditions': ['BR2_x86_64',
+ 'BR2_X86_CPU_HAS_MMX',
+ 'BR2_X86_CPU_HAS_SSE',
+ 'BR2_X86_CPU_HAS_SSE2',
+ 'BR2_X86_CPU_HAS_SSE3',
+ 'BR2_X86_CPU_HAS_SSSE3',
+ 'BR2_X86_CPU_HAS_SSE4',
+ 'BR2_X86_CPU_HAS_SSE42'],
+ 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
+ 'prefix': 'x86_64',
+ },
+ 'x86-core2': {
+ 'conditions': ['BR2_i386',
+ 'BR2_X86_CPU_HAS_MMX',
+ 'BR2_X86_CPU_HAS_SSE',
+ 'BR2_X86_CPU_HAS_SSE2',
+ 'BR2_X86_CPU_HAS_SSE3',
+ 'BR2_X86_CPU_HAS_SSSE3'],
+ 'test_options': ['BR2_i386', 'BR2_x86_core2'],
+ 'prefix': 'i686',
+ },
+ 'x86-i686': {
+ 'conditions': ['BR2_i386',
+ '!BR2_x86_i486',
+ '!BR2_x86_i586',
+ '!BR2_x86_x1000'],
+ 'test_options': ['BR2_i386',
+ 'BR2_x86_i686'],
+ 'prefix': 'i686',
+ },
+ 'xtensa-lx60': {
+ 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
+ 'prefix': 'xtensa',
+ },
+}
+
+
+class Toolchain:
+ def __init__(self, arch, libc, variant, version):
+ self.arch = arch
+ self.libc = libc
+ self.variant = variant
+ self.version = version
+ self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
+ self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
+ (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
+ self.fragment = requests.get(self.fragment_url).text.split("\n")
+ self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
+
+ @property
+ def tarball_url(self):
+ return os.path.join(BASE_URL, self.arch, "tarballs",
+ self.fname_prefix + ".tar.bz2")
+
+ @property
+ def hash_url(self):
+ return os.path.join(BASE_URL, self.arch, "tarballs",
+ self.fname_prefix + ".sha256")
+
+ @property
+ def fragment_url(self):
+ return os.path.join(BASE_URL, self.arch, "fragments",
+ self.fname_prefix + ".frag")
+
+ def gen_config_in_options(self, f):
+ f.write("config %s\n" % self.option_name)
+ f.write("\tbool \"%s %s %s %s\"\n" %
+ (self.arch, self.libc, self.variant, self.version))
+ for c in arches[self.arch]['conditions']:
+ f.write("\tdepends on %s\n" % c)
+ selects = []
+ for frag in self.fragment:
+ # libc type
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
+ elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
+ # all glibc toolchains have RPC support
+ selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+ elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
+ selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
+
+ # gcc version
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
+ m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
+ assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
+ selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
+
+ # kernel headers version
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
+ m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
+ assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
+ selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
+
+ # C++
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
+ selects.append("BR2_INSTALL_LIBSTDCPP")
+
+ # SSP
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
+ selects.append("BR2_TOOLCHAIN_HAS_SSP")
+
+ # wchar
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
+ selects.append("BR2_USE_WCHAR")
+
+ # locale
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
+ # locale implies the availability of wchar
+ selects.append("BR2_USE_WCHAR")
+ selects.append("BR2_ENABLE_LOCALE")
+
+ # thread support
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS")
+
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
+
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
+ selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
+
+ # RPC
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
+ selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
+
+ # D language
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
+ selects.append("BR2_TOOLCHAIN_HAS_DLANG")
+
+ # fortran
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
+ selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
+
+ # OpenMP
+ if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
+ selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
+
+ for select in selects:
+ f.write("\tselect %s\n" % select)
+
+ f.write("\thelp\n")
+
+ desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
+ (self.arch, self.libc)
+
+ if self.variant == "stable":
+ desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
+ else:
+ desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
+
+ f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
+ f.write("\n")
+ f.write("\t https://toolchains.bootlin.com/\n")
+
+ f.write("\n")
+
+ def gen_mk(self, f):
+ f.write("ifeq ($(%s),y)\n" % self.option_name)
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
+ (self.arch, self.libc, self.variant))
+ f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
+ os.path.join(BASE_URL, self.arch, "tarballs"))
+ f.write("endif\n\n")
+ pass
+
+ def gen_hash(self, f):
+ f.write("# From %s\n" % self.hash_url)
+ f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
+
+ def gen_test(self, f):
+ if self.variant == "stable":
+ variant = "Stable"
+ else:
+ variant = "BleedingEdge"
+ testname = "TestExternalToolchainBootlin" + \
+ self.arch.replace("-", "").capitalize() + \
+ self.libc.capitalize() + variant
+ f.write("\n\n")
+ f.write("class %s(TestExternalToolchain):\n" % testname)
+ f.write(" config = \"\"\"\n")
+ if 'test_options' in arches[self.arch]:
+ test_options = arches[self.arch]['test_options']
+ else:
+ test_options = arches[self.arch]['conditions']
+ for opt in test_options:
+ if opt.startswith("!"):
+ f.write(" # %s is not set\n" % opt[1:])
+ else:
+ f.write(" %s=y\n" % opt)
+ f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
+ f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
+ f.write(" %s=y\n" % self.option_name)
+ f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
+ f.write(" \"\"\"\n")
+ f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
+ f.write("\n")
+ f.write(" def test_run(self):\n")
+ f.write(" TestExternalToolchain.common_check(self)\n")
+
+ def __repr__(self):
+ return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
+ (self.arch, self.libc, self.variant, self.version, self.option_name)
+
+
+def get_toolchains():
+ toolchains = list()
+ for arch, details in arches.items():
+ print(arch)
+ url = os.path.join(BASE_URL, arch, "available_toolchains")
+ page = requests.get(url).text
+ fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
+ print(fnames)
+ # This dict will allow us to keep only the latest version for
+ # each toolchain.
+ tmp = dict()
+ for fname in fnames:
+ parts = fname.split('--')
+ assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
+ libc = parts[1]
+ if parts[2].startswith("stable-"):
+ variant = "stable"
+ version = parts[2][len("stable-"):]
+ elif parts[2].startswith("bleeding-edge-"):
+ variant = "bleeding-edge"
+ version = parts[2][len("bleeding-edge-"):]
+ tmp[(arch, libc, variant)] = version
+
+ toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
+
+ return toolchains
+
+
+def gen_config_in_options(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write(AUTOGENERATED_COMMENT)
+
+ f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
+ f.write("\tbool\n")
+ for arch, details in arches.items():
+ f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
+ f.write("\n")
+
+ f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
+
+ f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
+ f.write("\tdefault \"$(ARCH)-linux\"\n")
+
+ f.write("\n")
+
+ f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
+ f.write("\tdefault \"toolchain-external-bootlin\"\n")
+
+ f.write("\n")
+
+ f.write("choice\n")
+ f.write("\tprompt \"Bootlin toolchain variant\"\n")
+
+ for toolchain in toolchains:
+ toolchain.gen_config_in_options(f)
+
+ f.write("endchoice\n")
+ f.write("endif\n")
+
+
+def gen_mk(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write("#" * 80 + "\n")
+ f.write("#\n")
+ f.write("# toolchain-external-bootlin\n")
+ f.write("#\n")
+ f.write("#" * 80 + "\n")
+ f.write("\n")
+ f.write(AUTOGENERATED_COMMENT)
+ for toolchain in toolchains:
+ toolchain.gen_mk(f)
+ f.write("$(eval $(toolchain-external-package))\n")
+
+
+def gen_hash(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write(AUTOGENERATED_COMMENT)
+ for toolchain in toolchains:
+ toolchain.gen_hash(f)
+
+
+def gen_runtime_test(toolchains, fpath):
+ with open(fpath, "w") as f:
+ f.write(AUTOGENERATED_COMMENT)
+ f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
+ for toolchain in toolchains:
+ toolchain.gen_test(f)
+
+
+def gen_toolchains(toolchains):
+ maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
+ gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
+ gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
+ gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
+ gen_runtime_test(toolchains,
+ os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
+
+
+toolchains = get_toolchains()
+gen_toolchains(toolchains)
More information about the buildroot
mailing list