[Buildroot] [PATCH v2 1/5] support/scripts/gen-bootlin-toolchains: add new script to support Bootlin toolchains

Yann E. MORIN yann.morin.1998 at free.fr
Thu Aug 13 22:06:01 UTC 2020


Thomas, All,

On 2020-08-09 21:38 +0200, Thomas Petazzoni spake thusly:
> https://toolchains.bootlin.com/ has been providing for a few years a
> number of ready-to-use pre-built toolchains, for a wide range of
> architectures (which it turns out, are all built using Buildroot).
[--SNIP--]
> Signed-off-by: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
[--SNIP--]
> diff --git a/support/scripts/gen-bootlin-toolchains b/support/scripts/gen-bootlin-toolchains
> new file mode 100755
> index 0000000000..a991797903
> --- /dev/null
> +++ b/support/scripts/gen-bootlin-toolchains
> @@ -0,0 +1,475 @@
> +#!/usr/bin/env python3
> +
> +import htmllistparse

    Traceback (most recent call last):
      File "support/scripts/gen-bootlin-toolchains", line 3, in <module>
        import htmllistparse
    ModuleNotFoundError: No module named 'htmllistparse'

htmllistparse is not available on my distro (Ubuntu 19.10), and it is
not even available in the latest LTS either (https://packages.ubuntu.com/htmllistparse)

Sometimes, it is not possible to install from PyPi (enterprise-grade
machines barred from accessing Pypi for example).

Regards,
Yann E. MORIN.

> +import os.path
> +import re
> +import requests
> +import textwrap
> +
> +BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
> +
> +AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
> +# Do not edit
> +"""
> +
> +# In the below dict:
> +
> +# - 'conditions' indicate the cumulative conditions under which the
> +#   toolchain will be made available. In several situations, a given
> +#   toolchain is usable on several architectures variants (for
> +#   example, an ARMv6 toolchain can be used on ARMv7)
> +# - 'test_options' indicate one specific configuration where the
> +#   toolchain can be used. It is used to create the runtime test
> +#   cases. If 'test_options' does not exist, the code assumes it can
> +#   be made equal to 'conditions'
> +# - 'prefix' is the prefix of the cross-compilation toolchain tools
> +
> +arches = {
> +    'aarch64': {
> +        'conditions': ['BR2_aarch64'],
> +        'prefix': 'aarch64',
> +    },
> +    'aarch64be': {
> +        'conditions': ['BR2_aarch64_be'],
> +        'prefix': 'aarch64_be',
> +    },
> +    'arcle-750d': {
> +        'conditions': ['BR2_arcle', 'BR2_arc750d'],
> +        'prefix': 'arc',
> +    },
> +    'arcle-hs38': {
> +        'conditions': ['BR2_arcle', 'BR2_archs38'],
> +        'prefix': 'arc',
> +    },
> +    'armv5-eabi': {
> +        'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
> +        'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
> +        'prefix': 'arm',
> +    },
> +    'armv6-eabihf': {
> +        'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
> +        'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
> +        'prefix': 'arm',
> +    },
> +    'armv7-eabihf': {
> +        'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
> +        'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
> +        'prefix': 'arm',
> +    },
> +    'armv7m': {
> +        'conditions': ['BR2_ARM_CPU_ARMV7M'],
> +        'test_options': ['BR2_arm', 'BR2_cortex_m4'],
> +        'prefix': 'arm',
> +    },
> +    'm68k-68xxx': {
> +        'conditions': ['BR2_m68k_m68k'],
> +        'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
> +        'prefix': 'm68k',
> +    },
> +    'm68k-coldfire': {
> +        'conditions': ['BR2_m68k_cf'],
> +        'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
> +        'prefix': 'm68k',
> +    },
> +    'microblazebe': {
> +        'conditions': ['BR2_microblazebe'],
> +        'prefix': 'microblaze',
> +    },
> +    'microblazeel': {
> +        'conditions': ['BR2_microblazeel'],
> +        'prefix': 'microblazeel',
> +    },
> +    'mips32': {
> +        # Not sure it could be used by other mips32 variants?
> +        'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mips',
> +    },
> +    'mips32el': {
> +        # Not sure it could be used by other mips32el variants?
> +        'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mipsel',
> +    },
> +    'mips32r5el': {
> +        'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mipsel',
> +    },
> +    'mips32r6el': {
> +        'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mipsel',
> +    },
> +    'mips64': {
> +        # Not sure it could be used by other mips64 variants?
> +        'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mips64',
> +    },
> +    'mips64-n32': {
> +        # Not sure it could be used by other mips64 variants?
> +        'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mips64',
> +    },
> +    'mips64el-n32': {
> +        # Not sure it could be used by other mips64el variants?
> +        'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mips64el',
> +    },
> +    'mips64r6el-n32': {
> +        'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
> +        'prefix': 'mips64el',
> +    },
> +    'nios2': {
> +        'conditions': ['BR2_nios2'],
> +        'prefix': 'nios2',
> +    },
> +    'openrisc': {
> +        'conditions': ['BR2_or1k'],
> +        'prefix': 'or1k',
> +    },
> +    'powerpc-e500mc': {
> +        # Not sure it could be used by other powerpc variants?
> +        'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
> +        'prefix': 'powerpc',
> +    },
> +    'powerpc64-e5500': {
> +        'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
> +        'prefix': 'powerpc64',
> +    },
> +    'powerpc64-power8': {
> +        'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
> +        'prefix': 'powerpc64',
> +    },
> +    'powerpc64le-power8': {
> +        'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
> +        'prefix': 'powerpc64le',
> +    },
> +    'riscv32-ilp32d': {
> +        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
> +        'prefix': 'riscv32',
> +    },
> +    'riscv64': {
> +        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'],
> +        'prefix': 'riscv64',
> +    },
> +    'sh-sh4': {
> +        'conditions': ['BR2_sh', 'BR2_sh4'],
> +        'prefix': 'sh4',
> +    },
> +    'sh-sh4aeb': {
> +        'conditions': ['BR2_sh', 'BR2_sh4aeb'],
> +        'prefix': 'sh4aeb',
> +    },
> +    'sparc64': {
> +        'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
> +        'prefix': 'sparc64',
> +    },
> +    'sparcv8': {
> +        'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
> +        'prefix': 'sparc',
> +    },
> +    'x86-64-core-i7': {
> +        'conditions': ['BR2_x86_64',
> +                       'BR2_X86_CPU_HAS_MMX',
> +                       'BR2_X86_CPU_HAS_SSE',
> +                       'BR2_X86_CPU_HAS_SSE2',
> +                       'BR2_X86_CPU_HAS_SSE3',
> +                       'BR2_X86_CPU_HAS_SSSE3',
> +                       'BR2_X86_CPU_HAS_SSE4',
> +                       'BR2_X86_CPU_HAS_SSE42'],
> +        'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
> +        'prefix': 'x86_64',
> +    },
> +    'x86-core2': {
> +        'conditions': ['BR2_i386',
> +                       'BR2_X86_CPU_HAS_MMX',
> +                       'BR2_X86_CPU_HAS_SSE',
> +                       'BR2_X86_CPU_HAS_SSE2',
> +                       'BR2_X86_CPU_HAS_SSE3',
> +                       'BR2_X86_CPU_HAS_SSSE3'],
> +        'test_options': ['BR2_i386', 'BR2_x86_core2'],
> +        'prefix': 'i686',
> +    },
> +    'x86-i686': {
> +        'conditions': ['BR2_i386',
> +                       '!BR2_x86_i486',
> +                       '!BR2_x86_i586',
> +                       '!BR2_x86_x1000'],
> +        'test_options': ['BR2_i386',
> +                         'BR2_x86_i686'],
> +        'prefix': 'i686',
> +    },
> +    'xtensa-lx60': {
> +        'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
> +        'prefix': 'xtensa',
> +    },
> +}
> +
> +
> +class Toolchain:
> +    def __init__(self, arch, libc, variant, version):
> +        self.arch = arch
> +        self.libc = libc
> +        self.variant = variant
> +        self.version = version
> +        self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
> +        self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
> +            (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
> +        self.fragment = requests.get(self.fragment_url).text.split("\n")
> +        self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
> +
> +    @property
> +    def tarball_url(self):
> +        return os.path.join(BASE_URL, self.arch, "tarballs",
> +                            self.fname_prefix + ".tar.bz2")
> +
> +    @property
> +    def hash_url(self):
> +        return os.path.join(BASE_URL, self.arch, "tarballs",
> +                            self.fname_prefix + ".sha256")
> +
> +    @property
> +    def fragment_url(self):
> +        return os.path.join(BASE_URL, self.arch, "fragments",
> +                            self.fname_prefix + ".frag")
> +
> +    def gen_config_in_options(self, f):
> +        f.write("config %s\n" % self.option_name)
> +        f.write("\tbool \"%s %s %s %s\"\n" %
> +                (self.arch, self.libc, self.variant, self.version))
> +        for c in arches[self.arch]['conditions']:
> +            f.write("\tdepends on %s\n" % c)
> +        selects = []
> +        for frag in self.fragment:
> +            # libc type
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
> +                selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
> +            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
> +                selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
> +                # all glibc toolchains have RPC support
> +                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
> +            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
> +                selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
> +
> +            # gcc version
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
> +                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
> +                assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
> +                selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
> +
> +            # kernel headers version
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
> +                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
> +                assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
> +                selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
> +
> +            # C++
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
> +                selects.append("BR2_INSTALL_LIBSTDCPP")
> +
> +            # SSP
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
> +                selects.append("BR2_TOOLCHAIN_HAS_SSP")
> +
> +            # wchar
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
> +                selects.append("BR2_USE_WCHAR")
> +
> +            # locale
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
> +                # locale implies the availability of wchar
> +                selects.append("BR2_USE_WCHAR")
> +                selects.append("BR2_ENABLE_LOCALE")
> +
> +            # thread support
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
> +                selects.append("BR2_TOOLCHAIN_HAS_THREADS")
> +
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
> +                selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
> +
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
> +                selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
> +
> +            # RPC
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
> +                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
> +
> +            # D language
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
> +                selects.append("BR2_TOOLCHAIN_HAS_DLANG")
> +
> +            # fortran
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
> +                selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
> +
> +            # OpenMP
> +            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
> +                selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
> +
> +        for select in selects:
> +            f.write("\tselect %s\n" % select)
> +
> +        f.write("\thelp\n")
> +
> +        desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
> +            (self.arch, self.libc)
> +
> +        if self.variant == "stable":
> +            desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
> +        else:
> +            desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
> +
> +        f.write(textwrap.fill(desc, width=62, initial_indent="\t  ", subsequent_indent="\t  ") + "\n")
> +        f.write("\n")
> +        f.write("\t  https://toolchains.bootlin.com/\n")
> +
> +        f.write("\n")
> +
> +    def gen_mk(self, f):
> +        f.write("ifeq ($(%s),y)\n" % self.option_name)
> +        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
> +        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
> +                (self.arch, self.libc, self.variant))
> +        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
> +                os.path.join(BASE_URL, self.arch, "tarballs"))
> +        f.write("endif\n\n")
> +        pass
> +
> +    def gen_hash(self, f):
> +        f.write("# From %s\n" % self.hash_url)
> +        f.write("sha256  %s  %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
> +
> +    def gen_test(self, f):
> +        if self.variant == "stable":
> +            variant = "Stable"
> +        else:
> +            variant = "BleedingEdge"
> +        testname = "TestExternalToolchainBootlin" + \
> +            self.arch.replace("-", "").capitalize() + \
> +            self.libc.capitalize() + variant
> +        f.write("\n\n")
> +        f.write("class %s(TestExternalToolchain):\n" % testname)
> +        f.write("    config = \"\"\"\n")
> +        if 'test_options' in arches[self.arch]:
> +            test_options = arches[self.arch]['test_options']
> +        else:
> +            test_options = arches[self.arch]['conditions']
> +        for opt in test_options:
> +            if opt.startswith("!"):
> +                f.write("        # %s is not set\n" % opt[1:])
> +            else:
> +                f.write("        %s=y\n" % opt)
> +        f.write("        BR2_TOOLCHAIN_EXTERNAL=y\n")
> +        f.write("        BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
> +        f.write("        %s=y\n" % self.option_name)
> +        f.write("        # BR2_TARGET_ROOTFS_TAR is not set\n")
> +        f.write("        \"\"\"\n")
> +        f.write("    toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
> +        f.write("\n")
> +        f.write("    def test_run(self):\n")
> +        f.write("        TestExternalToolchain.common_check(self)\n")
> +
> +    def __repr__(self):
> +        return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
> +            (self.arch, self.libc, self.variant, self.version, self.option_name)
> +
> +
> +def get_toolchains():
> +    toolchains = list()
> +    for arch, details in arches.items():
> +        print(arch)
> +        url = os.path.join(BASE_URL, arch, "available_toolchains")
> +        cwd, listing = htmllistparse.fetch_listing(url)
> +        fnames = [f.name for f in listing]
> +        # Sorting the list so we have the most recent version last
> +        fnames.sort()
> +        # This dict will allow us to keep only the latest version for
> +        # each toolchain.
> +        tmp = dict()
> +        for fname in fnames:
> +            parts = fname.split('--')
> +            assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
> +            libc = parts[1]
> +            if parts[2].startswith("stable-"):
> +                variant = "stable"
> +                version = parts[2][len("stable-"):]
> +            elif parts[2].startswith("bleeding-edge-"):
> +                variant = "bleeding-edge"
> +                version = parts[2][len("bleeding-edge-"):]
> +            tmp[(arch, libc, variant)] = version
> +
> +        toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
> +
> +    return toolchains
> +
> +
> +def gen_config_in_options(toolchains, fpath):
> +    with open(fpath, "w") as f:
> +        f.write(AUTOGENERATED_COMMENT)
> +
> +        f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
> +        f.write("\tbool\n")
> +        for arch, details in arches.items():
> +            f.write("\tdefault y if %s\n" % " && ".join(details['conditions']))
> +        f.write("\n")
> +
> +        f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
> +
> +        f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
> +        f.write("\tdefault \"$(ARCH)-linux\"\n")
> +
> +        f.write("\n")
> +
> +        f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
> +        f.write("\tdefault \"toolchain-external-bootlin\"\n")
> +
> +        f.write("\n")
> +
> +        f.write("choice\n")
> +        f.write("\tprompt \"Bootlin toolchain variant\"\n")
> +
> +        for toolchain in toolchains:
> +            toolchain.gen_config_in_options(f)
> +
> +        f.write("endchoice\n")
> +        f.write("endif\n")
> +
> +
> +def gen_mk(toolchains, fpath):
> +    with open(fpath, "w") as f:
> +        f.write("#" * 80 + "\n")
> +        f.write("#\n")
> +        f.write("# toolchain-external-bootlin\n")
> +        f.write("#\n")
> +        f.write("#" * 80 + "\n")
> +        f.write("\n")
> +        f.write(AUTOGENERATED_COMMENT)
> +        for toolchain in toolchains:
> +            toolchain.gen_mk(f)
> +        f.write("$(eval $(toolchain-external-package))\n")
> +
> +
> +def gen_hash(toolchains, fpath):
> +    with open(fpath, "w") as f:
> +        f.write(AUTOGENERATED_COMMENT)
> +        for toolchain in toolchains:
> +            toolchain.gen_hash(f)
> +
> +
> +def gen_runtime_test(toolchains, fpath):
> +    with open(fpath, "w") as f:
> +        f.write(AUTOGENERATED_COMMENT)
> +        f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
> +        for toolchain in toolchains:
> +            toolchain.gen_test(f)
> +
> +
> +def gen_toolchains(toolchains):
> +    maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
> +    gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
> +    gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
> +    gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
> +    gen_runtime_test(toolchains,
> +                     os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
> +
> +
> +toolchains = get_toolchains()
> +gen_toolchains(toolchains)
> -- 
> 2.26.2
> 

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'



More information about the buildroot mailing list