[Buildroot] [PATCH v5 1/1] Makefile: Parallelize glibc locale generation

Yann E. MORIN yann.morin.1998 at free.fr
Sat Jul 17 09:43:50 UTC 2021


Gleb, All,

+Peter, Thomas, Arnout: for review. I was too involved in setting up
this change, so I am not in the best position to correctly review it.

Of course, if anyone else want to review/test this, they are welcome to
do so!

Regards,
Yann E. MORIN.

On 2021-01-03 17:15 +0000, Gleb Mazovetskiy spake thusly:
> Parallelizes locale generation based on `BR2_JLEVEL` setting.
> 
> Locale generation always runs during the finalize stage and can consume
> a significant amount of time. Parallelizing it greatly reduces that time
> on multi-core machines.
> 
> To parallelize it, we first invoke `localedef` for every locale in
> parallel with the `--no-archive` option. This creates the intermediate
> locale data instead of writing to the finally archive directly.
> 
> Then, we invoke `localedef` again once to create the archive from the
> intermediate compiled locale data files.
> 
> We have to do it this way because `localedef` does not do any locking
> when writing to the archive file, so calling it without `--no-archive`
> concurrently could result in a corrupt archive file or an archive file
> that is missing some locales.
> 
> Signed-off-by: Gleb Mazovetskiy <glex.spb at gmail.com>
> ---
>  Makefile                          | 29 +++++++---------------
>  support/misc/gen-glibc-locales.mk | 41 +++++++++++++++++++++++++++++++
>  2 files changed, 50 insertions(+), 20 deletions(-)
>  create mode 100644 support/misc/gen-glibc-locales.mk
> 
> diff --git a/Makefile b/Makefile
> index 4d334adcd6..dbeabf9e61 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -661,32 +661,21 @@ endef
>  TARGET_FINALIZE_HOOKS += TOOLCHAIN_ECLIPSE_REGISTER
>  endif
>  
> -# Generate locale data. Basically, we call the localedef program
> -# (built by the host-localedef package) for each locale. The input
> -# data comes preferably from the toolchain, or if the toolchain does
> -# not have them (Linaro toolchains), we use the ones available on the
> -# host machine.
> +# Generate locale data.
>  ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y)
>  GLIBC_GENERATE_LOCALES = $(call qstrip,$(BR2_GENERATE_LOCALE))
>  ifneq ($(GLIBC_GENERATE_LOCALES),)
>  PACKAGES += host-localedef
>  
>  define GENERATE_GLIBC_LOCALES
> -	$(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/
> -	$(Q)for locale in $(GLIBC_GENERATE_LOCALES) ; do \
> -		inputfile=`echo $${locale} | cut -f1 -d'.'` ; \
> -		charmap=`echo $${locale} | cut -f2 -d'.' -s` ; \
> -		if test -z "$${charmap}" ; then \
> -			charmap="UTF-8" ; \
> -		fi ; \
> -		echo "Generating locale $${inputfile}.$${charmap}" ; \
> -		I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
> -		$(HOST_DIR)/bin/localedef \
> -			--prefix=$(TARGET_DIR) \
> -			--$(call LOWERCASE,$(BR2_ENDIAN))-endian \
> -			-i $${inputfile} -f $${charmap} \
> -			$${locale} ; \
> -	done
> +	$(TARGET_CONFIGURE_OPTS) \
> +	$(MAKE) -j$(PARALLEL_JOBS) -f support/misc/gen-glibc-locales.mk \
> +		HOST_DIR=$(HOST_DIR) \
> +		TARGET_DIR=$(TARGET_DIR) \
> +		STAGING_DIR=$(STAGING_DIR) \
> +		ENDIAN=$(call LOWERCASE,$(BR2_ENDIAN)) \
> +		LOCALES="$(GLIBC_GENERATE_LOCALES)" \
> +		Q=$(Q)
>  endef
>  TARGET_FINALIZE_HOOKS += GENERATE_GLIBC_LOCALES
>  endif
> diff --git a/support/misc/gen-glibc-locales.mk b/support/misc/gen-glibc-locales.mk
> new file mode 100644
> index 0000000000..3db1e0dc11
> --- /dev/null
> +++ b/support/misc/gen-glibc-locales.mk
> @@ -0,0 +1,41 @@
> +# Generates glibc locale data for target.
> +
> +inputfile = $(firstword $(subst ., ,$(1)))
> +charmap = $(or $(word 2,$(subst ., ,$(1))),UTF-8)
> +
> +# Packages all the generated locale data into the final archive.
> +#
> +# We sort the file names to produce consistent output regardless of
> +# the `find` outputs order.
> +$(TARGET_DIR)/usr/lib/locale/locale-archive: $(LOCALES)
> +	$(Q)rm -f $(@)
> +	$(Q)find $(TARGET_DIR)/usr/lib/locale/ -maxdepth 1 -mindepth 1 -type d -print0 \
> +	| sort -z \
> +	| xargs -0 \
> +		$(HOST_DIR)/bin/localedef \
> +			--prefix=$(TARGET_DIR) \
> +			--$(ENDIAN)-endian \
> +			--add-to-archive
> +
> +# Generates locale data for each locale.
> +#
> +# The input data comes preferably from the toolchain, or if the toolchain
> +# does not have them (Linaro toolchains), we use the ones available on the
> +# host machine.
> +#
> +# Uses `localedef`, which is built by the `host-localedef` package.
> +$(LOCALES): | $(TARGET_DIR)/usr/lib/locale/
> +	$(Q)echo "Generating locale $(@)"
> +	$(Q)I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
> +	$(HOST_DIR)/bin/localedef \
> +		--prefix=$(TARGET_DIR) \
> +		--$(ENDIAN)-endian \
> +		--no-archive \
> +		-i $(call inputfile,$(@)) \
> +		-f $(call charmap,$(@)) \
> +		$(@)
> +
> +.PHONY: $(LOCALES)
> +
> +$(TARGET_DIR)/usr/lib/locale/:
> +	$(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/
> -- 
> 2.27.0
> 
> _______________________________________________
> buildroot mailing list
> buildroot at busybox.net
> http://lists.busybox.net/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'



More information about the buildroot mailing list