[PATCH 1/1] xargs: support -P <N> (parallel execution)
Johannes Schindelin
johannes.schindelin at gmx.de
Wed Aug 23 15:25:39 UTC 2017
The GNU variant of xargs supports a special, non-POSIX extension to run
processes in parallel, triggered by the -P <N> option.
This feature comes in handy e.g. when running Git's test suite outside
of the development environment using only BusyBox (which does not
support `make` nor `prove`, Git's go-to solutions for running the test
suite in parallel).
It so just happens that this patch also addresses the feature requested
in https://bugs.busybox.net/show_bug.cgi?id=9511.
Signed-off-by: Johannes Schindelin <johannes.schindelin at gmx.de>
---
findutils/xargs.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 111 insertions(+), 1 deletion(-)
diff --git a/findutils/xargs.c b/findutils/xargs.c
index c3d37a64d..cf9a8d474 100644
--- a/findutils/xargs.c
+++ b/findutils/xargs.c
@@ -60,6 +60,13 @@
//config: depends on XARGS
//config: help
//config: Support -I STR and -i[STR] options.
+//config:
+//config:config FEATURE_XARGS_SUPPORT_PARALLEL
+//config: bool "Enable -P N: processes to run in parallel"
+//config: default y
+//config: depends on XARGS
+//config: help
+//config: Support -P N option.
//applet:IF_XARGS(APPLET_NOEXEC(xargs, xargs, BB_DIR_USR_BIN, BB_SUID_DROP, xargs))
@@ -100,6 +107,10 @@ struct globals {
#endif
const char *eof_str;
int idx;
+#if ENABLE_FEATURE_XARGS_SUPPORT_PARALLEL
+ int max_procs, running_procs;
+ pid_t *procs;
+#endif
} FIX_ALIASING;
#define G (*(struct globals*)bb_common_bufsiz1)
#define INIT_G() do { \
@@ -107,14 +118,92 @@ struct globals {
G.eof_str = NULL; /* need to clear by hand because we are NOEXEC applet */ \
IF_FEATURE_XARGS_SUPPORT_REPL_STR(G.repl_str = "{}";) \
IF_FEATURE_XARGS_SUPPORT_REPL_STR(G.eol_ch = '\n';) \
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL(G.max_procs = 1;) \
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL(G.running_procs = 0;) \
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL(G.procs = NULL;) \
} while (0)
+#if ENABLE_FEATURE_XARGS_SUPPORT_PARALLEL
+static int wait_for_slot(int *idx)
+{
+ int status;
+ pid_t pid;
+
+ /* if less than max_procs running, set status to 0, return next free slot */
+ if (G.running_procs < G.max_procs) {
+ *idx = G.running_procs++;
+ return 0;
+ }
+
+ pid = safe_waitpid(-1, &status, 0);
+ if (pid < 0)
+ return pid;
+
+ for (*idx = 0; *idx < G.max_procs; (*idx)++)
+ if (G.procs[*idx] == pid) {
+ G.procs[*idx] = 0;
+ return WEXITSTATUS(status);
+ }
+
+ bb_error_msg("waitpid returned %"PRIu64" but we did not spawn it",
+ (uint64_t)pid);
+
+ return -1;
+}
+
+static int reap_remaining(void)
+{
+ int status, ret = 0, ret2;
+ while (G.running_procs) {
+ pid_t pid = safe_waitpid(-1, &status, 0);
+ if (pid < 0)
+ return errno == ENOENT ? 127 : 126;
+ G.running_procs--;
+ status = WEXITSTATUS(status);
+ if (status == 255)
+ ret2 = 124;
+ else if (status >= 0x180)
+ ret2 = 125;
+ else if (status)
+ ret2 = 123;
+ else
+ ret2 = 0;
+ if (ret < ret2)
+ ret = ret2;
+ }
+ return ret;
+}
+#endif /* SUPPORT_PARALLEL */
+
static int xargs_exec(void)
{
int status;
+#if !ENABLE_FEATURE_XARGS_SUPPORT_PARALLEL
status = spawn_and_wait(G.args);
+#else
+ if (!G.max_procs) {
+ pid_t p = spawn(G.args);
+ if (p == -1)
+ status = -1;
+ else {
+ status = 0;
+ G.running_procs++;
+ }
+ } else if (G.max_procs > 1) {
+ int idx = -1;
+ status = wait_for_slot(&idx);
+ if (status >= 0 && status < 0x180) {
+ pid_t p = spawn(G.args);
+ if (p < 0)
+ status = -1;
+ else
+ G.procs[idx] = p;
+ }
+ } else
+ status = spawn_and_wait(G.args);
+#endif
if (status < 0) {
bb_simple_perror_msg(G.args[0]);
return errno == ENOENT ? 127 : 126;
@@ -436,6 +525,9 @@ static int xargs_ask_confirmation(void)
//usage: IF_FEATURE_XARGS_SUPPORT_REPL_STR(
//usage: "\n -I STR Replace STR within PROG ARGS with input line"
//usage: )
+//usage: IF_FEATURE_XARGS_SUPPORT_PARALLEL(
+//usage: "\n -P N Run up to N processes in parallel"
+//usage: )
//usage: IF_FEATURE_XARGS_SUPPORT_TERMOPT(
//usage: "\n -x Exit if size is exceeded"
//usage: )
@@ -473,7 +565,8 @@ enum {
IF_FEATURE_XARGS_SUPPORT_CONFIRMATION("p") \
IF_FEATURE_XARGS_SUPPORT_TERMOPT( "x") \
IF_FEATURE_XARGS_SUPPORT_ZERO_TERM( "0") \
- IF_FEATURE_XARGS_SUPPORT_REPL_STR( "I:i::")
+ IF_FEATURE_XARGS_SUPPORT_REPL_STR( "I:i::") \
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL( "P:")
int xargs_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
int xargs_main(int argc, char **argv)
@@ -492,6 +585,7 @@ int xargs_main(int argc, char **argv)
#else
#define read_args process_stdin
#endif
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL(char *opt_P;)
INIT_G();
@@ -499,6 +593,7 @@ int xargs_main(int argc, char **argv)
"no-run-if-empty\0" No_argument "r",
&max_args, &max_chars, &G.eof_str, &G.eof_str
IF_FEATURE_XARGS_SUPPORT_REPL_STR(, &G.repl_str, &G.repl_str)
+ IF_FEATURE_XARGS_SUPPORT_PARALLEL(, &opt_P)
);
/* -E ""? You may wonder why not just omit -E?
@@ -512,6 +607,14 @@ int xargs_main(int argc, char **argv)
IF_FEATURE_XARGS_SUPPORT_REPL_STR(G.eol_ch = '\0';)
}
+#if ENABLE_FEATURE_XARGS_SUPPORT_PARALLEL
+ if (opt_P) {
+ G.max_procs = xatoi(opt_P);
+ if (G.max_procs > 1)
+ G.procs = xmalloc(sizeof(G.procs[0]) * G.max_procs);
+ }
+#endif
+
argv += optind;
argc -= optind;
if (!argv[0]) {
@@ -633,6 +736,13 @@ int xargs_main(int argc, char **argv)
overlapping_strcpy(buf, rem);
} /* while */
+#if ENABLE_FEATURE_XARGS_SUPPORT_PARALLEL
+ i = reap_remaining();
+ /* old child_error can be overridden by more serious error */
+ if (i > child_error)
+ child_error = i;
+#endif
+
if (ENABLE_FEATURE_CLEAN_UP) {
free(G.args);
free(buf);
--
2.14.1.windows.1.11.gc06fee21d46
More information about the busybox
mailing list